609 files changed, 63064 insertions, 13703 deletions

diff --git a/.gitignore b/.gitignore
index dc6738c079..f2a6dfb493 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,8 @@
 .dirstamp
 *.trs
 *.log
+# Calltool stuff
+.*.graph
 # Stuff made by our makefiles
 *.bak
 # Python droppings
@@ -38,6 +40,7 @@ uptime-*.json
 /Makefile
 /Makefile.in
 /aclocal.m4
+/ar-lib
 /autom4te.cache
 /build-stamp
 /compile
@@ -94,11 +97,6 @@ uptime-*.json
 /doc/tor.html
 /doc/tor.html.in
 /doc/tor.1.xml
-/doc/tor-fw-helper.1
-/doc/tor-fw-helper.1.in
-/doc/tor-fw-helper.html
-/doc/tor-fw-helper.html.in
-/doc/tor-fw-helper.1.xml
 /doc/tor-gencert.1
 /doc/tor-gencert.1.in
 /doc/tor-gencert.html
@@ -127,6 +125,9 @@ uptime-*.json
 /src/Makefile
 /src/Makefile.in
 
+# /src/trace
+/src/trace/libor-trace.a
+
 # /src/common/
 /src/common/Makefile
 /src/common/Makefile.in
@@ -172,6 +173,12 @@ uptime-*.json
 /src/or/libtor-testing.a
 /src/or/libtor.lib
 
+# /src/rust
+/src/rust/.cargo/config
+/src/rust/.cargo/registry
+/src/rust/target
+/src/rust/registry
+
 # /src/test
 /src/test/Makefile
 /src/test/Makefile.in
@@ -183,6 +190,7 @@ uptime-*.json
 /src/test/test-child
 /src/test/test-memwipe
 /src/test/test-ntor-cl
+/src/test/test-hs-ntor-cl
 /src/test/test-switch-id
 /src/test/test-timers
 /src/test/test_workqueue
@@ -191,11 +199,16 @@ uptime-*.json
 /src/test/test-bt-cl.exe
 /src/test/test-child.exe
 /src/test/test-ntor-cl.exe
+/src/test/test-hs-ntor-cl.exe
 /src/test/test-memwipe.exe
 /src/test/test-switch-id.exe
 /src/test/test-timers.exe
 /src/test/test_workqueue.exe
 
+# /src/test/fuzz
+/src/test/fuzz/fuzz-*
+/src/test/fuzz/lf-fuzz-*
+
 # /src/tools/
 /src/tools/tor-checkkey
 /src/tools/tor-resolve
@@ -214,12 +227,6 @@ uptime-*.json
 /src/trunnel/libor-trunnel-testing.a
 /src/trunnel/libor-trunnel.a
 
-# /src/tools/tor-fw-helper/
-/src/tools/tor-fw-helper/tor-fw-helper
-/src/tools/tor-fw-helper/tor-fw-helper.exe
-/src/tools/tor-fw-helper/Makefile
-/src/tools/tor-fw-helper/Makefile.in
-
 # /src/win32/
 /src/win32/Makefile
 /src/win32/Makefile.in
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000..8f80405650
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "src/ext/rust"]
+	path = src/ext/rust
+	url = https://git.torproject.org/tor-rust-dependencies
diff --git a/.travis.yml b/.travis.yml
index 6a3e1bfc01..8dcbf1fe9d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -60,6 +60,13 @@ env:
   global:
     ## The Travis CI environment allows us two cores, so let's use both.
     - MAKEFLAGS="-j 2"
+  matrix:
+    ## Leave at least one entry here or Travis seems to generate a
+    ## matrix entry with empty matrix environment variables.  Leaving
+    ## more than one entry causes unwanted matrix entries with
+    ## unspecified compilers.
+    - RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    # - RUST_OPTIONS=""
 
 matrix:
   ## Uncomment to allow the build to report success (with non-required
@@ -89,6 +96,8 @@ matrix:
   include:
     - compiler: gcc
     - compiler: gcc
+      env: RUST_OPTIONS=""
+    - compiler: gcc
       env: COVERAGE_OPTIONS="--enable-coverage"
     - compiler: gcc
       env: DISTCHECK="yes"
@@ -98,12 +107,15 @@ matrix:
     ## longer allows ptrace.
     - compiler: clang
       sudo: required
+    - compiler: clang
+      sudo: required
+      env: RUST_OPTIONS=""
 
 before_install:
   ## If we're on OSX, homebrew usually needs to updated first
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
   ## Download rustup
-  - curl -Ssf -o rustup.sh https://sh.rustup.rs
+  - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
   - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi
 
 install:
@@ -115,6 +127,14 @@ install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated xz         || brew upgrade xz;         }; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated libscrypt  || brew upgrade libscrypt;  }; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then { brew outdated zstd       || brew upgrade zstd;       }; fi
+  ## Install the stable channels of rustc and cargo and setup our toolchain environment
+  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain stable; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
+  ## Get some info about rustc and cargo
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which rustc; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then which cargo; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi
+  - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi
 
 script:
   - ./autogen.sh
diff --git a/ChangeLog b/ChangeLog
index 7a10a7ce96..b1f64c92f4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,2463 @@
-Changes in version 0.2.9.5-rc - 2016-1?-??
+Changes in version 0.3.1.3-alpha - 2017-06-08
+  Tor 0.3.1.3-alpha fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.1.3-alpha also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (torrc, crash):
+    - Fix a crash bug when using %include in torrc. Fixes bug 22417;
+      bugfix on 0.3.1.1-alpha. Patch by Daniel Pinto.
+
+  o Minor features (code style):
+    - Add "Falls through" comments to our codebase, in order to silence
+      GCC 7's -Wimplicit-fallthrough warnings. Patch from Andreas
+      Stieger. Closes ticket 22446.
+
+  o Minor features (diagnostic):
+    - Add logging messages to try to diagnose a rare bug that seems to
+      generate RSA->Ed25519 cross-certificates dated in the 1970s. We
+      think this is happening because of incorrect system clocks, but
+      we'd like to know for certain. Diagnostic for bug 22466.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (directory protocol):
+    - Check for libzstd >= 1.1, because older versions lack the
+      necessary streaming API. Fixes bug 22413; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (link handshake):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (storage directories):
+    - Always check for underflows in the cached storage directory usage.
+      If the usage does underflow, re-calculate it. Also, avoid a
+      separate underflow when the usage is not known. Fixes bug 22424;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (unit tests):
+    - The unit tests now pass on systems where localhost is misconfigured
+      to some IPv4 address other than 127.0.0.1. Fixes bug 6298; bugfix
+      on 0.0.9pre2.
+
+  o Documentation:
+    - Clarify the manpage for the (deprecated) torify script. Closes
+      ticket 6892.
+
+Changes in version 0.3.0.8 - 2017-06-08
+  Tor 0.3.0.8 fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.0.8 also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management, backport from 0.3.1.3-alpha):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha):
+    - Stop rejecting v3 hidden service descriptors because their size
+      did not match an old padding rule. Fixes bug 22447; bugfix on
+      tor-0.3.0.1-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (configuration, backport from 0.3.1.1-alpha):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (link handshake, backport from 0.3.1.3-alpha):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport from
+    0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.2.9.11 - 2017-06-08
+  Tor 0.2.9.11 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005. (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  Tor 0.2.9.11 also backports fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (future-proofing, backport from 0.3.0.7):
+    - Tor no longer refuses to download microdescriptors or descriptors if
+      they are listed as "published in the future".  This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor features (directory authorities, backport from 0.3.0.4-rc)
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port, backport from 0.3.0.6):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.0.7):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport
+    from 0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+Changes in version 0.2.8.14 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.7.8 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.2.6.12 - 2017-06-08
+  Tor 0.2.6.12 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.5.14 - 2017-06-08
+  Tor 0.2.5.14 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.4.29 - 2017-06-08
+  Tor 0.2.4.29 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.3.1.2-alpha - 2017-05-26
+  Tor 0.3.1.2-alpha is the second release in the 0.3.1.x series. It
+  fixes a few bugs found while testing 0.3.1.1-alpha, including a
+  memory corruption bug that affected relay stability.
+
+  o Major bugfixes (crash, relay):
+    - Fix a memory-corruption bug in relays that set MyFamily.
+      Previously, they would double-free MyFamily elements when making
+      the next descriptor or when changing their configuration. Fixes
+      bug 22368; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Log a better message when a directory authority replies to an
+      upload with an unexpected status code. Fixes bug 11121; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (memory leak, directory authority):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.3.1.1-alpha - 2017-05-22
+  Tor 0.3.1.1-alpha is the first release in the 0.3.1.x series. It
+  reduces the bandwidth usage for Tor's directory protocol, adds some
+  basic padding to resist netflow-based traffic analysis and to serve as
+  the basis of other padding in the future, and adds rust support to the
+  build system.
+
+  It also contains numerous other small features and improvements to
+  security, correctness, and performance.
+
+  Below are the changes since 0.3.0.7.
+
+  o Major features (directory protocol):
+    - Tor relays and authorities can now serve clients an abbreviated
+      version of the consensus document, containing only the changes
+      since an older consensus document that the client holds. Clients
+      now request these documents when available. When both client and
+      server use this new protocol, they will use far less bandwidth (up
+      to 94% less) to keep the client's consensus up-to-date. Implements
+      proposal 140; closes ticket 13339. Based on work by Daniel Martí.
+    - Tor can now compress directory traffic with lzma or with zstd
+      compression algorithms, which can deliver better bandwidth
+      performance. Because lzma is computationally expensive, it's only
+      used for documents that can be compressed once and served many
+      times. Support for these algorithms requires that tor is built
+      with the libzstd and/or liblzma libraries available. Implements
+      proposal 278; closes ticket 21662.
+    - Relays now perform the more expensive compression operations, and
+      consensus diff generation, in worker threads. This separation
+      avoids delaying the main thread when a new consensus arrives.
+
+  o Major features (experimental):
+    - Tor can now build modules written in Rust. To turn this on, pass
+      the "--enable-rust" flag to the configure script. It's not time to
+      get excited yet: currently, there is no actual Rust functionality
+      beyond some simple glue code, and a notice at startup to tell you
+      that Rust is running. Still, we hope that programmers and
+      packagers will try building Tor with Rust support, so that we can
+      find issues and solve portability problems. Closes ticket 22106.
+
+  o Major features (traffic analysis resistance):
+    - Connections between clients and relays now send a padding cell in
+      each direction every 1.5 to 9.5 seconds (tunable via consensus
+      parameters). This padding will not resist specialized
+      eavesdroppers, but it should be enough to make many ISPs' routine
+      network flow logging less useful in traffic analysis against
+      Tor users.
+
+      Padding is negotiated using Tor's link protocol, so both relays
+      and clients must upgrade for this to take effect. Clients may
+      still send padding despite the relay's version by setting
+      ConnectionPadding 1 in torrc, and may disable padding by setting
+      ConnectionPadding 0 in torrc. Padding may be minimized for mobile
+      users with the torrc option ReducedConnectionPadding. Implements
+      Proposal 251 and Section 2 of Proposal 254; closes ticket 16861.
+    - Relays will publish 24 hour totals of padding and non-padding cell
+      counts to their extra-info descriptors, unless PaddingStatistics 0
+      is set in torrc. These 24 hour totals are also rounded to
+      multiples of 10000.
+
+  o Major bugfixes (connection usage):
+    - We use NETINFO cells to try to determine if both relays involved
+      in a connection will agree on the canonical status of that
+      connection. We prefer the connections where this is the case for
+      extend cells, and try to close connections where relays disagree
+      on their canonical status early. Also, we now prefer the oldest
+      valid connection for extend cells. These two changes should reduce
+      the number of long-term connections that are kept open between
+      relays. Fixes bug 17604; bugfix on 0.2.5.5-alpha.
+    - Relays now log hourly statistics (look for
+      "channel_check_for_duplicates" lines) on the total number of
+      connections to other relays. If the number of connections per
+      relay is unexpectedly large, this log message is at notice level.
+      Otherwise it is at info.
+
+  o Major bugfixes (entry guards):
+    - Don't block bootstrapping when a primary bridge is offline and we
+      can't get its descriptor. Fixes bug 22325; fixes one case of bug
+      21969; bugfix on 0.3.0.3-alpha.
+
+  o Major bugfixes (linux TPROXY support):
+    - Fix a typo that had prevented TPROXY-based transparent proxying
+      from working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
+      Patch from "d4fq0fQAgoJ".
+
+  o Minor features (security, windows):
+    - Enable a couple of pieces of Windows hardening: one
+      (HeapEnableTerminationOnCorruption) that has been on-by-default
+      since Windows 8, and unavailable before Windows 7; and one
+      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
+      affect us, but shouldn't do any harm. Closes ticket 21953.
+
+  o Minor features (config options):
+    - Allow "%include" directives in torrc configuration files. These
+      directives import the settings from other files, or from all the
+      files in a directory. Closes ticket 1922. Code by Daniel Pinto.
+    - Make SAVECONF return an error when overwriting a torrc that has
+      includes. Using SAVECONF with the FORCE option will allow it to
+      overwrite torrc even if includes are used. Related to ticket 1922.
+    - Add "GETINFO config-can-saveconf" to tell controllers if SAVECONF
+      will work without the FORCE option. Related to ticket 1922.
+
+  o Minor features (controller):
+    - Warn the first time that a controller requests data in the long-
+      deprecated 'GETINFO network-status' format. Closes ticket 21703.
+
+  o Minor features (defaults):
+    - The default value for UseCreateFast is now 0: clients which
+      haven't yet received a consensus document will now use a proper
+      ntor handshake to talk to their directory servers whenever they
+      can. Closes ticket 21407.
+    - Onion key rotation and expiry intervals are now defined as a
+      network consensus parameter, per proposal 274. The default
+      lifetime of an onion key is increased from 7 to 28 days. Old onion
+      keys will expire after 7 days by default. This change will make
+      consensus diffs much smaller, and save significant bandwidth.
+      Closes ticket 21641.
+
+  o Minor features (fallback directory list):
+    - Update the fallback directory mirror whitelist and blacklist based
+      on operator emails. Closes task 21121.
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (hidden services, logging):
+    - Log a message when a hidden service descriptor has fewer
+      introduction points than specified in
+      HiddenServiceNumIntroductionPoints. Closes tickets 21598.
+    - Log a message when a hidden service reaches its introduction point
+      circuit limit, and when that limit is reset. Follow up to ticket
+      21594; closes ticket 21622.
+    - Warn user if multiple entries in EntryNodes and at least one
+      HiddenService are used together. Pinning EntryNodes along with a
+      hidden service can be possibly harmful; for instance see ticket
+      14917 or 21155. Closes ticket 21155.
+
+  o Minor features (linux seccomp2 sandbox):
+    - We now have a document storage backend compatible with the Linux
+      seccomp2 sandbox. This backend is used for consensus documents and
+      diffs between them; in the long term, we'd like to use it for
+      unparseable directory material too. Closes ticket 21645
+    - Increase the maximum allowed size passed to mprotect(PROT_WRITE)
+      from 1MB to 16MB. This was necessary with the glibc allocator in
+      order to allow worker threads to allocate more memory -- which in
+      turn is necessary because of our new use of worker threads for
+      compression. Closes ticket 22096.
+
+  o Minor features (logging):
+    - Log files are no longer created world-readable by default.
+      (Previously, most distributors would store the logs in a non-
+      world-readable location to prevent inappropriate access. This
+      change is an extra precaution.) Closes ticket 21729; patch
+      from toralf.
+
+  o Minor features (performance):
+    - Our Keccak (SHA-3) implementation now accesses memory more
+      efficiently, especially on little-endian systems. Closes
+      ticket 21737.
+    - Add an O(1) implementation of channel_find_by_global_id(), to
+      speed some controller functions.
+
+  o Minor features (relay, configuration):
+    - The MyFamily option may now be repeated as many times as desired,
+      for relays that want to configure large families. Closes ticket
+      4998; patch by Daniel Pinto.
+
+  o Minor features (safety):
+    - Add an explicit check to extrainfo_parse_entry_from_string() for
+      NULL inputs. We don't believe this can actually happen, but it may
+      help silence a warning from the Clang analyzer. Closes
+      ticket 21496.
+
+  o Minor features (testing):
+    - Add a "--disable-memory-sentinels" feature to help with fuzzing.
+      When Tor is compiled with this option, we disable a number of
+      redundant memory-safety failsafes that are intended to stop bugs
+      from becoming security issues. This makes it easier to hunt for
+      bugs that would be security issues without the failsafes turned
+      on. Closes ticket 21439.
+    - Add a general event-tracing instrumentation support to Tor. This
+      subsystem will enable developers and researchers to add fine-
+      grained instrumentation to their Tor instances, for use when
+      examining Tor network performance issues. There are no trace
+      events yet, and event-tracing is off by default unless enabled at
+      compile time. Implements ticket 13802.
+    - Improve our version parsing tests: add tests for typical version
+      components, add tests for invalid versions, including numeric
+      range and non-numeric prefixes. Unit tests 21278, 21450, and
+      21507. Partially implements 21470.
+
+  o Minor bugfixes (bandwidth accounting):
+    - Roll over monthly accounting at the configured hour and minute,
+      rather than always at 00:00. Fixes bug 22245; bugfix on 0.0.9rc1.
+      Found by Andrey Karpov with PVS-Studio.
+
+  o Minor bugfixes (code correctness):
+    - Accurately identify client connections by their lack of peer
+      authentication. This means that we bail out earlier if asked to
+      extend to a client. Follow-up to 21407. Fixes bug 21406; bugfix
+      on 0.2.4.23.
+
+  o Minor bugfixes (configuration):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection lifespan):
+    - Allow more control over how long TLS connections are kept open:
+      unify CircuitIdleTimeout and PredictedPortsRelevanceTime into a
+      single option called CircuitsAvailableTimeout. Also, allow the
+      consensus to control the default values for both this preference
+      and the lifespan of relay-to-relay connections. Fixes bug 17592;
+      bugfix on 0.2.5.5-alpha.
+    - Increase the initial circuit build timeout testing frequency, to
+      help ensure that ReducedConnectionPadding clients finish learning
+      a timeout before their orconn would expire. The initial testing
+      rate was set back in the days of TAP and before the Tor Browser
+      updater, when we had to be much more careful about new clients
+      making lots of circuits. With this change, a circuit build timeout
+      is learned in about 15-20 minutes, instead of 100-120 minutes.
+
+  o Minor bugfixes (controller):
+    - GETINFO onions/current and onions/detached no longer respond with
+      551 on empty lists. Fixes bug 21329; bugfix on 0.2.7.1-alpha.
+    - Trigger HS descriptor events on the control port when the client
+      fails to pick a hidden service directory for a hidden service.
+      This can happen if all the hidden service directories are in
+      ExcludeNodes, or they have all been queried within the last 15
+      minutes. Fixes bug 22042; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When rejecting a router descriptor for running an obsolete version
+      of Tor without ntor support, warn about the obsolete tor version,
+      not the missing ntor key. Fixes bug 20270; bugfix on 0.2.9.3-alpha.
+    - Prevent the shared randomness subsystem from asserting when
+      initialized by a bridge authority with an incomplete configuration
+      file. Fixes bug 21586; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (exit-side DNS):
+    - Fix an untriggerable assertion that checked the output of a
+      libevent DNS error, so that the assertion actually behaves as
+      expected. Fixes bug 22244; bugfix on 0.2.0.20-rc. Found by Andrey
+      Karpov using PVS-Studio.
+
+  o Minor bugfixes (fallback directories):
+    - Make the usage example in updateFallbackDirs.py actually work, and
+      explain what it does. Fixes bug 22270; bugfix on 0.3.0.3-alpha.
+    - Decrease the guard flag average required to be a fallback. This
+      allows us to keep relays that have their guard flag removed when
+      they restart. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+    - Decrease the minimum number of fallbacks to 100. Fixes bug 20913;
+      bugfix on 0.2.8.1-alpha.
+    - Make sure fallback directory mirrors have the same address, port,
+      and relay identity key for at least 30 days before they are
+      selected. Fixes bug 20913; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop printing a cryptic warning when a hidden service gets a
+      request to connect to a virtual port that it hasn't configured.
+      Fixes bug 16706; bugfix on 0.2.6.3-alpha.
+    - Simplify hidden service descriptor creation by using an existing
+      flag to check if an introduction point is established. Fixes bug
+      21599; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak at exit from the backtrace handler code.
+      Fixes bug 21788; bugfix on 0.2.5.2-alpha. Patch from Daniel Pinto.
+
+  o Minor bugfixes (protocol, logging):
+    - Downgrade a log statement about unexpected relay cells from "bug"
+      to "protocol warning", because there is at least one use case
+      where it can be triggered by a buggy tor implementation. Fixes bug
+      21293; bugfix on 0.1.1.14-alpha.
+
+  o Minor bugfixes (testing):
+    - Use unbuffered I/O for utility functions around the
+      process_handle_t type. This fixes unit test failures reported on
+      OpenBSD and FreeBSD. Fixes bug 21654; bugfix on 0.2.3.1-alpha.
+    - Make display of captured unit test log messages consistent. Fixes
+      bug 21510; bugfix on 0.2.9.3-alpha.
+    - Make test-network.sh always call chutney's test-network.sh.
+      Previously, this only worked on systems which had bash installed,
+      due to some bash-specific code in the script. Fixes bug 19699;
+      bugfix on 0.3.0.4-rc. Follow-up to ticket 21581.
+
+  o Minor bugfixes (voting consistency):
+    - Reject version numbers with non-numeric prefixes (such as +, -, or
+      whitespace). Disallowing whitespace prevents differential version
+      parsing between POSIX-based and Windows platforms. Fixes bug 21507
+      and part of 21508; bugfix on 0.0.8pre1.
+
+  o Minor bugfixes (windows, relay):
+    - Resolve "Failure from drain_fd: No error" warnings on Windows
+      relays. Fixes bug 21540; bugfix on 0.2.6.3-alpha.
+
+  o Code simplification and refactoring:
+    - Break up the 630-line function connection_dir_client_reached_eof()
+      into a dozen smaller functions. This change should help
+      maintainability and readability of the client directory code.
+    - Isolate our use of the openssl headers so that they are only
+      included from our crypto wrapper modules, and from tests that
+      examine those modules' internals. Closes ticket 21841.
+    - Simplify our API to launch directory requests, making it more
+      extensible and less error-prone. Now it's easier to add extra
+      headers to directory requests. Closes ticket 21646.
+    - Our base64 decoding functions no longer overestimate the output
+      space that they need when parsing unpadded inputs. Closes
+      ticket 17868.
+    - Remove unused "ROUTER_ADDED_NOTIFY_GENERATOR" internal value.
+      Resolves ticket 22213.
+    - The logic that directory caches use to spool request to clients,
+      serving them one part at a time so as not to allocate too much
+      memory, has been refactored for consistency. Previously there was
+      a separate spooling implementation per type of spoolable data. Now
+      there is one common spooling implementation, with extensible data
+      types. Closes ticket 21651.
+    - Tor's compression module now supports multiple backends. Part of
+      the implementation for proposal 278; closes ticket 21663.
+
+  o Documentation:
+    - Clarify the behavior of the KeepAliveIsolateSOCKSAuth sub-option.
+      Closes ticket 21873.
+    - Correct documentation about the default DataDirectory value.
+      Closes ticket 21151.
+    - Document the default behavior of NumEntryGuards and
+      NumDirectoryGuards correctly. Fixes bug 21715; bugfix
+      on 0.3.0.1-alpha.
+    - Document key=value pluggable transport arguments for Bridge lines
+      in torrc. Fixes bug 20341; bugfix on 0.2.5.1-alpha.
+    - Note that bandwidth-limiting options don't affect TCP headers or
+      DNS. Closes ticket 17170.
+
+  o Removed features (configuration options, all in ticket 22060):
+    - These configuration options are now marked Obsolete, and no longer
+      have any effect: AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ExcludeSingleHopRelays, FastFirstHopPK,
+      TLSECGroup, WarnUnsafeSocks. They were first marked as deprecated
+      in 0.2.9.2-alpha and have now been removed. The previous default
+      behavior is now always chosen; the previous (less secure) non-
+      default behavior is now unavailable.
+    - CloseHSClientCircuitsImmediatelyOnTimeout and
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout were deprecated in
+      0.2.9.2-alpha and now have been removed. HS circuits never close
+      on circuit build timeout; they have a longer timeout period.
+    - {Control,DNS,Dir,Socks,Trans,NATD,OR}ListenAddress were deprecated
+      in 0.2.9.2-alpha and now have been removed. Use the ORPort option
+      (and others) to configure listen-only and advertise-only addresses.
+
+  o Removed features (tools):
+    - We've removed the tor-checkkey tool from src/tools. Long ago, we
+      used it to help people detect RSA keys that were generated by
+      versions of Debian affected by CVE-2008-0166. But those keys have
+      been out of circulation for ages, and this tool is no longer
+      required. Closes ticket 21842.
+
+
+Changes in version 0.3.0.7 - 2017-05-15
+  Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions
+  of Tor 0.3.0.x, where an attacker could cause a Tor relay process
+  to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade;
+  clients are not affected.
+
+  o Major bugfixes (hidden service directory, security):
+    - Fix an assertion failure in the hidden service directory code, which
+      could be used by an attacker to remotely cause a Tor relay process to
+      exit. Relays running earlier versions of Tor 0.3.0.x should upgrade.
+      should upgrade. This security issue is tracked as TROVE-2017-002.
+      Fixes bug 22246; bugfix on 0.3.0.1-alpha.
+
+  o Minor features:
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (future-proofing):
+    - Tor no longer refuses to download microdescriptors or descriptors
+      if they are listed as "published in the future". This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.0.6 - 2017-04-26
+  Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series.
+
+  With the 0.3.0 series, clients and relays now use Ed25519 keys to
+  authenticate their link connections to relays, rather than the old
+  RSA1024 keys that they used before. (Circuit crypto has been
+  Curve25519-authenticated since 0.2.4.8-alpha.) We have also replaced
+  the guard selection and replacement algorithm to behave more robustly
+  in the presence of unreliable networks, and to resist guard-
+  capture attacks.
+
+  This series also includes numerous other small features and bugfixes,
+  along with more groundwork for the upcoming hidden-services revamp.
+
+  Per our stable release policy, we plan to support the Tor 0.3.0
+  release series for at least the next nine months, or for three months
+  after the first stable release of the 0.3.1 series: whichever is
+  longer. If you need a release with long-term support, we recommend
+  that you stay with the 0.2.9 series.
+
+  Below are the changes since 0.3.0.5-rc. For a list of all changes
+  since 0.2.9, see the ReleaseNotes file.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (crash prevention):
+    - Fix a (currently untriggerable, but potentially dangerous) crash
+      bug when base32-encoding inputs whose sizes are not a multiple of
+      5. Fixes bug 21894; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.5-rc - 2017-04-05
+  Tor 0.3.0.5-rc fixes a few remaining bugs, large and small, in the
+  0.3.0 release series.
+
+  This is the second release candidate in the Tor 0.3.0 series, and has
+  much fewer changes than the first. If we find no new bugs or
+  regressions here, the first stable 0.3.0 release will be nearly
+  identical to it.
+
+  o Major bugfixes (crash, directory connections):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (guard selection):
+    - Fix a guard selection bug where Tor would refuse to bootstrap in
+      some cases if the user swapped a bridge for another bridge in
+      their configuration file. Fixes bug 21771; bugfix on 0.3.0.1-alpha.
+      Reported by "torvlnt33r".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 7 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (compilation):
+    - Fix a warning when compiling hs_service.c. Previously, it had no
+      exported symbols when compiled for libor.a, resulting in a
+      compilation warning from clang. Fixes bug 21825; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Make hidden services check for failed intro point connections,
+      even when they have exceeded their intro point creation limit.
+      Fixes bug 21596; bugfix on 0.2.7.2-alpha. Reported by Alec Muffett.
+    - Make hidden services with 8 to 10 introduction points check for
+      failed circuits immediately after startup. Previously, they would
+      wait for 5 minutes before performing their first checks. Fixes bug
+      21594; bugfix on 0.2.3.9-alpha. Reported by Alec Muffett.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a memory leak when using GETCONF on a port option. Fixes bug
+      21682; bugfix on 0.3.0.3-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (tests):
+    - Run the entry_guard_parse_from_state_full() test with the time set
+      to a specific date. (The guard state that this test was parsing
+      contained guards that had expired since the test was first
+      written.) Fixes bug 21799; bugfix on 0.3.0.1-alpha.
+
+  o Documentation:
+    - Update the description of the directory server options in the
+      manual page, to clarify that a relay no longer needs to set
+      DirPort in order to be a directory cache. Closes ticket 21720.
+
+
+
+Changes in version 0.2.8.13 - 2017-03-03
+  Tor 0.2.8.13 backports a security fix from later Tor
+  releases.  Anybody running Tor 0.2.8.12 or earlier should upgrade to this
+  this release, if for some reason they cannot upgrade to a later
+  release series, and if they build Tor with the --enable-expensive-hardening
+  option.
+
+  Note that support for Tor 0.2.8.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.8.x series after 1 Jan 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.7.7 - 2017-03-03
+  Tor 0.2.7.7 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.7.6 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.7.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.7.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.6.11 - 2017-03-03
+  Tor 0.2.6.11 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.6.10 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.6.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.6.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+
+Changes in version 0.2.5.13 - 2017-03-03
+  Tor 0.2.5.13 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.5.13 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.5.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.5.x series after 1 May 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (crypto error-handling, backport from 0.2.7.2-alpha):
+    - Check for failures from crypto_early_init, and refuse to continue.
+      A previous typo meant that we could keep going with an
+      uninitialized crypto library, and would have OpenSSL initialize
+      its own PRNG. Fixes bug 16360; bugfix on 0.2.5.2-alpha, introduced
+      when implementing ticket 4900. Patch by "teor".
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.4.28 - 2017-03-03
+  Tor 0.2.4.28 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.4.27 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.4.x is ending soon: we will not issue
+  any fixes for the Tor 0.2.4.x series after 1 August 2017.  If you need
+  a Tor release series with long-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (DoS-resistance, backport from 0.2.7.1-alpha):
+    - Make it harder for attackers to overload hidden services with
+      introductions, by blocking multiple introduction requests on the
+      same circuit. Resolves ticket 15515.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.3.0.4-rc - 2017-03-01
+  Tor 0.3.0.4-rc fixes some remaining bugs, large and small, in the
+  0.3.0 release series, and introduces a few reliability features to
+  keep them from coming back.
+
+  This is the first release candidate in the Tor 0.3.0 series. If we
+  find no new bugs or regressions here, the first stable 0.3.0 release
+  will be nearly identical to it.
+
+  o Major bugfixes (bridges):
+    - When the same bridge is configured multiple times with the same
+      identity, but at different address:port combinations, treat those
+      bridge instances as separate guards. This fix restores the ability
+      of clients to configure the same bridge with multiple pluggable
+      transports. Fixes bug 21027; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service directory v3):
+    - Stop crashing on a failed v3 hidden service descriptor lookup
+      failure. Fixes bug 21471; bugfixes on tor-0.3.0.1-alpha.
+
+  o Major bugfixes (parsing):
+    - When parsing a malformed content-length field from an HTTP
+      message, do not read off the end of the buffer. This bug was a
+      potential remote denial-of-service attack against Tor clients and
+      relays. A workaround was released in October 2016, to prevent this
+      bug from crashing Tor. This is a fix for the underlying issue,
+      which should no longer matter (if you applied the earlier patch).
+      Fixes bug 20894; bugfix on 0.2.0.16-alpha. Bug found by fuzzing
+      using AFL (http://lcamtuf.coredump.cx/afl/).
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor feature (protocol versioning):
+    - Add new protocol version for proposal 224. HSIntro now advertises
+      version "3-4" and HSDir version "1-2". Fixes ticket 20656.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (reliability, crash):
+    - Try better to detect problems in buffers where they might grow (or
+      think they have grown) over 2 GB in size. Diagnostic for
+      bug 21369.
+
+  o Minor features (testing):
+    - During 'make test-network-all', if tor logs any warnings, ask
+      chutney to output them. Requires a recent version of chutney with
+      the 21572 patch. Implements 21570.
+
+  o Minor bugfixes (certificate expiration time):
+    - Avoid using link certificates that don't become valid till some
+      time in the future. Fixes bug 21420; bugfix on 0.2.4.11-alpha
+
+  o Minor bugfixes (code correctness):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+    - Remove a redundant check for the UseEntryGuards option from the
+      options_transition_affects_guards() function. Fixes bug 21492;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Allow relays to use directory mirrors without a DirPort: these
+      relays need to be contacted over their ORPorts using a begindir
+      connection. Fixes one case of bug 20711; bugfix on 0.2.8.2-alpha.
+    - Clarify the message logged when a remote relay is unexpectedly
+      missing an ORPort or DirPort: users were confusing this with a
+      local port. Fixes another case of bug 20711; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (guards):
+    - Don't warn about a missing guard state on timeout-measurement
+      circuits: they aren't supposed to be using guards. Fixes an
+      instance of bug 21007; bugfix on 0.3.0.1-alpha.
+    - Silence a BUG() warning when attempting to use a guard whose
+      descriptor we don't know, and make this scenario less likely to
+      happen. Fixes bug 21415; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Pass correct buffer length when encoding legacy ESTABLISH_INTRO
+      cells. Previously, we were using sizeof() on a pointer, instead of
+      the real destination buffer. Fortunately, that value was only used
+      to double-check that there was enough room--which was already
+      enforced elsewhere. Fixes bug 21553; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Fix Raspbian build issues related to missing socket errno in
+      test_util.c. Fixes bug 21116; bugfix on tor-0.2.8.2. Patch
+      by "hein".
+    - Rename "make fuzz" to "make test-fuzz-corpora", since it doesn't
+      actually fuzz anything. Fixes bug 21447; bugfix on 0.3.0.3-alpha.
+    - Use bash in src/test/test-network.sh. This ensures we reliably
+      call chutney's newer tools/test-network.sh when available. Fixes
+      bug 21562; bugfix on 0.2.9.1-alpha.
+
+  o Documentation:
+    - Small fixes to the fuzzing documentation. Closes ticket 21472.
+
+
+Changes in version 0.2.9.10 - 2017-03-01
+  Tor 0.2.9.10 backports a security fix from later Tor release.  It also
+  includes fixes for some major issues affecting directory authorities,
+  LibreSSL compatibility, and IPv6 correctness.
+
+  The Tor 0.2.9.x release series is now marked as a long-term-support
+  series.  We intend to backport security fixes to 0.2.9.x until at
+  least January of 2020.
+
+  o Major bugfixes (directory authority, 0.3.0.3-alpha):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (IPv6 Exits, backport from 0.3.0.3-alpha):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing, also in 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (directory authorities, also in 0.3.0.4-rc):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (portability, compilation, backport from 0.3.0.3-alpha):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor bugfixes (code correctness, also in 0.3.0.4-rc):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (tor-resolve, backport from 0.3.0.3-alpha):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+
+Changes in version 0.3.0.3-alpha - 2017-02-03
+  Tor 0.3.0.3-alpha fixes a few significant bugs introduced over the
+  0.3.0.x development series, including some that could cause
+  authorities to behave badly. There is also a fix for a longstanding
+  bug that could prevent IPv6 exits from working. Tor 0.3.0.3-alpha also
+  includes some smaller features and bugfixes.
+
+  The Tor 0.3.0.x release series is now in patch-freeze: no additional
+  features will be considered for inclusion in 0.3.0.x. We suspect that
+  some bugs will probably remain, however, and we encourage people to
+  test this release.
+
+  o Major bugfixes (directory authority):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+    - When deciding whether we have just found a router to be reachable,
+      do not penalize it for not having performed an Ed25519 link
+      handshake if it does not claim to support an Ed25519 handshake.
+      Previously, we would treat such relays as non-running. Fixes bug
+      21107; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (entry guards):
+    - Stop trying to build circuits through entry guards for which we
+      have no descriptor. Also, stop crashing in the case that we *do*
+      accidentally try to build a circuit in such a state. Fixes bug
+      21242; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (IPv6 Exits):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Minor feature (client):
+    - Enable IPv6 traffic on the SocksPort by default. To disable this,
+      a user will have to specify "NoIPv6Traffic". Closes ticket 21269.
+
+  o Minor feature (fallback scripts):
+    - Add a check_existing mode to updateFallbackDirs.py, which checks
+      if fallbacks in the hard-coded list are working. Closes ticket
+      20174. Patch by haxxpop.
+
+  o Minor features (ciphersuite selection):
+    - Clients now advertise a list of ciphersuites closer to the ones
+      preferred by Firefox. Closes part of ticket 15426.
+    - Allow relays to accept a wider range of ciphersuites, including
+      chacha20-poly1305 and AES-CCM. Closes the other part of 15426.
+
+  o Minor features (controller, configuration):
+    - Each of the *Port options, such as SocksPort, ORPort, ControlPort,
+      and so on, now comes with a __*Port variant that will not be saved
+      to the torrc file by the controller's SAVECONF command. This
+      change allows TorBrowser to set up a single-use domain socket for
+      each time it launches Tor. Closes ticket 20956.
+    - The GETCONF command can now query options that may only be
+      meaningful in context-sensitive lists. This allows the controller
+      to query the mixed SocksPort/__SocksPort style options introduced
+      in feature 20956. Implements ticket 21300.
+
+  o Minor features (portability, compilation):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor features (relay):
+    - We now allow separation of exit and relay traffic to different
+      source IP addresses, using the OutboundBindAddressExit and
+      OutboundBindAddressOR options respectively. Closes ticket 17975.
+      Written by Michael Sonntag.
+
+  o Minor bugfix (logging):
+    - Don't recommend the use of Tor2web in non-anonymous mode.
+      Recommending Tor2web is a bad idea because the client loses all
+      anonymity. Tor2web should only be used in specific cases by users
+      who *know* and understand the issues. Fixes bug 21294; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (client):
+    - Always recover from failures in extend_info_from_node(), in an
+      attempt to prevent any recurrence of bug 21242. Fixes bug 21372;
+      bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (client, entry guards):
+    - Fix a bug warning (with backtrace) when we fail a channel that
+      circuits to fallback directories on it. Fixes bug 21128; bugfix
+      on 0.3.0.1-alpha.
+    - Fix a spurious bug warning (with backtrace) when removing an
+      expired entry guard. Fixes bug 21129; bugfix on 0.3.0.1-alpha.
+    - Fix a bug of the new guard algorithm where tor could stall for up
+      to 10 minutes before retrying a guard after a long period of no
+      network. Fixes bug 21052; bugfix on 0.3.0.1-alpha.
+    - Do not try to build circuits until we have descriptors for our
+      primary entry guards. Related to fix for bug 21242.
+
+  o Minor bugfixes (configure, autoconf):
+    - Rename the configure option --enable-expensive-hardening to
+      --enable-fragile-hardening. Expensive hardening makes the tor
+      daemon abort when some kinds of issues are detected. Thus, it
+      makes tor more at risk of remote crashes but safer against RCE or
+      heartbleed bug category. We now try to explain this issue in a
+      message from the configure script. Fixes bug 21290; bugfix
+      on 0.2.5.4-alpha.
+
+  o Minor bugfixes (controller):
+    - Restore the (deprecated) DROPGUARDS controller command. Fixes bug
+      20824; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Clean up the code for expiring intro points with no associated
+      circuits. It was causing, rarely, a service with some expiring
+      introduction points to not open enough additional introduction
+      points. Fixes part of bug 21302; bugfix on 0.2.7.2-alpha.
+    - Stop setting the torrc option HiddenServiceStatistics to "0" just
+      because we're not a bridge or relay. Instead, we preserve whatever
+      value the user set (or didn't set). Fixes bug 21150; bugfix
+      on 0.2.6.2-alpha.
+    - Resolve two possible underflows which could lead to creating and
+      closing a lot of introduction point circuits in a non-stop loop.
+      Fixes bug 21302; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Use "OpenBSD" compiler macro instead of "OPENBSD" or "__OpenBSD__".
+      It is supported by OpenBSD itself, and also by most OpenBSD
+      variants (such as Bitrig). Fixes bug 20980; bugfix
+      on 0.1.2.1-alpha.
+    - When mapping a file of length greater than SIZE_MAX, do not
+      silently truncate its contents. This issue could occur on 32 bit
+      systems with large file support and files which are larger than 4
+      GB. Fixes bug 21134; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+  o Minor bugfixes (Windows services):
+    - Be sure to initialize the monotonic time subsystem before using
+      it, even when running as an NT service. Fixes bug 21356; bugfix
+      on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.2-alpha - 2017-01-23
+  Tor 0.3.0.2-alpha fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  Tor 0.3.0.2-alpha also improves how exit relays and clients handle DNS
+  time-to-live values, makes directory authorities enforce the 1-to-1
+  mapping of relay RSA identity keys to ED25519 identity keys, fixes a
+  client-side onion service reachability bug, does better at selecting
+  the set of fallback directories, and more.
+
+  o Major bugfixes (security, also in 0.2.9.9):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided."  This hardening option, like
+      others, can turn survivable bugs into crashes--and having it on by
+      default made a (relatively harmless) integer overflow bug into a
+      denial-of-service bug. Fixes bug 21278 (TROVE-2017-001); bugfix on
+      0.2.9.1-alpha.
+
+  o Major features (security):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major features (directory authority, security):
+    - The default for AuthDirPinKeys is now 1: directory authorities
+      will reject relays where the RSA identity key matches a previously
+      seen value, but the Ed25519 key has changed. Closes ticket 18319.
+
+  o Major bugfixes (client, guard, crash):
+    - In circuit_get_global_origin_list(), return the actual list of
+      origin circuits. The previous version of this code returned the
+      list of all the circuits, and could have caused strange bugs,
+      including possible crashes. Fixes bug 21118; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (client, onion service, also in 0.2.9.9):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Major bugfixes (DNS):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Minor features (controller):
+    - Add "GETINFO sr/current" and "GETINFO sr/previous" keys, to expose
+      shared-random values to the controller. Closes ticket 19925.
+
+  o Minor features (entry guards):
+    - Add UseEntryGuards to TEST_OPTIONS_DEFAULT_VALUES in order to not
+      break regression tests.
+    - Require UseEntryGuards when UseBridges is set, in order to make
+      sure bridges aren't bypassed. Resolves ticket 20502.
+
+  o Minor features (fallback directories):
+    - Select 200 fallback directories for each release. Closes
+      ticket 20881.
+    - Allow 3 fallback relays per operator, which is safe now that we
+      are choosing 200 fallback relays. Closes ticket 20912.
+    - Exclude relays affected by bug 20499 from the fallback list.
+      Exclude relays from the fallback list if they are running versions
+      known to be affected by bug 20499, or if in our tests they deliver
+      a stale consensus (i.e. one that expired more than 24 hours ago).
+      Closes ticket 20539.
+    - Reduce the minimum fallback bandwidth to 1 MByte/s. Part of
+      ticket 18828.
+    - Require fallback directories to have the same address and port for
+      7 days (now that we have enough relays with this stability).
+      Relays whose OnionOO stability timer is reset on restart by bug
+      18050 should upgrade to Tor 0.2.8.7 or later, which has a fix for
+      this issue. Closes ticket 20880; maintains short-term fix
+      in 0.2.8.2-alpha.
+    - Require fallbacks to have flags for 90% of the time (weighted
+      decaying average), rather than 95%. This allows at least 73% of
+      clients to bootstrap in the first 5 seconds without contacting an
+      authority. Part of ticket 18828.
+    - Annotate updateFallbackDirs.py with the bandwidth and consensus
+      weight for each candidate fallback. Closes ticket 20878.
+    - Make it easier to change the output sort order of fallbacks.
+      Closes ticket 20822.
+    - Display the relay fingerprint when downloading consensuses from
+      fallbacks. Closes ticket 20908.
+
+  o Minor features (geoip, also in 0.2.9.9):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (next-gen onion service directories):
+    - Remove the "EnableOnionServicesV3" consensus parameter that we
+      introduced in 0.3.0.1-alpha: relays are now always willing to act
+      as v3 onion service directories. Resolves ticket 19899.
+
+  o Minor features (linting):
+    - Enhance the changes file linter to warn on Tor versions that are
+      prefixed with "tor-". Closes ticket 21096.
+
+  o Minor features (logging):
+    - In several places, describe unset ed25519 keys as "<unset>",
+      rather than the scary "AAAAAAAA...AAA". Closes ticket 21037.
+
+  o Minor bugfix (control protocol):
+    - The reply to a "GETINFO config/names" request via the control
+      protocol now spells the type "Dependent" correctly. This is a
+      breaking change in the control protocol. (The field seems to be
+      ignored by the most common known controllers.) Fixes bug 18146;
+      bugfix on 0.1.1.4-alpha.
+
+  o Minor bugfixes (bug resilience):
+    - Fix an unreachable size_t overflow in base64_decode(). Fixes bug
+      19222; bugfix on 0.2.0.9-alpha. Found by Guido Vranken; fixed by
+      Hans Jerry Illikainen.
+
+  o Minor bugfixes (build):
+    - Replace obsolete Autoconf macros with their modern equivalent and
+      prevent similar issues in the future. Fixes bug 20990; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (client, guards):
+    - Fix bug where Tor would think that there are circuits waiting for
+      better guards even though those circuits have been freed. Fixes
+      bug 21142; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (config):
+    - Don't assert on startup when trying to get the options list and
+      LearnCircuitBuildTimeout is set to 0: we are currently parsing the
+      options so of course they aren't ready yet. Fixes bug 21062;
+      bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (controller):
+    - Make the GETINFO interface for inquiring about entry guards
+      support the new guards backend. Fixes bug 20823; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (dead code):
+    - Remove a redundant check for PidFile changes at runtime in
+      options_transition_allowed(): this check is already performed
+      regardless of whether the sandbox is active. Fixes bug 21123;
+      bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (documentation):
+    - Update the tor manual page to document every option that can not
+      be changed while tor is running. Fixes bug 21122.
+
+  o Minor bugfixes (fallback directories):
+    - Stop failing when a relay has no uptime data in
+      updateFallbackDirs.py. Fixes bug 20945; bugfix on 0.2.8.1-alpha.
+    - Avoid checking fallback candidates' DirPorts if they are down in
+      OnionOO. When a relay operator has multiple relays, this
+      prioritizes relays that are up over relays that are down. Fixes
+      bug 20926; bugfix on 0.2.8.3-alpha.
+    - Stop failing when OUTPUT_COMMENTS is True in updateFallbackDirs.py.
+      Fixes bug 20877; bugfix on 0.2.8.3-alpha.
+
+  o Minor bugfixes (guards, bootstrapping):
+    - When connecting to a directory guard during bootstrap, do not mark
+      the guard as successful until we receive a good-looking directory
+      response from it. Fixes bug 20974; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Fix the config reload pruning of old vs new services so it
+      actually works when both ephemeral and non-ephemeral services are
+      configured. Fixes bug 21054; bugfix on 0.3.0.1-alpha.
+    - Allow the number of introduction points to be as low as 0, rather
+      than as low as 3. Fixes bug 21033; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (IPv6):
+    - Make IPv6-using clients try harder to find an IPv6 directory
+      server. Fixes bug 20999; bugfix on 0.2.8.2-alpha.
+    - When IPv6 addresses have not been downloaded yet (microdesc
+      consensus documents don't list relay IPv6 addresses), use hard-
+      coded addresses for authorities, fallbacks, and configured
+      bridges. Now IPv6-only clients can use microdescriptors. Fixes bug
+      20996; bugfix on b167e82 from 19608 in 0.2.8.5-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a memory leak when configuring hidden services. Fixes bug
+      20987; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (portability, also in 0.2.9.9):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Honor DataDirectoryGroupReadable when tor is a relay. Previously,
+      initializing the keys would reset the DataDirectory to 0700
+      instead of 0750 even if DataDirectoryGroupReadable was set to 1.
+      Fixes bug 19953; bugfix on 0.0.2pre16. Patch by "redfish".
+
+  o Minor bugfixes (testing):
+    - Remove undefined behavior from the backtrace generator by removing
+      its signal handler. Fixes bug 21026; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Allow the unit tests to pass even when DNS lookups of bogus
+      addresses do not fail as expected. Fixes bug 20862 and 20863;
+      bugfix on unit tests introduced in 0.2.8.1-alpha
+      through 0.2.9.4-alpha.
+
+  o Code simplification and refactoring:
+    - Refactor code to manipulate global_origin_circuit_list into
+      separate functions. Closes ticket 20921.
+
+  o Documentation (formatting):
+    - Clean up formatting of tor.1 man page and HTML doc, where <pre>
+      blocks were incorrectly appearing. Closes ticket 20885.
+
+  o Documentation (man page):
+    - Clarify many options in tor.1 and add some min/max values for
+      HiddenService options. Closes ticket 21058.
+
+
+Changes in version 0.2.9.9 - 2017-01-23
+  Tor 0.2.9.9 fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  This release also resolves a client-side onion service reachability
+  bug, and resolves a pair of small portability issues.
+
+  o Major bugfixes (security):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes -- and having
+      it on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Major bugfixes (client, onion service):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.3.0.1-alpha - 2016-12-19
+  Tor 0.3.0.1-alpha is the first alpha release in the 0.3.0 development
+  series. It strengthens Tor's link and circuit handshakes by
+  identifying relays by their Ed25519 keys, improves the algorithm that
+  clients use to choose and maintain their list of guards, and includes
+  additional backend support for the next-generation hidden service
+  design. It also contains numerous other small features and
+  improvements to security, correctness, and performance.
+
+  Below are the changes since 0.2.9.8.
+
+  o Major features (guard selection algorithm):
+    - Tor's guard selection algorithm has been redesigned from the
+      ground up, to better support unreliable networks and restrictive
+      sets of entry nodes, and to better resist guard-capture attacks by
+      hostile local networks. Implements proposal 271; closes
+      ticket 19877.
+
+  o Major features (next-generation hidden services):
+    - Relays can now handle v3 ESTABLISH_INTRO cells as specified by
+      prop224 aka "Next Generation Hidden Services". Service and clients
+      don't use this functionality yet. Closes ticket 19043. Based on
+      initial code by Alec Heifetz.
+    - Relays now support the HSDir version 3 protocol, so that they can
+      can store and serve v3 descriptors. This is part of the next-
+      generation onion service work detailled in proposal 224. Closes
+      ticket 17238.
+
+  o Major features (protocol, ed25519 identity keys):
+    - Relays now use Ed25519 to prove their Ed25519 identities and to
+      one another, and to clients. This algorithm is faster and more
+      secure than the RSA-based handshake we've been doing until now.
+      Implements the second big part of proposal 220; Closes
+      ticket 15055.
+    - Clients now support including Ed25519 identity keys in the EXTEND2
+      cells they generate. By default, this is controlled by a consensus
+      parameter, currently disabled. You can turn this feature on for
+      testing by setting ExtendByEd25519ID in your configuration. This
+      might make your traffic appear different than the traffic
+      generated by other users, however. Implements part of ticket
+      15056; part of proposal 220.
+    - Relays now understand requests to extend to other relays by their
+      Ed25519 identity keys. When an Ed25519 identity key is included in
+      an EXTEND2 cell, the relay will only extend the circuit if the
+      other relay can prove ownership of that identity. Implements part
+      of ticket 15056; part of proposal 220.
+
+  o Major bugfixes (scheduler):
+    - Actually compare circuit policies in ewma_cmp_cmux(). This bug
+      caused the channel scheduler to behave more or less randomly,
+      rather than preferring channels with higher-priority circuits.
+      Fixes bug 20459; bugfix on 0.2.6.2-alpha.
+
+  o Minor features (controller):
+    - When HSFETCH arguments cannot be parsed, say "Invalid argument"
+      rather than "unrecognized." Closes ticket 20389; patch from
+      Ivan Markin.
+
+  o Minor features (diagnostic, directory client):
+    - Warn when we find an unexpected inconsistency in directory
+      download status objects. Prevents some negative consequences of
+      bug 20593.
+
+  o Minor features (directory authority):
+    - Add a new authority-only AuthDirTestEd25519LinkKeys option (on by
+      default) to control whether authorities should try to probe relays
+      by their Ed25519 link keys. This option will go away in a few
+      releases--unless we encounter major trouble in our ed25519 link
+      protocol rollout, in which case it will serve as a safety option.
+
+  o Minor features (directory cache):
+    - Relays and bridges will now refuse to serve the consensus they
+      have if they know it is too old for a client to use. Closes
+      ticket 20511.
+
+  o Minor features (ed25519 link handshake):
+    - Advertise support for the ed25519 link handshake using the
+      subprotocol-versions mechanism, so that clients can tell which
+      relays can identity themselves by Ed25519 ID. Closes ticket 20552.
+
+  o Minor features (fingerprinting resistence, authentication):
+    - Extend the length of RSA keys used for TLS link authentication to
+      2048 bits. (These weren't used for forward secrecy; for forward
+      secrecy, we used P256.) Closes ticket 13752.
+
+  o Minor features (infrastructure):
+    - Implement smartlist_add_strdup() function. Replaces the use of
+      smartlist_add(sl, tor_strdup(str)). Closes ticket 20048.
+
+  o Minor bugfixes (client):
+    - When clients that use bridges start up with a cached consensus on
+      disk, they were ignoring it and downloading a new one. Now they
+      use the cached one. Fixes bug 20269; bugfix on 0.2.3.12-alpha.
+
+  o Minor bugfixes (configuration):
+    - Accept non-space whitespace characters after the severity level in
+      the `Log` option. Fixes bug 19965; bugfix on 0.2.1.1-alpha.
+    - Support "TByte" and "TBytes" units in options given in bytes.
+      "TB", "terabyte(s)", "TBit(s)" and "terabit(s)" were already
+      supported. Fixes bug 20622; bugfix on 0.2.0.14-alpha.
+
+  o Minor bugfixes (consensus weight):
+    - Add new consensus method that initializes bw weights to 1 instead
+      of 0. This prevents a zero weight from making it all the way to
+      the end (happens in small testing networks) and causing an error.
+      Fixes bug 14881; bugfix on 0.2.2.17-alpha.
+
+  o Minor bugfixes (descriptors):
+    - Correctly recognise downloaded full descriptors as valid, even
+      when using microdescriptors as circuits. This affects clients with
+      FetchUselessDescriptors set, and may affect directory authorities.
+      Fixes bug 20839; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (directory system):
+    - Download all consensus flavors, descriptors, and authority
+      certificates when FetchUselessDescriptors is set, regardless of
+      whether tor is a directory cache or not. Fixes bug 20667; bugfix
+      on all recent tor versions.
+    - Bridges and relays now use microdescriptors (like clients do)
+      rather than old-style router descriptors. Now bridges will blend
+      in with clients in terms of the circuits they build. Fixes bug
+      6769; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (ed25519 certificates):
+    - Correctly interpret ed25519 certificates that would expire some
+      time after 19 Jan 2038. Fixes bug 20027; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop ignoring misconfigured hidden services. Instead, refuse to
+      start tor until the misconfigurations have been corrected. Fixes
+      bug 20559; bugfix on multiple commits in 0.2.7.1-alpha
+      and earlier.
+
+  o Minor bugfixes (memory leak at exit):
+    - Fix a small harmless memory leak at exit of the previously unused
+      RSA->Ed identity cross-certificate. Fixes bug 17779; bugfix
+      on 0.2.7.2-alpha.
+
+  o Minor bugfixes (util):
+    - When finishing writing a file to disk, if we were about to replace
+      the file with the temporary file created before and we fail to
+      replace it, remove the temporary file so it doesn't stay on disk.
+      Fixes bug 20646; bugfix on tor-0.2.0.7-alpha. Patch by fk.
+
+  o Minor bugfixes (Windows):
+    - Check for getpagesize before using it to mmap files. This fixes
+      compilation in some MinGW environments. Fixes bug 20530; bugfix on
+      0.1.2.1-alpha. Reported by "ice".
+
+  o Code simplification and refactoring:
+    - Abolish all global guard context in entrynodes.c; replace with new
+      guard_selection_t structure as preparation for proposal 271.
+      Closes ticket 19858.
+    - Introduce rend_service_is_ephemeral() that tells if given onion
+      service is ephemeral. Replace unclear NULL-checkings for service
+      directory with this function. Closes ticket 20526.
+    - Extract magic numbers in circuituse.c into defined variables.
+    - Refactor circuit_is_available_for_use to remove unnecessary check.
+    - Refactor circuit_predict_and_launch_new for readability and
+      testability. Closes ticket 18873.
+    - Refactor large if statement in purpose_needs_anonymity to use
+      switch statement instead. Closes part of ticket 20077.
+    - Refactor the hashing API to return negative values for errors, as
+      is done as throughout the codebase. Closes ticket 20717.
+    - Remove data structures that were used to index or_connection
+      objects by their RSA identity digests. These structures are fully
+      redundant with the similar structures used in the
+      channel abstraction.
+    - Remove duplicate code in the channel_write_*cell() functions.
+      Closes ticket 13827; patch from Pingl.
+    - Remove redundant behavior of is_sensitive_dir_purpose, refactor to
+      use only purpose_needs_anonymity. Closes part of ticket 20077.
+    - The code to generate and parse EXTEND and EXTEND2 cells has been
+      replaced with code automatically generated by the
+      "trunnel" utility.
+
+  o Documentation:
+    - Include the "TBits" unit in Tor's man page. Fixes part of bug
+      20622; bugfix on tor-0.2.5.1-alpha.
+    - Change '1' to 'weight_scale' in consensus bw weights calculation
+      comments, as that is reality. Closes ticket 20273. Patch
+      from pastly.
+    - Correct the value for AuthDirGuardBWGuarantee in the manpage, from
+      250 KBytes to 2 MBytes. Fixes bug 20435; bugfix
+      on tor-0.2.5.6-alpha.
+    - Stop the man page from incorrectly stating that HiddenServiceDir
+      must already exist. Fixes 20486.
+    - Clarify that when ClientRejectInternalAddresses is enabled (which
+      is the default), multicast DNS hostnames for machines on the local
+      network (of the form *.local) are also rejected. Closes
+      ticket 17070.
+
+  o Removed features:
+    - The AuthDirMaxServersPerAuthAddr option no longer exists: The same
+      limit for relays running on a single IP applies to authority IP
+      addresses as well as to non-authority IP addresses. Closes
+      ticket 20960.
+    - The UseDirectoryGuards torrc option no longer exists: all users
+      that use entry guards will also use directory guards. Related to
+      proposal 271; implements part of ticket 20831.
+
+  o Testing:
+    - New unit tests for tor_htonll(). Closes ticket 19563. Patch
+      from "overcaffeinated".
+    - Perform the coding style checks when running the tests and fail
+      when coding style violations are found. Closes ticket 5500.
+    - Add tests for networkstatus_compute_bw_weights_v10.
+    - Add unit tests circuit_predict_and_launch_new.
+    - Extract dummy_origin_circuit_new so it can be used by other
+      test functions.
+
+
+Changes in version 0.2.8.12 - 2016-12-19
+  Tor 0.2.8.12 backports a fix for a medium-severity issue (bug 21018
+  below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  It also includes an updated list of fallback directories, backported
+  from 0.2.9.
+
+  Now that the Tor 0.2.9 series is stable, only major bugfixes will be
+  backported to 0.2.8 in the future.
+
+  o Major bugfixes (parsing, security, backported from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list, backported from 0.2.9.8):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (geoip, backported from 0.2.9.7-rc):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.8 - 2016-12-19
+  Tor 0.2.9.8 is the first stable release of the Tor 0.2.9 series.
+
+  The Tor 0.2.9 series makes mandatory a number of security features
+  that were formerly optional. It includes support for a new shared-
+  randomness protocol that will form the basis for next generation
+  hidden services, includes a single-hop hidden service mode for
+  optimizing .onion services that don't actually want to be hidden,
+  tries harder not to overload the directory authorities with excessive
+  downloads, and supports a better protocol versioning scheme for
+  improved compatibility with other implementations of the Tor protocol.
+
+  And of course, there are numerous other bugfixes and improvements.
+
+  This release also includes a fix for a medium-severity issue (bug
+  21018 below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  Below are the changes since 0.2.9.7-rc. For a list of all changes
+  since 0.2.8, see the ReleaseNotes file.
+
+  o Major bugfixes (parsing, security):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+
+Changes in version 0.2.9.7-rc - 2016-12-12
+  Tor 0.2.9.7-rc fixes a few small bugs remaining in Tor 0.2.9.6-rc,
+  including a few that had prevented tests from passing on
+  some platforms.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfix (build):
+    - The current Git revision when building from a local repository is
+      now detected correctly when using git worktrees. Fixes bug 20492;
+      bugfix on 0.2.3.9-alpha.
+
+  o Minor bugfixes (directory authority):
+    - When computing old Tor protocol line version in protover, we were
+      looking at 0.2.7.5 twice instead of a specific case for
+      0.2.9.1-alpha. Fixes bug 20810; bugfix on 0.2.9.4-alpha.
+
+  o Minor bugfixes (download scheduling):
+    - Resolve a "bug" warning when considering a download schedule whose
+      delay had approached INT_MAX. Fixes 20875; bugfix on 0.2.9.5-alpha.
+
+  o Minor bugfixes (logging):
+    - Downgrade a harmless log message about the
+      pending_entry_connections list from "warn" to "info". Mitigates
+      bug 19926.
+
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak when receiving AF_UNIX connections on a
+      SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
+    - When moving a signed descriptor object from a source to an
+      existing destination, free the allocated memory inside that
+      destination object. Fixes bug 20715; bugfix on 0.2.8.3-alpha.
+
+  o Minor bugfixes (memory leak, use-after-free, linux seccomp2 sandbox):
+    - Fix a memory leak and use-after-free error when removing entries
+      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
+      0.2.5.5-alpha. Patch from "cypherpunks".
+
+  o Minor bugfixes (portability):
+    - Use the correct spelling of MAC_OS_X_VERSION_10_12 on configure.ac
+      Fixes bug 20935; bugfix on 0.2.9.6-rc.
+
+  o Minor bugfixes (unit tests):
+    - Stop expecting NetBSD unit tests to report success for ipfw. Part
+      of a fix for bug 19960; bugfix on 0.2.9.5-alpha.
+    - Fix tolerances in unit tests for monotonic time comparisons
+      between nanoseconds and microseconds. Previously, we accepted a 10
+      us difference only, which is not realistic on every platform's
+      clock_gettime(). Fixes bug 19974; bugfix on 0.2.9.1-alpha.
+    - Remove a double-free in the single onion service unit test. Stop
+      ignoring a return value. Make future changes less error-prone.
+      Fixes bug 20864; bugfix on 0.2.9.6-rc.
+
+
+Changes in version 0.2.8.11 - 2016-12-08
+  Tor 0.2.8.11 backports fixes for additional portability issues that
+  could prevent Tor from building correctly on OSX Sierra, or with
+  OpenSSL 1.1. Affected users should upgrade; others can safely stay
+  with 0.2.8.10.
+
+  o Minor bugfixes (portability):
+    - Avoid compilation errors when building on OSX Sierra. Sierra began
+      to support the getentropy() and clock_gettime() APIs, but created
+      a few problems in doing so. Tor 0.2.9 has a more thorough set of
+      workarounds; in 0.2.8, we are just using the /dev/urandom and mach
+      monotonic time interfaces. Fixes bug 20865. Bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+
+
+Changes in version 0.2.8.10 - 2016-12-02
+  Tor 0.2.8.10 backports a fix for a bug that would sometimes make clients
+  unusable after they left standby mode. It also backports fixes for
+  a few portability issues and a small but problematic memory leak.
+
+  o Major bugfixes (client reliability, backport from 0.2.9.5-alpha):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client performance, backport from 0.2.9.5-alpha):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.6-rc):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.2.9.5-alpha):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+Changes in version 0.2.9.6-rc - 2016-12-02
+  Tor 0.2.9.6-rc fixes a few remaining bugs found in the previous alpha
+  version. We hope that it will be ready to become stable soon, and we
+  encourage everyone to test this release. If no showstopper bugs are
+  found here, the next 0.2.9 release will be stable.
+
+  o Major bugfixes (relay, resolver, logging):
+    - For relays that don't know their own address, avoid attempting a
+      local hostname resolve for each descriptor we download. This
+      will cut down on the number of "Success: chose address 'x.x.x.x'"
+      log lines, and also avoid confusing clock jumps if the resolver
+      is slow. Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (client, fascistfirewall):
+    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
+      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Stop ignoring the anonymity status of saved keys for hidden
+      services and single onion services when first starting tor.
+      Instead, refuse to start tor if any hidden service key has been
+      used in a different hidden service anonymity mode. Fixes bug
+      20638; bugfix on 17178 in 0.2.9.3-alpha; reported by ahf.
+
+  o Minor bugfixes (portability):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+    - Run correctly when built on Windows build environments that
+      require _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
+
+  o Minor bugfixes (single onion services, Tor2web):
+    - Stop complaining about long-term one-hop circuits deliberately
+      created by single onion services and Tor2web. These log messages
+      are intended to diagnose issue 8387, which relates to circuits
+      hanging around forever for no reason. Fixes bug 20613; bugfix on
+      0.2.9.1-alpha. Reported by "pastly".
+
+  o Minor bugfixes (unit tests):
+    - Stop spurious failures in the local interface address discovery
+      unit tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by
+      Neel Chauhan.
+
+  o Documentation:
+    - Correct the minimum bandwidth value in torrc.sample, and queue a
+      corresponding change for torrc.minimal. Closes ticket 20085.
+
+
+Changes in version 0.2.9.5-alpha - 2016-11-08
+  Tor 0.2.9.5-alpha fixes numerous bugs discovered in the previous alpha
+  version. We believe one or two probably remain, and we encourage
+  everyone to test this release.
+
+  o Major bugfixes (client performance):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client reliability):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (download scheduling):
+    - When using an exponential backoff schedule, do not give up on
+      downloading just because we have failed a bunch of times. Since
+      each delay is longer than the last, retrying indefinitely won't
+      hurt. Fixes bug 20536; bugfix on 0.2.9.1-alpha.
+    - If a consensus expires while we are waiting for certificates to
+      download, stop waiting for certificates.
+    - If we stop waiting for certificates less than a minute after we
+      started downloading them, do not consider the certificate download
+      failure a separate failure. Fixes bug 20533; bugfix
+      on 0.2.0.9-alpha.
+    - Remove the maximum delay on exponential-backoff scheduling. Since
+      we now allow an infinite number of failures (see ticket 20536), we
+      must now allow the time to grow longer on each failure. Fixes part
+      of bug 20534; bugfix on 0.2.9.1-alpha.
+    - Make our initial download delays closer to those from 0.2.8. Fixes
+      another part of bug 20534; bugfix on 0.2.9.1-alpha.
+    - When determining when to download a directory object, handle times
+      after 2038 if the operating system supports them. (Someday this
+      will be important!) Fixes bug 20587; bugfix on 0.2.8.1-alpha.
+    - When using exponential backoff in test networks, use a lower
+      exponent, so the delays do not vary as much. This helps test
+      networks bootstrap consistently. Fixes bug 20597; bugfix on 20499.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (client directory scheduling):
+    - Treat "relay too busy to answer request" as a failed request and a
+      reason to back off on our retry frequency. This is safe now that
+      exponential backoffs retry indefinitely, and avoids a bug where we
+      would reset our download schedule erroneously. Fixes bug 20593;
+      bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (client, logging):
+    - Remove a BUG warning in circuit_pick_extend_handshake(). Instead,
+      assume all nodes support EXTEND2. Use ntor whenever a key is
+      available. Fixes bug 20472; bugfix on 0.2.9.3-alpha.
+    - On DNSPort, stop logging a BUG warning on a failed hostname
+      lookup. Fixes bug 19869; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (hidden services):
+    - When configuring hidden services, check every hidden service
+      directory's permissions. Previously, we only checked the last
+      hidden service. Fixes bug 20529; bugfix the work to fix 13942
+      in 0.2.6.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
+      removed the ECDH ciphers which caused the tests to fail on
+      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay bootstrap):
+    - Ensure relays don't make multiple connections during bootstrap.
+      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+    - Avoid a small memory leak when informing worker threads about
+      rotated onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
+    - Do not try to parallelize workers more than 16x without the user
+      explicitly configuring us to do so, even if we do detect more than
+      16 CPU cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Start correctly when creating a single onion service in a
+      directory that did not previously exist. Fixes bug 20484; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (testing):
+    - Avoid a unit test failure on systems with over 16 detectable CPU
+      cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Documentation:
+    - Clarify that setting HiddenServiceNonAnonymousMode requires you to
+      also set "SOCKSPort 0". Fixes bug 20487; bugfix on 0.2.9.3-alpha.
+    - Module-level documentation for several more modules. Closes
+      tickets 19287 and 19290.
 
 
 Changes in version 0.2.8.9 - 2016-10-17
@@ -123,7 +2582,7 @@ Changes in version 0.2.9.4-alpha - 2016-10-17
   o Minor bugfixes (getpass):
     - Defensively fix a non-triggerable heap corruption at do_getpass()
       to protect ourselves from mistakes in the future. Fixes bug
-      #19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
+      19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
       by nherring.
 
   o Minor bugfixes (hidden service):
@@ -194,7 +2653,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
 
   o Major bugfixes (circuit building):
     - Hidden service client-to-intro-point and service-to-rendezvous-
-      point cicruits use the TAP key supplied by the protocol, to avoid
+      point circuits use the TAP key supplied by the protocol, to avoid
       epistemic attacks. Fixes bug 19163; bugfix on 0.2.4.18-rc.
 
   o Major bugfixes (compilation, OpenBSD):
@@ -216,8 +2675,9 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
       OpenSSL 0.9.7 or later since 2009. Closes ticket 19998.
 
   o Minor feature (fallback directories):
-    - Remove broken entries from the hard-coded fallback directory list.
-      Closes ticket 20190; patch by teor.
+    - Remove 8 fallbacks that are no longer suitable, leaving 81 of the
+      100 fallbacks originally introduced in Tor 0.2.8.2-alpha in March
+      2016. Closes ticket 20190; patch by teor.
 
   o Minor features (geoip, also in 0.2.8.8):
     - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
@@ -312,7 +2772,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
     - Document the default PathsNeededToBuildCircuits value that's used
       by clients when the directory authorities don't set
       min_paths_for_circs_pct. Fixes bug 20117; bugfix on 02c320916e02
-      in tor-0.2.4.10-alpha. Patch by teor, reported by Jesse V.
+      in 0.2.4.10-alpha. Patch by teor, reported by Jesse V.
     - Fix manual for the User option: it takes a username, not a UID.
       Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have
       a manpage!).
@@ -351,12 +2811,12 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
 
   o Minor bugfixes (options):
     - Check the consistency of UseEntryGuards and EntryNodes more
-      reliably. Fixes bug 20074; bugfix on tor- 0.2.4.12-alpha. Patch
+      reliably. Fixes bug 20074; bugfix on 0.2.4.12-alpha. Patch
       by teor.
     - Stop changing the configured value of UseEntryGuards on
       authorities and Tor2web clients. Fixes bug 20074; bugfix on
-      commits 51fc6799 in tor-0.1.1.16-rc and acda1735 in tor-0.2.4.3-
-      alpha. Patch by teor.
+      commits 51fc6799 in 0.1.1.16-rc and acda1735 in 0.2.4.3-alpha.
+      Patch by teor.
 
   o Minor bugfixes (Tor2web):
     - Prevent Tor2web clients running hidden services, these services
@@ -395,8 +2855,9 @@ Changes in version 0.2.8.8 - 2016-09-23
       this one.
 
   o Minor feature (fallback directories):
-    - Remove broken fallbacks from the hard-coded fallback directory
-      list. Closes ticket 20190; patch by teor.
+    - Remove 8 fallbacks that are no longer suitable, leaving 81 of the
+      100 fallbacks originally introduced in Tor 0.2.8.2-alpha in March
+      2016. Closes ticket 20190; patch by teor.
 
   o Minor features (geoip):
     - Update geoip and geoip6 to the September 6 2016 Maxmind GeoLite2
@@ -850,8 +3311,9 @@ Changes in version 0.2.8.6 - 2016-08-02
       is signed. Fixes bug 19682; bugfix on 0.2.8.1-alpha.
 
   o Minor bugfixes (fallback directories):
-    - Remove a fallback that was on the hardcoded list, then opted-out.
-      Fixes bug 19782; update to fallback list from 0.2.8.2-alpha.
+    - Remove 1 fallback that was on the hardcoded list, then opted-out,
+      leaving 89 of the 100 fallbacks originally introduced in Tor
+      0.2.8.2-alpha in March 2016. Closes ticket 19782; patch by teor.
 
   o Minor bugfixes (Linux seccomp2 sandbox):
     - Allow more syscalls when running with "Sandbox 1" enabled:
@@ -875,7 +3337,7 @@ Changes in version 0.2.8.5-rc - 2016-07-07
   o Major bugfixes (heartbeat):
     - Fix a regression that would crash Tor when the periodic
       "heartbeat" log messages were disabled. Fixes bug 19454; bugfix on
-      tor-0.2.8.1-alpha. Reported by "kubaku".
+      0.2.8.1-alpha. Reported by "kubaku".
 
   o Minor features (build):
     - Tor now again builds with the recent OpenSSL 1.1 development
@@ -923,8 +3385,9 @@ Changes in version 0.2.8.5-rc - 2016-07-07
     - Update fallback whitelist and blacklist based on relay operator
       emails. Blacklist unsuitable (non-working, over-volatile)
       fallbacks. Resolves ticket 19071. Patch by teor.
-    - Update hard-coded fallback list to remove unsuitable fallbacks.
-      Resolves ticket 19071. Patch by teor.
+    - Remove 10 unsuitable fallbacks, leaving 90 of the 100 fallbacks
+      originally introduced in Tor 0.2.8.2-alpha in March 2016. Closes
+      ticket 19071; patch by teor.
 
 
 Changes in version 0.2.8.4-rc - 2016-06-15
@@ -1021,7 +3484,7 @@ Changes in version 0.2.8.3-alpha - 2016-05-26
 
   o Major bugfixes (testing):
     - Fix a bug that would block 'make test-network-all' on systems where
-      IPv6 packets were lost. Fixes bug 19008; bugfix on tor-0.2.7.3-rc.
+      IPv6 packets were lost. Fixes bug 19008; bugfix on 0.2.7.3-rc.
     - Avoid "WSANOTINITIALISED" warnings in the unit tests. Fixes bug 18668;
       bugfix on 0.2.8.1-alpha.
 
@@ -1034,10 +3497,12 @@ Changes in version 0.2.8.3-alpha - 2016-05-26
     - Give each fallback the same weight for client selection; restrict
       fallbacks to one per operator; report fallback directory detail
       changes when rebuilding list; add new fallback directory mirrors
-      to the whitelist; update fallback directories based on the latest
-      OnionOO data; and any other minor simplifications and fixes.
-      Closes tasks 17158, 17905, 18749, bug 18689, and fixes part of bug
-      18812 on 0.2.8.1-alpha; patch by "teor".
+      to the whitelist; and many other minor simplifications and fixes.
+      Closes tasks 17905, 18749, bug 18689, and fixes part of bug 18812 on
+      0.2.8.1-alpha; patch by "teor".
+    - Replace the 21 fallbacks generated in January 2016 and included in
+      Tor 0.2.8.1-alpha, with a list of 100 fallbacks generated in March
+      2016. Closes task 17158; patch by "teor".
 
   o Minor features (geoip):
     - Update geoip and geoip6 to the May 4 2016 Maxmind GeoLite2
@@ -1307,7 +3772,7 @@ Changes in version 0.2.8.2-alpha - 2016-03-28
       testing versions of the static libraries. Fixes bug 18490; bugfix
       on 0.2.7.1-alpha.
     - Avoid spurious failures from configure files related to calling
-      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18625; bugfix on
+      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18626; bugfix on
       0.2.0.1-alpha. Patch from "cypherpunks".
     - Silence spurious clang-scan warnings in the ed25519_donna code by
       explicitly initializing some objects. Fixes bug 18384; bugfix on
@@ -1481,10 +3946,11 @@ Changes in version 0.2.8.1-alpha - 2016-02-04
       should reduces failures due to fallback churn. Implements ticket
       4483. Patch by "teor". Implements IPv4 portions of proposal 210 by
       "mikeperry" and "teor".
-    - Include a trial list of default fallback directories, based on an
-      opt-in survey of suitable relays. Doing this should make clients
-      bootstrap more quickly and reliably, and reduce the load on the
-      directory authorities. Closes ticket 15775. Patch by "teor".
+    - Include a trial list of 21 default fallback directories, generated
+      in January 2016, based on an opt-in survey of suitable relays.
+      Doing this should make clients bootstrap more quickly and reliably,
+      and reduce the load on the directory authorities. Closes ticket
+      15775. Patch by "teor".
       Candidates identified using an OnionOO script by "weasel", "teor",
       "gsathya", and "karsten".
     - Previously only relays that explicitly opened a directory port
diff --git a/Doxyfile.in b/Doxyfile.in
index a39348f2cb..4cf9c30ded 100644
--- a/Doxyfile.in
+++ b/Doxyfile.in
@@ -446,12 +446,6 @@ MAX_INITIALIZER_LINES  = 30
 
 SHOW_USED_FILES        = YES
 
-# If the sources in your project are distributed over multiple directories 
-# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy 
-# in the documentation. The default is NO.
-
-SHOW_DIRECTORIES       = NO
-
 # Set the SHOW_FILES tag to NO to disable the generation of the Files page.
 # This will remove the Files entry from the Quick Index and from the 
 # Folder Tree View (if specified). The default is YES.
@@ -760,12 +754,6 @@ HTML_FOOTER            =
 
 HTML_STYLESHEET        = 
 
-# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, 
-# files or namespaces will be aligned in HTML using tables. If set to 
-# NO a bullet list will be used.
-
-HTML_ALIGN_MEMBERS     = YES
-
 # If the GENERATE_HTMLHELP tag is set to YES, additional index files 
 # will be generated that can be used as input for tools like the 
 # Microsoft HTML help workshop to generate a compiled HTML help file (.chm) 
@@ -1047,18 +1035,6 @@ GENERATE_XML           = NO
 
 XML_OUTPUT             = xml
 
-# The XML_SCHEMA tag can be used to specify an XML schema, 
-# which can be used by a validating XML parser to check the 
-# syntax of the XML files.
-
-XML_SCHEMA             = 
-
-# The XML_DTD tag can be used to specify an XML DTD, 
-# which can be used by a validating XML parser to check the 
-# syntax of the XML files.
-
-XML_DTD                = 
-
 # If the XML_PROGRAMLISTING tag is set to YES Doxygen will 
 # dump the program listings (including syntax highlighting 
 # and cross-referencing information) to the XML output. Note that 
@@ -1264,7 +1240,7 @@ HAVE_DOT               = NO
 # DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory 
 # containing the font.
 
-DOT_FONTNAME           = FreeSans
+DOT_FONTNAME           =
 
 # By default doxygen will tell dot to use the output directory to look for the 
 # FreeSans.ttf font (which doxygen will put there itself). If you specify a 
diff --git a/LICENSE b/LICENSE
index 56d1002251..3d0f8c121b 100644
--- a/LICENSE
+++ b/LICENSE
@@ -13,7 +13,7 @@ Tor is distributed under this license:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-Copyright (c) 2007-2016, The Tor Project, Inc.
+Copyright (c) 2007-2017, The Tor Project, Inc.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
diff --git a/Makefile.am b/Makefile.am
index f06d0e3734..1a6d8e7d12 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,6 +1,6 @@
 # Copyright (c) 2001-2004, Roger Dingledine
 # Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-# Copyright (c) 2007-2015, The Tor Project, Inc.
+# Copyright (c) 2007-2017, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 ACLOCAL_AMFLAGS = -I m4
@@ -9,13 +9,14 @@ noinst_LIBRARIES=
 EXTRA_DIST=
 noinst_HEADERS=
 bin_PROGRAMS=
+EXTRA_PROGRAMS=
 CLEANFILES=
 TESTS=
 noinst_PROGRAMS=
 DISTCLEANFILES=
 bin_SCRIPTS=
 AM_CPPFLAGS=
-AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@
+AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@
 SHELL=@SHELL@
 
 if COVERAGE_ENABLED
@@ -24,6 +25,12 @@ else
 TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT)
 endif
 
+if USE_RUST
+rust_ldadd=$(top_builddir)/src/rust/target/release/libtor_util.a
+else
+rust_ldadd=
+endif
+
 include src/include.am
 include doc/include.am
 include contrib/include.am
@@ -34,7 +41,8 @@ EXTRA_DIST+= \
 	LICENSE						\
 	Makefile.nmake					\
 	README						\
-	ReleaseNotes
+	ReleaseNotes					\
+	scripts/maint/checkSpace.pl
 
 ## This tells etags how to find mockable function definitions.
 AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
@@ -52,6 +60,12 @@ TEST_CFLAGS=
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
+TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
+
+if LIBFUZZER_ENABLED
+TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div
+# not "edge"
+endif
 
 TEST_NETWORK_ALL_LOG_DIR=$(top_builddir)/test_network_log
 TEST_NETWORK_ALL_DRIVER_FLAGS=--color-tests yes
@@ -82,6 +96,8 @@ doxygen:
 test: all
 	$(top_builddir)/src/test/test
 
+check-local: check-spaces
+
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
 		echo '$$CHUTNEY_PATH was not set.'; \
@@ -126,6 +142,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/
 	done; \
 	for f in $$flavors; do \
 		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
+		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \
 	done; \
 	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
 	! grep -q FAIL test_network_log/*.trs
@@ -181,11 +198,14 @@ coverage-html-full: all
 # Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
 # tinytest*.[ch]
 check-spaces:
-	$(top_srcdir)/scripts/maint/checkSpace.pl -C \
+if USE_PERL
+	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
 		$(top_srcdir)/src/common/*.[ch] \
 		$(top_srcdir)/src/or/*.[ch] \
 		$(top_srcdir)/src/test/*.[ch] \
+		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
+endif
 
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
@@ -217,6 +237,10 @@ mostlyclean-local:
 	rm -rf $(top_builddir)/doc/doxygen
 	rm -rf $(TEST_NETWORK_ALL_LOG_DIR)
 
+clean-local:
+	rm -rf $(top_builddir)/src/rust/target
+	rm -rf $(top_builddir)/src/rust/.cargo/registry
+
 # This relies on some internal details of how automake implements
 # distcheck.  We check two directories because automake-1.15 changed
 # from $(distdir)/_build to $(distdir)/_build/sub.
diff --git a/ReleaseNotes b/ReleaseNotes
index af61a4d739..1e56ffaf89 100644
--- a/ReleaseNotes
+++ b/ReleaseNotes
@@ -2,6 +2,2200 @@ This document summarizes new features and bugfixes in each stable release
 of Tor. If you want to see more detailed descriptions of the changes in
 each development snapshot, see the ChangeLog file.
 
+Changes in version 0.3.0.8 - 2017-06-08
+  Tor 0.3.0.8 fixes a pair of bugs that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-004 and TROVE-2017-005.
+
+  Tor 0.3.0.8 also includes fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure when a hidden service
+      handles a malformed BEGIN cell. Fixes bug 22493, tracked as
+      TROVE-2017-004 and as CVE-2017-0375; bugfix on 0.3.0.1-alpha.
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (relays, key management, backport from 0.3.1.3-alpha):
+    - Regenerate link and authentication certificates whenever the key
+      that signs them changes; also, regenerate link certificates
+      whenever the signed key changes. Previously, these processes were
+      only weakly coupled, and we relays could (for minutes to hours)
+      wind up with an inconsistent set of keys and certificates, which
+      other relays would not accept. Fixes two cases of bug 22460;
+      bugfix on 0.3.0.1-alpha.
+    - When sending an Ed25519 signing->link certificate in a CERTS cell,
+      send the certificate that matches the x509 certificate that we
+      used on the TLS connection. Previously, there was a race condition
+      if the TLS context rotated after we began the TLS handshake but
+      before we sent the CERTS cell. Fixes a case of bug 22460; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha):
+    - Stop rejecting v3 hidden service descriptors because their size
+      did not match an old padding rule. Fixes bug 22447; bugfix on
+      tor-0.3.0.1-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (configuration, backport from 0.3.1.1-alpha):
+    - Do not crash when starting with LearnCircuitBuildTimeout 0. Fixes
+      bug 22252; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (link handshake, backport from 0.3.1.3-alpha):
+    - Lower the lifetime of the RSA->Ed25519 cross-certificate to six
+      months, and regenerate it when it is within one month of expiring.
+      Previously, we had generated this certificate at startup with a
+      ten-year lifetime, but that could lead to weird behavior when Tor
+      was started with a grossly inaccurate clock. Mitigates bug 22466;
+      mitigation on 0.3.0.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport from
+    0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+
+Changes in version 0.2.9.11 - 2017-06-08
+  Tor 0.2.9.11 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005. (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  Tor 0.2.9.11 also backports fixes for several key management bugs
+  that sometimes made relays unreliable, as well as several other
+  bugfixes described below.
+
+  o Major bugfixes (hidden service, relay, security, backport
+    from 0.3.1.3-alpha):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Major bugfixes (relay, link handshake, backport from 0.3.1.3-alpha):
+    - When performing the v3 link handshake on a TLS connection, report
+      that we have the x509 certificate that we actually used on that
+      connection, even if we have changed certificates since that
+      connection was first opened. Previously, we would claim to have
+      used our most recent x509 link certificate, which would sometimes
+      make the link handshake fail. Fixes one case of bug 22460; bugfix
+      on 0.2.3.6-alpha.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor features (future-proofing, backport from 0.3.0.7):
+    - Tor no longer refuses to download microdescriptors or descriptors if
+      they are listed as "published in the future".  This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor features (directory authorities, backport from 0.3.0.4-rc)
+    - Directory authorities now reject relays running versions
+      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
+      suffer from bug 20499 and don't keep their consensus cache
+      up-to-date. Resolves ticket 20509.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (control port, backport from 0.3.0.6):
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (correctness, backport from 0.3.1.3-alpha):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.0.7):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (memory leak, directory authority, backport
+    from 0.3.1.2-alpha):
+    - When directory authorities reject a router descriptor due to
+      keypinning, free the router descriptor rather than leaking the
+      memory. Fixes bug 22370; bugfix on 0.2.7.2-alpha.
+
+Changes in version 0.2.8.14 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (fallback directory list, backport from 0.3.1.3-alpha):
+    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
+      December 2016 (of which ~126 were still functional) with a list of
+      151 fallbacks (32 new, 119 unchanged, 58 removed) generated in May
+      2017. Resolves ticket 21564.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.7.8 - 2017-06-08
+  Tor 0.2.7.8 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.2.6.12 - 2017-06-08
+  Tor 0.2.6.12 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.5.14 - 2017-06-08
+  Tor 0.2.5.14 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+Changes in version 0.2.4.29 - 2017-06-08
+  Tor 0.2.4.29 backports a fix for a bug that would allow an attacker to
+  remotely crash a hidden service with an assertion failure. Anyone
+  running a hidden service should upgrade to this version, or to some
+  other version with fixes for TROVE-2017-005.  (Versions before 0.3.0
+  are not affected by TROVE-2017-004.)
+
+  o Major bugfixes (hidden service, relay, security):
+    - Fix a remotely triggerable assertion failure caused by receiving a
+      BEGIN_DIR cell on a hidden service rendezvous circuit. Fixes bug
+      22494, tracked as TROVE-2017-005 and CVE-2017-0376; bugfix
+      on 0.2.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (correctness):
+    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
+      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
+
+
+Changes in version 0.3.0.7 - 2017-05-15
+  Tor 0.3.0.7 fixes a medium-severity security bug in earlier versions
+  of Tor 0.3.0.x, where an attacker could cause a Tor relay process
+  to exit. Relays running earlier versions of Tor 0.3.0.x should upgrade;
+  clients are not affected.
+
+  o Major bugfixes (hidden service directory, security):
+    - Fix an assertion failure in the hidden service directory code, which
+      could be used by an attacker to remotely cause a Tor relay process to
+      exit. Relays running earlier versions of Tor 0.3.0.x should upgrade.
+      should upgrade. This security issue is tracked as TROVE-2017-002.
+      Fixes bug 22246; bugfix on 0.3.0.1-alpha.
+
+  o Minor features:
+    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (future-proofing):
+    - Tor no longer refuses to download microdescriptors or descriptors
+      if they are listed as "published in the future". This change will
+      eventually allow us to stop listing meaningful "published" dates
+      in microdescriptor consensuses, and thereby allow us to reduce the
+      resources required to download consensus diffs by over 50%.
+      Implements part of ticket 21642; implements part of proposal 275.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - The getpid() system call is now permitted under the Linux seccomp2
+      sandbox, to avoid crashing with versions of OpenSSL (and other
+      libraries) that attempt to learn the process's PID by using the
+      syscall rather than the VDSO code. Fixes bug 21943; bugfix
+      on 0.2.5.1-alpha.
+
+
+Changes in version 0.3.0.6 - 2017-04-26
+  Tor 0.3.0.6 is the first stable release of the Tor 0.3.0 series.
+
+  With the 0.3.0 series, clients and relays now use Ed25519 keys to
+  authenticate their link connections to relays, rather than the old
+  RSA1024 keys that they used before. (Circuit crypto has been
+  Curve25519-authenticated since 0.2.4.8-alpha.) We have also replaced
+  the guard selection and replacement algorithm to behave more robustly
+  in the presence of unreliable networks, and to resist guard-
+  capture attacks.
+
+  This series also includes numerous other small features and bugfixes,
+  along with more groundwork for the upcoming hidden-services revamp.
+
+  Per our stable release policy, we plan to support the Tor 0.3.0
+  release series for at least the next nine months, or for three months
+  after the first stable release of the 0.3.1 series: whichever is
+  longer. If you need a release with long-term support, we recommend
+  that you stay with the 0.2.9 series.
+
+  Below are the changes since 0.2.9.10. For a list of only the changes
+  since 0.3.0.5-rc, see the ChangeLog file.
+
+  o Major features (directory authority, security):
+    - The default for AuthDirPinKeys is now 1: directory authorities
+      will reject relays where the RSA identity key matches a previously
+      seen value, but the Ed25519 key has changed. Closes ticket 18319.
+
+  o Major features (guard selection algorithm):
+    - Tor's guard selection algorithm has been redesigned from the
+      ground up, to better support unreliable networks and restrictive
+      sets of entry nodes, and to better resist guard-capture attacks by
+      hostile local networks. Implements proposal 271; closes
+      ticket 19877.
+
+  o Major features (next-generation hidden services):
+    - Relays can now handle v3 ESTABLISH_INTRO cells as specified by
+      prop224 aka "Next Generation Hidden Services". Service and clients
+      don't use this functionality yet. Closes ticket 19043. Based on
+      initial code by Alec Heifetz.
+    - Relays now support the HSDir version 3 protocol, so that they can
+      can store and serve v3 descriptors. This is part of the next-
+      generation onion service work detailled in proposal 224. Closes
+      ticket 17238.
+
+  o Major features (protocol, ed25519 identity keys):
+    - Clients now support including Ed25519 identity keys in the EXTEND2
+      cells they generate. By default, this is controlled by a consensus
+      parameter, currently disabled. You can turn this feature on for
+      testing by setting ExtendByEd25519ID in your configuration. This
+      might make your traffic appear different than the traffic
+      generated by other users, however. Implements part of ticket
+      15056; part of proposal 220.
+    - Relays now understand requests to extend to other relays by their
+      Ed25519 identity keys. When an Ed25519 identity key is included in
+      an EXTEND2 cell, the relay will only extend the circuit if the
+      other relay can prove ownership of that identity. Implements part
+      of ticket 15056; part of proposal 220.
+    - Relays now use Ed25519 to prove their Ed25519 identities and to
+      one another, and to clients. This algorithm is faster and more
+      secure than the RSA-based handshake we've been doing until now.
+      Implements the second big part of proposal 220; Closes
+      ticket 15055.
+
+  o Major features (security):
+    - Change the algorithm used to decide DNS TTLs on client and server
+      side, to better resist DNS-based correlation attacks like the
+      DefecTor attack of Greschbach, Pulls, Roberts, Winter, and
+      Feamster. Now relays only return one of two possible DNS TTL
+      values, and clients are willing to believe DNS TTL values up to 3
+      hours long. Closes ticket 19769.
+
+  o Major bugfixes (client, onion service, also in 0.2.9.9):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Major bugfixes (crash, directory connections):
+    - Fix a rare crash when sending a begin cell on a circuit whose
+      linked directory connection had already been closed. Fixes bug
+      21576; bugfix on 0.2.9.3-alpha. Reported by Alec Muffett.
+
+  o Major bugfixes (directory authority):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (DNS):
+    - Fix a bug that prevented exit nodes from caching DNS records for
+      more than 60 seconds. Fixes bug 19025; bugfix on 0.2.4.7-alpha.
+
+  o Major bugfixes (IPv6 Exits):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+    - When parsing a malformed content-length field from an HTTP
+      message, do not read off the end of the buffer. This bug was a
+      potential remote denial-of-service attack against Tor clients and
+      relays. A workaround was released in October 2016, to prevent this
+      bug from crashing Tor. This is a fix for the underlying issue,
+      which should no longer matter (if you applied the earlier patch).
+      Fixes bug 20894; bugfix on 0.2.0.16-alpha. Bug found by fuzzing
+      using AFL (http://lcamtuf.coredump.cx/afl/).
+
+  o Major bugfixes (scheduler):
+    - Actually compare circuit policies in ewma_cmp_cmux(). This bug
+      caused the channel scheduler to behave more or less randomly,
+      rather than preferring channels with higher-priority circuits.
+      Fixes bug 20459; bugfix on 0.2.6.2-alpha.
+
+  o Major bugfixes (security, also in 0.2.9.9):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes--and having it
+      on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Minor feature (client):
+    - Enable IPv6 traffic on the SocksPort by default. To disable this,
+      a user will have to specify "NoIPv6Traffic". Closes ticket 21269.
+
+  o Minor feature (fallback scripts):
+    - Add a check_existing mode to updateFallbackDirs.py, which checks
+      if fallbacks in the hard-coded list are working. Closes ticket
+      20174. Patch by haxxpop.
+
+  o Minor feature (protocol versioning):
+    - Add new protocol version for proposal 224. HSIntro now advertises
+      version "3-4" and HSDir version "1-2". Fixes ticket 20656.
+
+  o Minor features (ciphersuite selection):
+    - Allow relays to accept a wider range of ciphersuites, including
+      chacha20-poly1305 and AES-CCM. Closes the other part of 15426.
+    - Clients now advertise a list of ciphersuites closer to the ones
+      preferred by Firefox. Closes part of ticket 15426.
+
+  o Minor features (controller):
+    - Add "GETINFO sr/current" and "GETINFO sr/previous" keys, to expose
+      shared-random values to the controller. Closes ticket 19925.
+    - When HSFETCH arguments cannot be parsed, say "Invalid argument"
+      rather than "unrecognized." Closes ticket 20389; patch from
+      Ivan Markin.
+
+  o Minor features (controller, configuration):
+    - Each of the *Port options, such as SocksPort, ORPort, ControlPort,
+      and so on, now comes with a __*Port variant that will not be saved
+      to the torrc file by the controller's SAVECONF command. This
+      change allows TorBrowser to set up a single-use domain socket for
+      each time it launches Tor. Closes ticket 20956.
+    - The GETCONF command can now query options that may only be
+      meaningful in context-sensitive lists. This allows the controller
+      to query the mixed SocksPort/__SocksPort style options introduced
+      in feature 20956. Implements ticket 21300.
+
+  o Minor features (diagnostic, directory client):
+    - Warn when we find an unexpected inconsistency in directory
+      download status objects. Prevents some negative consequences of
+      bug 20593.
+
+  o Minor features (directory authorities):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (directory authority):
+    - Add a new authority-only AuthDirTestEd25519LinkKeys option (on by
+      default) to control whether authorities should try to probe relays
+      by their Ed25519 link keys. This option will go away in a few
+      releases--unless we encounter major trouble in our ed25519 link
+      protocol rollout, in which case it will serve as a safety option.
+
+  o Minor features (directory cache):
+    - Relays and bridges will now refuse to serve the consensus they
+      have if they know it is too old for a client to use. Closes
+      ticket 20511.
+
+  o Minor features (ed25519 link handshake):
+    - Advertise support for the ed25519 link handshake using the
+      subprotocol-versions mechanism, so that clients can tell which
+      relays can identity themselves by Ed25519 ID. Closes ticket 20552.
+
+  o Minor features (entry guards):
+    - Add UseEntryGuards to TEST_OPTIONS_DEFAULT_VALUES in order to not
+      break regression tests.
+    - Require UseEntryGuards when UseBridges is set, in order to make
+      sure bridges aren't bypassed. Resolves ticket 20502.
+
+  o Minor features (fallback directories):
+    - Allow 3 fallback relays per operator, which is safe now that we
+      are choosing 200 fallback relays. Closes ticket 20912.
+    - Annotate updateFallbackDirs.py with the bandwidth and consensus
+      weight for each candidate fallback. Closes ticket 20878.
+    - Display the relay fingerprint when downloading consensuses from
+      fallbacks. Closes ticket 20908.
+    - Exclude relays affected by bug 20499 from the fallback list.
+      Exclude relays from the fallback list if they are running versions
+      known to be affected by bug 20499, or if in our tests they deliver
+      a stale consensus (i.e. one that expired more than 24 hours ago).
+      Closes ticket 20539.
+    - Make it easier to change the output sort order of fallbacks.
+      Closes ticket 20822.
+    - Reduce the minimum fallback bandwidth to 1 MByte/s. Part of
+      ticket 18828.
+    - Require fallback directories to have the same address and port for
+      7 days (now that we have enough relays with this stability).
+      Relays whose OnionOO stability timer is reset on restart by bug
+      18050 should upgrade to Tor 0.2.8.7 or later, which has a fix for
+      this issue. Closes ticket 20880; maintains short-term fix
+      in 0.2.8.2-alpha.
+    - Require fallbacks to have flags for 90% of the time (weighted
+      decaying average), rather than 95%. This allows at least 73% of
+      clients to bootstrap in the first 5 seconds without contacting an
+      authority. Part of ticket 18828.
+    - Select 200 fallback directories for each release. Closes
+      ticket 20881.
+
+  o Minor features (fingerprinting resistence, authentication):
+    - Extend the length of RSA keys used for TLS link authentication to
+      2048 bits. (These weren't used for forward secrecy; for forward
+      secrecy, we used P256.) Closes ticket 13752.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (geoip, also in 0.2.9.9):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (infrastructure):
+    - Implement smartlist_add_strdup() function. Replaces the use of
+      smartlist_add(sl, tor_strdup(str)). Closes ticket 20048.
+
+  o Minor features (linting):
+    - Enhance the changes file linter to warn on Tor versions that are
+      prefixed with "tor-". Closes ticket 21096.
+
+  o Minor features (logging):
+    - In several places, describe unset ed25519 keys as "<unset>",
+      rather than the scary "AAAAAAAA...AAA". Closes ticket 21037.
+
+  o Minor features (portability, compilation):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor features (relay):
+    - We now allow separation of exit and relay traffic to different
+      source IP addresses, using the OutboundBindAddressExit and
+      OutboundBindAddressOR options respectively. Closes ticket 17975.
+      Written by Michael Sonntag.
+
+  o Minor features (reliability, crash):
+    - Try better to detect problems in buffers where they might grow (or
+      think they have grown) over 2 GB in size. Diagnostic for
+      bug 21369.
+
+  o Minor features (testing):
+    - During 'make test-network-all', if tor logs any warnings, ask
+      chutney to output them. Requires a recent version of chutney with
+      the 21572 patch. Implements 21570.
+
+  o Minor bugfix (control protocol):
+    - The reply to a "GETINFO config/names" request via the control
+      protocol now spells the type "Dependent" correctly. This is a
+      breaking change in the control protocol. (The field seems to be
+      ignored by the most common known controllers.) Fixes bug 18146;
+      bugfix on 0.1.1.4-alpha.
+    - The GETINFO extra-info/digest/<digest> command was broken because
+      of a wrong base16 decode return value check, introduced when
+      refactoring that API. Fixes bug 22034; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfix (logging):
+    - Don't recommend the use of Tor2web in non-anonymous mode.
+      Recommending Tor2web is a bad idea because the client loses all
+      anonymity. Tor2web should only be used in specific cases by users
+      who *know* and understand the issues. Fixes bug 21294; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (bug resilience):
+    - Fix an unreachable size_t overflow in base64_decode(). Fixes bug
+      19222; bugfix on 0.2.0.9-alpha. Found by Guido Vranken; fixed by
+      Hans Jerry Illikainen.
+
+  o Minor bugfixes (build):
+    - Replace obsolete Autoconf macros with their modern equivalent and
+      prevent similar issues in the future. Fixes bug 20990; bugfix
+      on 0.1.0.1-rc.
+
+  o Minor bugfixes (certificate expiration time):
+    - Avoid using link certificates that don't become valid till some
+      time in the future. Fixes bug 21420; bugfix on 0.2.4.11-alpha
+
+  o Minor bugfixes (client):
+    - Always recover from failures in extend_info_from_node(), in an
+      attempt to prevent any recurrence of bug 21242. Fixes bug 21372;
+      bugfix on 0.2.3.1-alpha.
+    - When clients that use bridges start up with a cached consensus on
+      disk, they were ignoring it and downloading a new one. Now they
+      use the cached one. Fixes bug 20269; bugfix on 0.2.3.12-alpha.
+
+  o Minor bugfixes (code correctness):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (config):
+    - Don't assert on startup when trying to get the options list and
+      LearnCircuitBuildTimeout is set to 0: we are currently parsing the
+      options so of course they aren't ready yet. Fixes bug 21062;
+      bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (configuration):
+    - Accept non-space whitespace characters after the severity level in
+      the `Log` option. Fixes bug 19965; bugfix on 0.2.1.1-alpha.
+    - Support "TByte" and "TBytes" units in options given in bytes.
+      "TB", "terabyte(s)", "TBit(s)" and "terabit(s)" were already
+      supported. Fixes bug 20622; bugfix on 0.2.0.14-alpha.
+
+  o Minor bugfixes (configure, autoconf):
+    - Rename the configure option --enable-expensive-hardening to
+      --enable-fragile-hardening. Expensive hardening makes the tor
+      daemon abort when some kinds of issues are detected. Thus, it
+      makes tor more at risk of remote crashes but safer against RCE or
+      heartbleed bug category. We now try to explain this issue in a
+      message from the configure script. Fixes bug 21290; bugfix
+      on 0.2.5.4-alpha.
+
+  o Minor bugfixes (consensus weight):
+    - Add new consensus method that initializes bw weights to 1 instead
+      of 0. This prevents a zero weight from making it all the way to
+      the end (happens in small testing networks) and causing an error.
+      Fixes bug 14881; bugfix on 0.2.2.17-alpha.
+
+  o Minor bugfixes (crash prevention):
+    - Fix an (currently untriggerable, but potentially dangerous) crash
+      bug when base32-encoding inputs whose sizes are not a multiple of
+      5. Fixes bug 21894; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (dead code):
+    - Remove a redundant check for PidFile changes at runtime in
+      options_transition_allowed(): this check is already performed
+      regardless of whether the sandbox is active. Fixes bug 21123;
+      bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (descriptors):
+    - Correctly recognise downloaded full descriptors as valid, even
+      when using microdescriptors as circuits. This affects clients with
+      FetchUselessDescriptors set, and may affect directory authorities.
+      Fixes bug 20839; bugfix on 0.2.3.2-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Allow relays to use directory mirrors without a DirPort: these
+      relays need to be contacted over their ORPorts using a begindir
+      connection. Fixes one case of bug 20711; bugfix on 0.2.8.2-alpha.
+    - Clarify the message logged when a remote relay is unexpectedly
+      missing an ORPort or DirPort: users were confusing this with a
+      local port. Fixes another case of bug 20711; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (directory system):
+    - Bridges and relays now use microdescriptors (like clients do)
+      rather than old-style router descriptors. Now bridges will blend
+      in with clients in terms of the circuits they build. Fixes bug
+      6769; bugfix on 0.2.3.2-alpha.
+    - Download all consensus flavors, descriptors, and authority
+      certificates when FetchUselessDescriptors is set, regardless of
+      whether tor is a directory cache or not. Fixes bug 20667; bugfix
+      on all recent tor versions.
+
+  o Minor bugfixes (documentation):
+    - Update the tor manual page to document every option that can not
+      be changed while tor is running. Fixes bug 21122.
+
+  o Minor bugfixes (ed25519 certificates):
+    - Correctly interpret ed25519 certificates that would expire some
+      time after 19 Jan 2038. Fixes bug 20027; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (fallback directories):
+    - Avoid checking fallback candidates' DirPorts if they are down in
+      OnionOO. When a relay operator has multiple relays, this
+      prioritizes relays that are up over relays that are down. Fixes
+      bug 20926; bugfix on 0.2.8.3-alpha.
+    - Stop failing when OUTPUT_COMMENTS is True in updateFallbackDirs.py.
+      Fixes bug 20877; bugfix on 0.2.8.3-alpha.
+    - Stop failing when a relay has no uptime data in
+      updateFallbackDirs.py. Fixes bug 20945; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (hidden service):
+    - Clean up the code for expiring intro points with no associated
+      circuits. It was causing, rarely, a service with some expiring
+      introduction points to not open enough additional introduction
+      points. Fixes part of bug 21302; bugfix on 0.2.7.2-alpha.
+    - Resolve two possible underflows which could lead to creating and
+      closing a lot of introduction point circuits in a non-stop loop.
+      Fixes bug 21302; bugfix on 0.2.7.2-alpha.
+    - Stop setting the torrc option HiddenServiceStatistics to "0" just
+      because we're not a bridge or relay. Instead, we preserve whatever
+      value the user set (or didn't set). Fixes bug 21150; bugfix
+      on 0.2.6.2-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Make hidden services check for failed intro point connections,
+      even when they have exceeded their intro point creation limit.
+      Fixes bug 21596; bugfix on 0.2.7.2-alpha. Reported by Alec Muffett.
+    - Make hidden services with 8 to 10 introduction points check for
+      failed circuits immediately after startup. Previously, they would
+      wait for 5 minutes before performing their first checks. Fixes bug
+      21594; bugfix on 0.2.3.9-alpha. Reported by Alec Muffett.
+    - Stop ignoring misconfigured hidden services. Instead, refuse to
+      start tor until the misconfigurations have been corrected. Fixes
+      bug 20559; bugfix on multiple commits in 0.2.7.1-alpha
+      and earlier.
+
+  o Minor bugfixes (IPv6):
+    - Make IPv6-using clients try harder to find an IPv6 directory
+      server. Fixes bug 20999; bugfix on 0.2.8.2-alpha.
+    - When IPv6 addresses have not been downloaded yet (microdesc
+      consensus documents don't list relay IPv6 addresses), use hard-
+      coded addresses for authorities, fallbacks, and configured
+      bridges. Now IPv6-only clients can use microdescriptors. Fixes bug
+      20996; bugfix on b167e82 from 19608 in 0.2.8.5-alpha.
+
+  o Minor bugfixes (memory leak at exit):
+    - Fix a small harmless memory leak at exit of the previously unused
+      RSA->Ed identity cross-certificate. Fixes bug 17779; bugfix
+      on 0.2.7.2-alpha.
+
+  o Minor bugfixes (onion services):
+    - Allow the number of introduction points to be as low as 0, rather
+      than as low as 3. Fixes bug 21033; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (portability):
+    - Use "OpenBSD" compiler macro instead of "OPENBSD" or "__OpenBSD__".
+      It is supported by OpenBSD itself, and also by most OpenBSD
+      variants (such as Bitrig). Fixes bug 20980; bugfix
+      on 0.1.2.1-alpha.
+
+  o Minor bugfixes (portability, also in 0.2.9.9):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (relay):
+    - Avoid a double-marked-circuit warning that could happen when we
+      receive DESTROY cells under heavy load. Fixes bug 20059; bugfix
+      on 0.1.0.1-rc.
+    - Honor DataDirectoryGroupReadable when tor is a relay. Previously,
+      initializing the keys would reset the DataDirectory to 0700
+      instead of 0750 even if DataDirectoryGroupReadable was set to 1.
+      Fixes bug 19953; bugfix on 0.0.2pre16. Patch by "redfish".
+
+  o Minor bugfixes (testing):
+    - Fix Raspbian build issues related to missing socket errno in
+      test_util.c. Fixes bug 21116; bugfix on 0.2.8.2. Patch by "hein".
+    - Remove undefined behavior from the backtrace generator by removing
+      its signal handler. Fixes bug 21026; bugfix on 0.2.5.2-alpha.
+    - Use bash in src/test/test-network.sh. This ensures we reliably
+      call chutney's newer tools/test-network.sh when available. Fixes
+      bug 21562; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+  o Minor bugfixes (unit tests):
+    - Allow the unit tests to pass even when DNS lookups of bogus
+      addresses do not fail as expected. Fixes bug 20862 and 20863;
+      bugfix on unit tests introduced in 0.2.8.1-alpha
+      through 0.2.9.4-alpha.
+
+  o Minor bugfixes (util):
+    - When finishing writing a file to disk, if we were about to replace
+      the file with the temporary file created before and we fail to
+      replace it, remove the temporary file so it doesn't stay on disk.
+      Fixes bug 20646; bugfix on 0.2.0.7-alpha. Patch by fk.
+
+  o Minor bugfixes (Windows services):
+    - Be sure to initialize the monotonic time subsystem before using
+      it, even when running as an NT service. Fixes bug 21356; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (Windows):
+    - Check for getpagesize before using it to mmap files. This fixes
+      compilation in some MinGW environments. Fixes bug 20530; bugfix on
+      0.1.2.1-alpha. Reported by "ice".
+
+  o Code simplification and refactoring:
+    - Abolish all global guard context in entrynodes.c; replace with new
+      guard_selection_t structure as preparation for proposal 271.
+      Closes ticket 19858.
+    - Extract magic numbers in circuituse.c into defined variables.
+    - Introduce rend_service_is_ephemeral() that tells if given onion
+      service is ephemeral. Replace unclear NULL-checkings for service
+      directory with this function. Closes ticket 20526.
+    - Refactor circuit_is_available_for_use to remove unnecessary check.
+    - Refactor circuit_predict_and_launch_new for readability and
+      testability. Closes ticket 18873.
+    - Refactor code to manipulate global_origin_circuit_list into
+      separate functions. Closes ticket 20921.
+    - Refactor large if statement in purpose_needs_anonymity to use
+      switch statement instead. Closes part of ticket 20077.
+    - Refactor the hashing API to return negative values for errors, as
+      is done as throughout the codebase. Closes ticket 20717.
+    - Remove data structures that were used to index or_connection
+      objects by their RSA identity digests. These structures are fully
+      redundant with the similar structures used in the
+      channel abstraction.
+    - Remove duplicate code in the channel_write_*cell() functions.
+      Closes ticket 13827; patch from Pingl.
+    - Remove redundant behavior of is_sensitive_dir_purpose, refactor to
+      use only purpose_needs_anonymity. Closes part of ticket 20077.
+    - The code to generate and parse EXTEND and EXTEND2 cells has been
+      replaced with code automatically generated by the
+      "trunnel" utility.
+
+  o Documentation (formatting):
+    - Clean up formatting of tor.1 man page and HTML doc, where <pre>
+      blocks were incorrectly appearing. Closes ticket 20885.
+
+  o Documentation (man page):
+    - Clarify many options in tor.1 and add some min/max values for
+      HiddenService options. Closes ticket 21058.
+
+  o Documentation:
+    - Change '1' to 'weight_scale' in consensus bw weights calculation
+      comments, as that is reality. Closes ticket 20273. Patch
+      from pastly.
+    - Clarify that when ClientRejectInternalAddresses is enabled (which
+      is the default), multicast DNS hostnames for machines on the local
+      network (of the form *.local) are also rejected. Closes
+      ticket 17070.
+    - Correct the value for AuthDirGuardBWGuarantee in the manpage, from
+      250 KBytes to 2 MBytes. Fixes bug 20435; bugfix on 0.2.5.6-alpha.
+    - Include the "TBits" unit in Tor's man page. Fixes part of bug
+      20622; bugfix on 0.2.5.1-alpha.
+    - Small fixes to the fuzzing documentation. Closes ticket 21472.
+    - Stop the man page from incorrectly stating that HiddenServiceDir
+      must already exist. Fixes 20486.
+    - Update the description of the directory server options in the
+      manual page, to clarify that a relay no longer needs to set
+      DirPort in order to be a directory cache. Closes ticket 21720.
+
+  o Removed features:
+    - The AuthDirMaxServersPerAuthAddr option no longer exists: The same
+      limit for relays running on a single IP applies to authority IP
+      addresses as well as to non-authority IP addresses. Closes
+      ticket 20960.
+    - The UseDirectoryGuards torrc option no longer exists: all users
+      that use entry guards will also use directory guards. Related to
+      proposal 271; implements part of ticket 20831.
+
+  o Testing:
+    - Add tests for networkstatus_compute_bw_weights_v10.
+    - Add unit tests circuit_predict_and_launch_new.
+    - Extract dummy_origin_circuit_new so it can be used by other
+      test functions.
+    - New unit tests for tor_htonll(). Closes ticket 19563. Patch
+      from "overcaffeinated".
+    - Perform the coding style checks when running the tests and fail
+      when coding style violations are found. Closes ticket 5500.
+
+
+Changes in version 0.2.8.13 - 2017-03-03
+  Tor 0.2.8.13 backports a security fix from later Tor
+  releases.  Anybody running Tor 0.2.8.12 or earlier should upgrade to this
+  this release, if for some reason they cannot upgrade to a later
+  release series, and if they build Tor with the --enable-expensive-hardening
+  option.
+
+  Note that support for Tor 0.2.8.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.8.x series after 1 Jan 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.7.7 - 2017-03-03
+  Tor 0.2.7.7 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.7.6 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.7.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.7.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.6.11 - 2017-03-03
+  Tor 0.2.6.11 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.6.10 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.6.x is ending this year: we will not issue
+  any fixes for the Tor 0.2.6.x series after 1 August 2017.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+
+Changes in version 0.2.5.13 - 2017-03-03
+  Tor 0.2.5.13 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.5.13 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.5.x is ending next year: we will not issue
+  any fixes for the Tor 0.2.5.x series after 1 May 2018.  If you need
+  a Tor release series with longer-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, client, DNS proxy, backport from 0.2.8.3-alpha):
+    - Stop a crash that could occur when a client running with DNSPort
+      received a query with multiple address types, and the first
+      address type was not supported. Found and fixed by Scott Dial.
+      Fixes bug 18710; bugfix on 0.2.5.4-alpha.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (crypto error-handling, backport from 0.2.7.2-alpha):
+    - Check for failures from crypto_early_init, and refuse to continue.
+      A previous typo meant that we could keep going with an
+      uninitialized crypto library, and would have OpenSSL initialize
+      its own PRNG. Fixes bug 16360; bugfix on 0.2.5.2-alpha, introduced
+      when implementing ticket 4900. Patch by "teor".
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.4.28 - 2017-03-03
+  Tor 0.2.4.28 backports a number of security fixes from later Tor
+  releases.  Anybody running Tor 0.2.4.27 or earlier should upgrade to
+  this release, if for some reason they cannot upgrade to a later
+  release series.
+
+  Note that support for Tor 0.2.4.x is ending soon: we will not issue
+  any fixes for the Tor 0.2.4.x series after 1 August 2017.  If you need
+  a Tor release series with long-term support, we recommend Tor 0.2.9.x.
+
+  o Directory authority changes (backport from 0.2.8.5-rc):
+    - Urras is no longer a directory authority. Closes ticket 19271.
+
+  o Directory authority changes (backport from 0.2.9.2-alpha):
+    - The "Tonga" bridge authority has been retired; the new bridge
+      authority is "Bifroest". Closes tickets 19728 and 19690.
+
+  o Directory authority key updates (backport from 0.2.8.1-alpha):
+    - Update the V3 identity key for the dannenberg directory authority:
+      it was changed on 18 November 2015. Closes task 17906. Patch
+      by "teor".
+
+  o Major features (security fixes, backport from 0.2.9.4-alpha):
+    - Prevent a class of security bugs caused by treating the contents
+      of a buffer chunk as if they were a NUL-terminated string. At
+      least one such bug seems to be present in all currently used
+      versions of Tor, and would allow an attacker to remotely crash
+      most Tor instances, especially those compiled with extra compiler
+      hardening. With this defense in place, such bugs can't crash Tor,
+      though we should still fix them as they occur. Closes ticket
+      20384 (TROVE-2016-10-001).
+
+  o Major bugfixes (parsing, security, backport from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major bugfixes (security, correctness, backport from 0.2.7.4-rc):
+    - Fix an error that could cause us to read 4 bytes before the
+      beginning of an openssl string. This bug could be used to cause
+      Tor to crash on systems with unusual malloc implementations, or
+      systems with unusual hardening installed. Fixes bug 17404; bugfix
+      on 0.2.3.6-alpha.
+
+  o Major bugfixes (security, pointers, backport from 0.2.8.2-alpha):
+    - Avoid a difficult-to-trigger heap corruption attack when extending
+      a smartlist to contain over 16GB of pointers. Fixes bug 18162;
+      bugfix on 0.1.1.11-alpha, which fixed a related bug incompletely.
+      Reported by Guido Vranken.
+
+  o Major bugfixes (dns proxy mode, crash, backport from 0.2.8.2-alpha):
+    - Avoid crashing when running as a DNS proxy. Fixes bug 16248;
+      bugfix on 0.2.0.1-alpha. Patch from "cypherpunks".
+
+  o Major bugfixes (guard selection, backport from 0.2.7.6):
+    - Actually look at the Guard flag when selecting a new directory
+      guard. When we implemented the directory guard design, we
+      accidentally started treating all relays as if they have the Guard
+      flag during guard selection, leading to weaker anonymity and worse
+      performance. Fixes bug 17772; bugfix on 0.2.4.8-alpha. Discovered
+      by Mohsen Imani.
+
+  o Major bugfixes (key management, backport from 0.2.8.3-alpha):
+    - If OpenSSL fails to generate an RSA key, do not retain a dangling
+      pointer to the previous (uninitialized) key value. The impact here
+      should be limited to a difficult-to-trigger crash, if OpenSSL is
+      running an engine that makes key generation failures possible, or
+      if OpenSSL runs out of memory. Fixes bug 19152; bugfix on
+      0.2.1.10-alpha. Found by Yuan Jochen Kang, Suman Jana, and
+      Baishakhi Ray.
+
+  o Major bugfixes (parsing, backported from 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (security, memory erasure, backport from 0.2.8.1-alpha):
+    - Make memwipe() do nothing when passed a NULL pointer or buffer of
+      zero size. Check size argument to memwipe() for underflow. Fixes
+      bug 18089; bugfix on 0.2.3.25 and 0.2.4.6-alpha. Reported by "gk",
+      patch by "teor".
+
+  o Minor features (bug-resistance, backport from 0.2.8.2-alpha):
+    - Make Tor survive errors involving connections without a
+      corresponding event object. Previously we'd fail with an
+      assertion; now we produce a log message. Related to bug 16248.
+
+  o Minor features (DoS-resistance, backport from 0.2.7.1-alpha):
+    - Make it harder for attackers to overload hidden services with
+      introductions, by blocking multiple introduction requests on the
+      same circuit. Resolves ticket 15515.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (compilation, backport from 0.2.7.6):
+    - Fix a compilation warning with Clang 3.6: Do not check the
+      presence of an address which can never be NULL. Fixes bug 17781.
+
+  o Minor bugfixes (hidden service, backport from 0.2.7.1-alpha):
+    - Fix an out-of-bounds read when parsing invalid INTRODUCE2 cells on
+      a client authorized hidden service. Fixes bug 15823; bugfix
+      on 0.2.1.6-alpha.
+
+
+Changes in version 0.2.9.10 - 2017-03-01
+  Tor 0.2.9.10 backports a security fix from later Tor release.  It also
+  includes fixes for some major issues affecting directory authorities,
+  LibreSSL compatibility, and IPv6 correctness.
+
+  The Tor 0.2.9.x release series is now marked as a long-term-support
+  series.  We intend to backport security fixes to 0.2.9.x until at
+  least January of 2020.
+
+  o Major bugfixes (directory authority, 0.3.0.3-alpha):
+    - During voting, when marking a relay as a probable sybil, do not
+      clear its BadExit flag: sybils can still be bad in other ways
+      too. (We still clear the other flags.) Fixes bug 21108; bugfix
+      on 0.2.0.13-alpha.
+
+  o Major bugfixes (IPv6 Exits, backport from 0.3.0.3-alpha):
+    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects
+      any IPv6 addresses. Instead, only reject a port over IPv6 if the
+      exit policy rejects that port on more than an IPv6 /16 of
+      addresses. This bug was made worse by 17027 in 0.2.8.1-alpha,
+      which rejected a relay's own IPv6 address by default. Fixes bug
+      21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
+
+  o Major bugfixes (parsing, also in 0.3.0.4-rc):
+    - Fix an integer underflow bug when comparing malformed Tor
+      versions. This bug could crash Tor when built with
+      --enable-expensive-hardening, or on Tor 0.2.9.1-alpha through Tor
+      0.2.9.8, which were built with -ftrapv by default. In other cases
+      it was harmless. Part of TROVE-2017-001. Fixes bug 21278; bugfix
+      on 0.0.8pre1. Found by OSS-Fuzz.
+
+  o Minor features (directory authorities, also in 0.3.0.4-rc):
+    - Directory authorities now reject descriptors that claim to be
+      malformed versions of Tor. Helps prevent exploitation of
+      bug 21278.
+    - Reject version numbers with components that exceed INT32_MAX.
+      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
+      Fixes bug 21450; bugfix on 0.0.8pre1.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor features (portability, compilation, backport from 0.3.0.3-alpha):
+    - Autoconf now checks to determine if OpenSSL structures are opaque,
+      instead of explicitly checking for OpenSSL version numbers. Part
+      of ticket 21359.
+    - Support building with recent LibreSSL code that uses opaque
+      structures. Closes ticket 21359.
+
+  o Minor bugfixes (code correctness, also in 0.3.0.4-rc):
+    - Repair a couple of (unreachable or harmless) cases of the risky
+      comparison-by-subtraction pattern that caused bug 21278.
+
+  o Minor bugfixes (tor-resolve, backport from 0.3.0.3-alpha):
+    - The tor-resolve command line tool now rejects hostnames over 255
+      characters in length. Previously, it would silently truncate them,
+      which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
+      Patch by "junglefowl".
+
+
+Changes in version 0.2.9.9 - 2017-01-23
+  Tor 0.2.9.9 fixes a denial-of-service bug where an attacker could
+  cause relays and clients to crash, even if they were not built with
+  the --enable-expensive-hardening option. This bug affects all 0.2.9.x
+  versions, and also affects 0.3.0.1-alpha: all relays running an affected
+  version should upgrade.
+
+  This release also resolves a client-side onion service reachability
+  bug, and resolves a pair of small portability issues.
+
+  o Major bugfixes (security):
+    - Downgrade the "-ftrapv" option from "always on" to "only on when
+      --enable-expensive-hardening is provided." This hardening option,
+      like others, can turn survivable bugs into crashes -- and having
+      it on by default made a (relatively harmless) integer overflow bug
+      into a denial-of-service bug. Fixes bug 21278 (TROVE-2017-001);
+      bugfix on 0.2.9.1-alpha.
+
+  o Major bugfixes (client, onion service):
+    - Fix a client-side onion service reachability bug, where multiple
+      socks requests to an onion service (or a single slow request)
+      could cause us to mistakenly mark some of the service's
+      introduction points as failed, and we cache that failure so
+      eventually we run out and can't reach the service. Also resolves a
+      mysterious "Remote server sent bogus reason code 65021" log
+      warning. The bug was introduced in ticket 17218, where we tried to
+      remember the circuit end reason as a uint16_t, which mangled
+      negative values. Partially fixes bug 21056 and fixes bug 20307;
+      bugfix on 0.2.8.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
+      Country database.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing when Tor is built using headers that contain
+      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
+      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix
+      on 0.2.9.1-alpha.
+    - Fix Libevent detection on platforms without Libevent 1 headers
+      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
+
+
+Changes in version 0.2.8.12 - 2016-12-19
+  Tor 0.2.8.12 backports a fix for a medium-severity issue (bug 21018
+  below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  It also includes an updated list of fallback directories, backported
+  from 0.2.9.
+
+  Now that the Tor 0.2.9 series is stable, only major bugfixes will be
+  backported to 0.2.8 in the future.
+
+  o Major bugfixes (parsing, security, backported from 0.2.9.8):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Minor features (fallback directory list, backported from 0.2.9.8):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (geoip, backported from 0.2.9.7-rc):
+    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
+      Country database.
+
+
+Changes in version 0.2.9.8 - 2016-12-19
+  Tor 0.2.9.8 is the first stable release of the Tor 0.2.9 series.
+
+  The Tor 0.2.9 series makes mandatory a number of security features
+  that were formerly optional. It includes support for a new shared-
+  randomness protocol that will form the basis for next generation
+  hidden services, includes a single-hop hidden service mode for
+  optimizing .onion services that don't actually want to be hidden,
+  tries harder not to overload the directory authorities with excessive
+  downloads, and supports a better protocol versioning scheme for
+  improved compatibility with other implementations of the Tor protocol.
+
+  And of course, there are numerous other bugfixes and improvements.
+
+  This release also includes a fix for a medium-severity issue (bug
+  21018 below) where Tor clients could crash when attempting to visit a
+  hostile hidden service. Clients are recommended to upgrade as packages
+  become available for their systems.
+
+  Below are listed the changes since Tor 0.2.8.11.  For a list of
+  changes since 0.2.9.7-rc, see the ChangeLog file.
+
+  o New system requirements:
+    - When building with OpenSSL, Tor now requires version 1.0.1 or
+      later. OpenSSL 1.0.0 and earlier are no longer supported by the
+      OpenSSL team, and should not be used. Closes ticket 20303.
+    - Tor now requires Libevent version 2.0.10-stable or later. Older
+      versions of Libevent have less efficient backends for several
+      platforms, and lack the DNS code that we use for our server-side
+      DNS support. This implements ticket 19554.
+    - Tor now requires zlib version 1.2 or later, for security,
+      efficiency, and (eventually) gzip support. (Back when we started,
+      zlib 1.1 and zlib 1.0 were still found in the wild. 1.2 was
+      released in 2003. We recommend the latest version.)
+
+  o Deprecated features:
+    - A number of DNS-cache-related sub-options for client ports are now
+      deprecated for security reasons, and may be removed in a future
+      version of Tor. (We believe that client-side DNS caching is a bad
+      idea for anonymity, and you should not turn it on.) The options
+      are: CacheDNS, CacheIPv4DNS, CacheIPv6DNS, UseDNSCache,
+      UseIPv4Cache, and UseIPv6Cache.
+    - A number of options are deprecated for security reasons, and may
+      be removed in a future version of Tor. The options are:
+      AllowDotExit, AllowInvalidNodes, AllowSingleHopCircuits,
+      AllowSingleHopExits, ClientDNSRejectInternalAddresses,
+      CloseHSClientCircuitsImmediatelyOnTimeout,
+      CloseHSServiceRendCircuitsImmediatelyOnTimeout,
+      ExcludeSingleHopRelays, FastFirstHopPK, TLSECGroup,
+      UseNTorHandshake, and WarnUnsafeSocks.
+    - The *ListenAddress options are now deprecated as unnecessary: the
+      corresponding *Port options should be used instead. These options
+      may someday be removed. The affected options are:
+      ControlListenAddress, DNSListenAddress, DirListenAddress,
+      NATDListenAddress, ORListenAddress, SocksListenAddress,
+      and TransListenAddress.
+
+  o Major bugfixes (parsing, security, new since 0.2.9.7-rc):
+    - Fix a bug in parsing that could cause clients to read a single
+      byte past the end of an allocated region. This bug could be used
+      to cause hardened clients (built with --enable-expensive-hardening)
+      to crash if they tried to visit a hostile hidden service. Non-
+      hardened clients are only affected depending on the details of
+      their platform's memory allocator. Fixes bug 21018; bugfix on
+      0.2.0.8-alpha. Found by using libFuzzer. Also tracked as TROVE-
+      2016-12-002 and as CVE-2016-1254.
+
+  o Major features (build, hardening):
+    - Tor now builds with -ftrapv by default on compilers that support
+      it. This option detects signed integer overflow (which C forbids),
+      and turns it into a hard-failure. We do not apply this option to
+      code that needs to run in constant time to avoid side-channels;
+      instead, we use -fwrapv in that code. Closes ticket 17983.
+    - When --enable-expensive-hardening is selected, stop applying the
+      clang/gcc sanitizers to code that needs to run in constant time.
+      Although we are aware of no introduced side-channels, we are not
+      able to prove that there are none. Related to ticket 17983.
+
+  o Major features (circuit building, security):
+    - Authorities, relays, and clients now require ntor keys in all
+      descriptors, for all hops (except for rare hidden service protocol
+      cases), for all circuits, and for all other roles. Part of
+      ticket 19163.
+    - Authorities, relays, and clients only use ntor, except for
+      rare cases in the hidden service protocol. Part of ticket 19163.
+
+  o Major features (compilation):
+    - Our big list of extra GCC warnings is now enabled by default when
+      building with GCC (or with anything like Clang that claims to be
+      GCC-compatible). To make all warnings into fatal compilation
+      errors, pass --enable-fatal-warnings to configure. Closes
+      ticket 19044.
+    - Use the Autoconf macro AC_USE_SYSTEM_EXTENSIONS to automatically
+      turn on C and POSIX extensions. (Previously, we attempted to do
+      this on an ad hoc basis.) Closes ticket 19139.
+
+  o Major features (directory authorities, hidden services):
+    - Directory authorities can now perform the shared randomness
+      protocol specified by proposal 250. Using this protocol, directory
+      authorities generate a global fresh random value every day. In the
+      future, this value will be used by hidden services to select
+      HSDirs. This release implements the directory authority feature;
+      the hidden service side will be implemented in the future as part
+      of proposal 224. Resolves ticket 16943; implements proposal 250.
+
+  o Major features (downloading, random exponential backoff):
+    - When we fail to download an object from a directory service, wait
+      for an (exponentially increasing) randomized amount of time before
+      retrying, rather than a fixed interval as we did before. This
+      prevents a group of Tor instances from becoming too synchronized,
+      or a single Tor instance from becoming too predictable, in its
+      download schedule. Closes ticket 15942.
+
+  o Major features (resource management):
+    - Tor can now notice it is about to run out of sockets, and
+      preemptively close connections of lower priority. (This feature is
+      off by default for now, since the current prioritizing method is
+      yet not mature enough. You can enable it by setting
+      "DisableOOSCheck 0", but watch out: it might close some sockets
+      you would rather have it keep.) Closes ticket 18640.
+
+  o Major features (single-hop "hidden" services):
+    - Add experimental HiddenServiceSingleHopMode and
+      HiddenServiceNonAnonymousMode options. When both are set to 1,
+      every hidden service on that Tor instance becomes a non-anonymous
+      Single Onion Service. Single Onions make one-hop (direct)
+      connections to their introduction and rendezvous points. One-hop
+      circuits make Single Onion servers easily locatable, but clients
+      remain location-anonymous. This is compatible with the existing
+      hidden service implementation, and works on the current Tor
+      network without any changes to older relays or clients. Implements
+      proposal 260, completes ticket 17178. Patch by teor and asn.
+
+  o Major features (subprotocol versions):
+    - Tor directory authorities now vote on a set of recommended
+      "subprotocol versions", and on a set of required subprotocol
+      versions. Clients and relays that lack support for a _required_
+      subprotocol version will not start; those that lack support for a
+      _recommended_ subprotocol version will warn the user to upgrade.
+      This change allows compatible implementations of the Tor protocol(s)
+      to exist without pretending to be 100% bug-compatible with
+      particular releases of Tor itself. Closes ticket 19958; implements
+      part of proposal 264.
+
+  o Major bugfixes (circuit building):
+    - Hidden service client-to-intro-point and service-to-rendezvous-
+      point circuits use the TAP key supplied by the protocol, to avoid
+      epistemic attacks. Fixes bug 19163; bugfix on 0.2.4.18-rc.
+
+  o Major bugfixes (download scheduling):
+    - Avoid resetting download status for consensuses hourly, since we
+      already have another, smarter retry mechanism. Fixes bug 8625;
+      bugfix on 0.2.0.9-alpha.
+    - If a consensus expires while we are waiting for certificates to
+      download, stop waiting for certificates.
+    - If we stop waiting for certificates less than a minute after we
+      started downloading them, do not consider the certificate download
+      failure a separate failure. Fixes bug 20533; bugfix
+      on 0.2.0.9-alpha.
+    - When using exponential backoff in test networks, use a lower
+      exponent, so the delays do not vary as much. This helps test
+      networks bootstrap consistently. Fixes bug 20597; bugfix on 20499.
+
+  o Major bugfixes (exit policies):
+    - Avoid disclosing exit outbound bind addresses, configured port
+      bind addresses, and local interface addresses in relay descriptors
+      by default under ExitPolicyRejectPrivate. Instead, only reject
+      these (otherwise unlisted) addresses if
+      ExitPolicyRejectLocalInterfaces is set. Fixes bug 18456; bugfix on
+      0.2.7.2-alpha. Patch by teor.
+
+  o Major bugfixes (hidden services):
+    - Allow Tor clients with appropriate controllers to work with
+      FetchHidServDescriptors set to 0. Previously, this option also
+      disabled descriptor cache lookup, thus breaking hidden services
+      entirely. Fixes bug 18704; bugfix on 0.2.0.20-rc. Patch by "twim".
+    - Clients now require hidden services to include the TAP keys for
+      their intro points in the hidden service descriptor. This prevents
+      an inadvertent upgrade to ntor, which a malicious hidden service
+      could use to distinguish clients by consensus version. Fixes bug
+      20012; bugfix on 0.2.4.8-alpha. Patch by teor.
+
+  o Major bugfixes (relay, resolver, logging):
+    - For relays that don't know their own address, avoid attempting a
+      local hostname resolve for each descriptor we download. This
+      will cut down on the number of "Success: chose address 'x.x.x.x'"
+      log lines, and also avoid confusing clock jumps if the resolver
+      is slow. Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
+
+  o Minor features (port flags):
+    - Add new flags to the *Port options to give finer control over which
+      requests are allowed. The flags are NoDNSRequest, NoOnionTraffic,
+      and the synthetic flag OnionTrafficOnly, which is equivalent to
+      NoDNSRequest, NoIPv4Traffic, and NoIPv6Traffic. Closes enhancement
+      18693; patch by "teor".
+
+  o Minor features (build, hardening):
+    - Detect and work around a libclang_rt problem that would prevent
+      clang from finding __mulodi4() on some 32-bit platforms, and thus
+      keep -ftrapv from linking on those systems. Closes ticket 19079.
+    - When building on a system without runtime support for the runtime
+      hardening options, try to log a useful warning at configuration
+      time, rather than an incomprehensible warning at link time. If
+      expensive hardening was requested, this warning becomes an error.
+      Closes ticket 18895.
+
+  o Minor features (client, directory):
+    - Since authorities now omit all routers that lack the Running and
+      Valid flags, we assume that any relay listed in the consensus must
+      have those flags. Closes ticket 20001; implements part of
+      proposal 272.
+
+  o Minor features (code safety):
+    - In our integer-parsing functions, ensure that the maximum value we
+      allow is no smaller than the minimum value. Closes ticket 19063;
+      patch from "U+039b".
+
+  o Minor features (compilation, portability):
+    - Compile correctly on MacOS 10.12 (aka "Sierra"). Closes
+      ticket 20241.
+
+  o Minor features (config):
+    - Warn users when descriptor and port addresses are inconsistent.
+      Mitigates bug 13953; patch by teor.
+
+  o Minor features (controller):
+    - Allow controllers to configure basic client authorization on
+      hidden services when they create them with the ADD_ONION controller
+      command. Implements ticket 15588. Patch by "special".
+    - Fire a STATUS_SERVER controller event whenever the hibernation
+      status changes between "awake"/"soft"/"hard". Closes ticket 18685.
+    - Implement new GETINFO queries for all downloads that use
+      download_status_t to schedule retries. This allows controllers to
+      examine the schedule for pending downloads. Closes ticket 19323.
+
+  o Minor features (development tools, etags):
+    - Teach the "make tags" Makefile target how to correctly find
+      "MOCK_IMPL" function definitions. Patch from nherring; closes
+      ticket 16869.
+
+  o Minor features (directory authority):
+    - After voting, if the authorities decide that a relay is not
+      "Valid", they no longer include it in the consensus at all. Closes
+      ticket 20002; implements part of proposal 272.
+    - Directory authorities now only give the Guard flag to a relay if
+      they are also giving it the Stable flag. This change allows us to
+      simplify path selection for clients. It should have minimal effect
+      in practice, since >99% of Guards already have the Stable flag.
+      Implements ticket 18624.
+    - Directory authorities now write their v3-status-votes file out to
+      disk earlier in the consensus process, so we have a record of the
+      votes even if we abort the consensus process. Resolves
+      ticket 19036.
+
+  o Minor features (fallback directory list, new since 0.2.9.7-rc):
+    - Replace the 81 remaining fallbacks of the 100 originally
+      introduced in Tor 0.2.8.3-alpha in March 2016, with a list of 177
+      fallbacks (123 new, 54 existing, 27 removed) generated in December
+      2016. Resolves ticket 20170.
+
+  o Minor features (hidden service):
+    - Stop being so strict about the payload length of "rendezvous1"
+      cells. We used to be locked in to the "TAP" handshake length, and
+      now we can handle better handshakes like "ntor". Resolves
+      ticket 18998.
+
+  o Minor features (infrastructure, time):
+    - Tor now includes an improved timer backend, so that we can
+      efficiently support tens or hundreds of thousands of concurrent
+      timers, as will be needed for some of our planned anti-traffic-
+      analysis work. This code is based on William Ahern's "timeout.c"
+      project, which implements a "tickless hierarchical timing wheel".
+      Closes ticket 18365.
+    - Tor now uses the operating system's monotonic timers (where
+      available) for internal fine-grained timing. Previously we would
+      look at the system clock, and then attempt to compensate for the
+      clock running backwards. Closes ticket 18908.
+
+  o Minor features (logging):
+    - Add a set of macros to check nonfatal assertions, for internal
+      use. Migrating more of our checks to these should help us avoid
+      needless crash bugs. Closes ticket 18613.
+    - Provide a more useful warning message when configured with an
+      invalid Nickname. Closes ticket 18300; patch from "icanhasaccount".
+    - When dumping unparseable router descriptors, optionally store them
+      in separate files, named by digest, up to a configurable size
+      limit. You can change the size limit by setting the
+      MaxUnparseableDescSizeToLog option, and disable this feature by
+      setting that option to 0. Closes ticket 18322.
+
+  o Minor features (performance):
+    - Change the "optimistic data" extension from "off by default" to
+      "on by default". The default was ordinarily overridden by a
+      consensus option, but when clients were bootstrapping for the
+      first time, they would not have a consensus to get the option
+      from. Changing this default saves a round-trip during startup.
+      Closes ticket 18815.
+
+  o Minor features (relay, usability):
+    - When the directory authorities refuse a bad relay's descriptor,
+      encourage the relay operator to contact us. Many relay operators
+      won't notice this line in their logs, but it's a win if even a few
+      learn why we don't like what their relay was doing. Resolves
+      ticket 18760.
+
+  o Minor features (security, TLS):
+    - Servers no longer support clients that lack AES ciphersuites.
+      (3DES is no longer considered an acceptable cipher.) We believe
+      that no such Tor clients currently exist, since Tor has required
+      OpenSSL 0.9.7 or later since 2009. Closes ticket 19998.
+
+  o Minor features (testing):
+    - Disable memory protections on OpenBSD when performing our unit
+      tests for memwipe(). The test deliberately invokes undefined
+      behavior, and the OpenBSD protections interfere with this. Patch
+      from "rubiate". Closes ticket 20066.
+    - Move the test-network.sh script to chutney, and modify tor's test-
+      network.sh to call the (newer) chutney version when available.
+      Resolves ticket 19116. Patch by teor.
+    - Use the lcov convention for marking lines as unreachable, so that
+      we don't count them when we're generating test coverage data.
+      Update our coverage tools to understand this convention. Closes
+      ticket 16792.
+    - Our link-handshake unit tests now check that when invalid
+      handshakes fail, they fail with the error messages we expected.
+    - Our unit testing code that captures log messages no longer
+      prevents them from being written out if the user asked for them
+      (by passing --debug or --info or --notice or --warn to the "test"
+      binary). This change prevents us from missing unexpected log
+      messages simply because we were looking for others. Related to
+      ticket 19999.
+    - The unit tests now log all warning messages with the "BUG" flag.
+      Previously, they only logged errors by default. This change will
+      help us make our testing code more correct, and make sure that we
+      only hit this code when we mean to. In the meantime, however,
+      there will be more warnings in the unit test logs than before.
+      This is preparatory work for ticket 19999.
+    - The unit tests now treat any failure of a "tor_assert_nonfatal()"
+      assertion as a test failure.
+    - We've done significant work to make the unit tests run faster.
+
+  o Minor features (testing, ipv6):
+    - Add the hs-ipv6 chutney target to make test-network-all's IPv6
+      tests. Remove bridges+hs, as it's somewhat redundant. This
+      requires a recent chutney version that supports IPv6 clients,
+      relays, and authorities. Closes ticket 20069; patch by teor.
+    - Add the single-onion and single-onion-ipv6 chutney targets to
+      "make test-network-all". This requires a recent chutney version
+      with the single onion network flavors (git c72a652 or later).
+      Closes ticket 20072; patch by teor.
+
+  o Minor features (Tor2web):
+    - Make Tor2web clients respect ReachableAddresses. This feature was
+      inadvertently enabled in 0.2.8.6, then removed by bugfix 19973 on
+      0.2.8.7. Implements feature 20034. Patch by teor.
+
+  o Minor features (unix domain sockets):
+    - When configuring a unix domain socket for a SocksPort,
+      ControlPort, or Hidden service, you can now wrap the address in
+      quotes, using C-style escapes inside the quotes. This allows unix
+      domain socket paths to contain spaces. Resolves ticket 18753.
+
+  o Minor features (user interface):
+    - Tor now supports the ability to declare options deprecated, so
+      that we can recommend that people stop using them. Previously, this
+      was done in an ad-hoc way. There is a new --list-deprecated-options
+      command-line option to list all of the deprecated options. Closes
+      ticket 19820.
+
+  o Minor features (virtual addresses):
+    - Increase the maximum number of bits for the IPv6 virtual network
+      prefix from 16 to 104. In this way, the condition for address
+      allocation is less restrictive. Closes ticket 20151; feature
+      on 0.2.4.7-alpha.
+
+  o Minor bug fixes (circuits):
+    - Use the CircuitBuildTimeout option whenever
+      LearnCircuitBuildTimeout is disabled. Previously, we would respect
+      the option when a user disabled it, but not when it was disabled
+      because some other option was set. Fixes bug 20073; bugfix on
+      0.2.4.12-alpha. Patch by teor.
+
+  o Minor bugfixes (build):
+    - The current Git revision when building from a local repository is
+      now detected correctly when using git worktrees. Fixes bug 20492;
+      bugfix on 0.2.3.9-alpha.
+
+  o Minor bugfixes (relay address discovery):
+    - Stop reordering IP addresses returned by the OS. This makes it
+      more likely that Tor will guess the same relay IP address every
+      time. Fixes issue 20163; bugfix on 0.2.7.1-alpha, ticket 17027.
+      Reported by René Mayrhofer, patch by "cypherpunks".
+
+  o Minor bugfixes (memory allocation):
+    - Change how we allocate memory for large chunks on buffers, to
+      avoid a (currently impossible) integer overflow, and to waste less
+      space when allocating unusually large chunks. Fixes bug 20081;
+      bugfix on 0.2.0.16-alpha. Issue identified by Guido Vranken.
+
+  o Minor bugfixes (bootstrap):
+    - Remember the directory server we fetched the consensus or previous
+      certificates from, and use it to fetch future authority
+      certificates. This change improves bootstrapping performance.
+      Fixes bug 18963; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (circuits):
+    - Make sure extend_info_from_router() is only called on servers.
+      Fixes bug 19639; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (client, fascistfirewall):
+    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
+      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
+
+  o Minor bugfixes (client, unix domain sockets):
+    - Disable IsolateClientAddr when using AF_UNIX backed SocksPorts as
+      the client address is meaningless. Fixes bug 20261; bugfix
+      on 0.2.6.3-alpha.
+
+  o Minor bugfixes (code style):
+    - Fix an integer signedness conversion issue in the case conversion
+      tables. Fixes bug 19168; bugfix on 0.2.1.11-alpha.
+
+  o Minor bugfixes (compilation):
+    - Build correctly on versions of libevent2 without support for
+      evutil_secure_rng_add_bytes(). Fixes bug 19904; bugfix
+      on 0.2.5.4-alpha.
+    - When building with Clang, use a full set of GCC warnings.
+      (Previously, we included only a subset, because of the way we
+      detected them.) Fixes bug 19216; bugfix on 0.2.0.1-alpha.
+    - Detect Libevent2 functions correctly on systems that provide
+      libevent2, but where libevent1 is linked with -levent. Fixes bug
+      19904; bugfix on 0.2.2.24-alpha. Patch from Rubiate.
+    - Run correctly when built on Windows build environments that
+      require _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
+
+  o Minor bugfixes (configuration):
+    - When parsing quoted configuration values from the torrc file,
+      handle Windows line endings correctly. Fixes bug 19167; bugfix on
+      0.2.0.16-alpha. Patch from "Pingl".
+
+  o Minor bugfixes (directory authority):
+    - Authorities now sort the "package" lines in their votes, for ease
+      of debugging. (They are already sorted in consensus documents.)
+      Fixes bug 18840; bugfix on 0.2.6.3-alpha.
+    - Die with a more useful error when the operator forgets to place
+      the authority_signing_key file into the keys directory. This
+      avoids an uninformative assert & traceback about having an invalid
+      key. Fixes bug 20065; bugfix on 0.2.0.1-alpha.
+    - When allowing private addresses, mark Exits that only exit to
+      private locations as such. Fixes bug 20064; bugfix
+      on 0.2.2.9-alpha.
+    - When parsing a detached signature, make sure we use the length of
+      the digest algorithm instead of a hardcoded DIGEST256_LEN in
+      order to avoid comparing bytes out-of-bounds with a smaller digest
+      length such as SHA1. Fixes bug 19066; bugfix on 0.2.2.6-alpha.
+
+  o Minor bugfixes (getpass):
+    - Defensively fix a non-triggerable heap corruption at do_getpass()
+      to protect ourselves from mistakes in the future. Fixes bug
+      19223; bugfix on 0.2.7.3-rc. Bug found by Guido Vranken, patch
+      by nherring.
+
+  o Minor bugfixes (guard selection):
+    - Don't mark guards as unreachable if connection_connect() fails.
+      That function fails for local reasons, so it shouldn't reveal
+      anything about the status of the guard. Fixes bug 14334; bugfix
+      on 0.2.3.10-alpha.
+    - Use a single entry guard even if the NumEntryGuards consensus
+      parameter is not provided. Fixes bug 17688; bugfix
+      on 0.2.5.6-alpha.
+
+  o Minor bugfixes (hidden services):
+    - Increase the minimum number of internal circuits we preemptively
+      build from 2 to 3, so a circuit is available when a client
+      connects to another onion service. Fixes bug 13239; bugfix
+      on 0.1.0.1-rc.
+    - Allow hidden services to run on IPv6 addresses even when the
+      IPv6Exit option is not set. Fixes bug 18357; bugfix
+      on 0.2.4.7-alpha.
+    - Stop logging intro point details to the client log on certain
+      error conditions. Fixed as part of bug 20012; bugfix on
+      0.2.4.8-alpha. Patch by teor.
+    - When deleting an ephemeral hidden service, close its intro points
+      even if they are not completely open. Fixes bug 18604; bugfix
+      on 0.2.7.1-alpha.
+    - When configuring hidden services, check every hidden service
+      directory's permissions. Previously, we only checked the last
+      hidden service. Fixes bug 20529; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (IPv6, testing):
+    - Check for IPv6 correctly on Linux when running test networks.
+      Fixes bug 19905; bugfix on 0.2.7.3-rc; patch by teor.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Add permission to run the sched_yield() and sigaltstack() system
+      calls, in order to support versions of Tor compiled with asan or
+      ubsan code that use these calls. Now "sandbox 1" and
+      "--enable-expensive-hardening" should be compatible on more
+      systems. Fixes bug 20063; bugfix on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Downgrade a harmless log message about the
+      pending_entry_connections list from "warn" to "info". Mitigates
+      bug 19926.
+    - Log a more accurate message when we fail to dump a microdescriptor.
+      Fixes bug 17758; bugfix on 0.2.2.8-alpha. Patch from Daniel Pinto.
+    - When logging a directory ownership mismatch, log the owning
+      username correctly. Fixes bug 19578; bugfix on 0.2.2.29-beta.
+    - When we are unable to remove the bw_accounting file, do not warn
+      if the reason we couldn't remove it was that it didn't exist.
+      Fixes bug 19964; bugfix on 0.2.5.4-alpha. Patch from pastly.
+
+  o Minor bugfixes (memory leak):
+    - Fix a series of slow memory leaks related to parsing torrc files
+      and options. Fixes bug 19466; bugfix on 0.2.1.6-alpha.
+    - Avoid a small memory leak when informing worker threads about
+      rotated onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
+    - Fix a small memory leak when receiving AF_UNIX connections on a
+      SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
+    - When moving a signed descriptor object from a source to an
+      existing destination, free the allocated memory inside that
+      destination object. Fixes bug 20715; bugfix on 0.2.8.3-alpha.
+    - Fix a memory leak and use-after-free error when removing entries
+      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
+      0.2.5.5-alpha. Patch from "cypherpunks".
+    - Fix a small, uncommon memory leak that could occur when reading a
+      truncated ed25519 key file. Fixes bug 18956; bugfix
+      on 0.2.6.1-alpha.
+
+  o Minor bugfixes (option parsing):
+    - Count unix sockets when counting client listeners (SOCKS, Trans,
+      NATD, and DNS). This has no user-visible behavior changes: these
+      options are set once, and never read. Required for correct
+      behavior in ticket 17178. Fixes bug 19677; bugfix on
+      0.2.6.3-alpha. Patch by teor.
+
+  o Minor bugfixes (options):
+    - Check the consistency of UseEntryGuards and EntryNodes more
+      reliably. Fixes bug 20074; bugfix on 0.2.4.12-alpha. Patch
+      by teor.
+    - Stop changing the configured value of UseEntryGuards on
+      authorities and Tor2web clients. Fixes bug 20074; bugfix on
+      commits 51fc6799 in 0.1.1.16-rc and acda1735 in 0.2.4.3-alpha.
+      Patch by teor.
+
+  o Minor bugfixes (relay):
+    - Ensure relays don't make multiple connections during bootstrap.
+      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
+    - Do not try to parallelize workers more than 16x without the user
+      explicitly configuring us to do so, even if we do detect more than
+      16 CPU cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+
+  o Minor bugfixes (testing):
+    - The test-stem and test-network makefile targets now depend only on
+      the tor binary that they are testing. Previously, they depended on
+      "make all". Fixes bug 18240; bugfix on 0.2.8.2-alpha. Based on a
+      patch from "cypherpunks".
+    - Allow clients to retry HSDirs much faster in test networks. Fixes
+      bug 19702; bugfix on 0.2.7.1-alpha. Patch by teor.
+    - Avoid a unit test failure on systems with over 16 detectable CPU
+      cores. Fixes bug 19968; bugfix on 0.2.3.1-alpha.
+    - Let backtrace tests work correctly under AddressSanitizer:
+      disable ASAN's detection of segmentation faults while running
+      test_bt.sh, so that we can make sure that our own backtrace
+      generation code works. Fixes bug 18934; bugfix
+      on 0.2.5.2-alpha. Patch from "cypherpunks".
+    - Fix the test-network-all target on out-of-tree builds by using the
+      correct path to the test driver script. Fixes bug 19421; bugfix
+      on 0.2.7.3-rc.
+    - Stop spurious failures in the local interface address discovery
+      unit tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by
+      Neel Chauhan.
+    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
+      removed the ECDH ciphers which caused the tests to fail on
+      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
+    - The tor_tls_server_info_callback unit test no longer crashes when
+      debug-level logging is turned on. Fixes bug 20041; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (time):
+    - Improve overflow checks in tv_udiff and tv_mdiff. Fixes bug 19483;
+      bugfix on all released tor versions.
+    - When computing the difference between two times in milliseconds,
+      we now round to the nearest millisecond correctly. Previously, we
+      could sometimes round in the wrong direction. Fixes bug 19428;
+      bugfix on 0.2.2.2-alpha.
+
+  o Minor bugfixes (Tor2web):
+    - Prevent Tor2web clients from running hidden services: these services
+      are not anonymous due to the one-hop client paths. Fixes bug
+      19678. Patch by teor.
+
+  o Minor bugfixes (user interface):
+    - Display a more accurate number of suppressed messages in the log
+      rate-limiter. Previously, there was a potential integer overflow
+      in the counter. Now, if the number of messages hits a maximum, the
+      rate-limiter doesn't count any further. Fixes bug 19435; bugfix
+      on 0.2.4.11-alpha.
+    - Fix a typo in the passphrase prompt for the ed25519 identity key.
+      Fixes bug 19503; bugfix on 0.2.7.2-alpha.
+
+  o Code simplification and refactoring:
+    - Remove redundant declarations of the MIN macro. Closes
+      ticket 18889.
+    - Rename tor_dup_addr() to tor_addr_to_str_dup() to avoid confusion.
+      Closes ticket 18462; patch from "icanhasaccount".
+    - Split the 600-line directory_handle_command_get function into
+      separate functions for different URL types. Closes ticket 16698.
+
+  o Documentation:
+    - Add module-level internal documentation for 36 C files that
+      previously didn't have a high-level overview. Closes ticket 20385.
+    - Correct the IPv6 syntax in our documentation for the
+      VirtualAddrNetworkIPv6 torrc option. Closes ticket 19743.
+    - Correct the minimum bandwidth value in torrc.sample, and queue a
+      corresponding change for torrc.minimal. Closes ticket 20085.
+    - Fix spelling of "--enable-tor2web-mode" in the manpage. Closes
+      ticket 19153. Patch from "U+039b".
+    - Module-level documentation for several more modules. Closes
+      tickets 19287 and 19290.
+    - Document the --passphrase-fd option in the tor manpage. Fixes bug
+      19504; bugfix on 0.2.7.3-rc.
+    - Document the default PathsNeededToBuildCircuits value that's used
+      by clients when the directory authorities don't set
+      min_paths_for_circs_pct. Fixes bug 20117; bugfix on 0.2.4.10-alpha.
+      Patch by teor, reported by Jesse V.
+    - Fix manual for the User option: it takes a username, not a UID.
+      Fixes bug 19122; bugfix on 0.0.2pre16 (the first version to have
+      a manpage!).
+    - Fix the description of the --passphrase-fd option in the
+      tor-gencert manpage. The option is used to pass the number of a
+      file descriptor to read the passphrase from, not to read the file
+      descriptor from. Fixes bug 19505; bugfix on 0.2.0.20-alpha.
+
+  o Removed code:
+    - We no longer include the (dead, deprecated) bufferevent code in
+      Tor. Closes ticket 19450. Based on a patch from "U+039b".
+
+  o Removed features:
+    - Remove support for "GET /tor/bytes.txt" DirPort request, and
+      "GETINFO dir-usage" controller request, which were only available
+      via a compile-time option in Tor anyway. Feature was added in
+      0.2.2.1-alpha. Resolves ticket 19035.
+    - There is no longer a compile-time option to disable support for
+      TransPort. (If you don't want TransPort, just don't use it.) Patch
+      from "U+039b". Closes ticket 19449.
+
+  o Testing:
+    - Run more workqueue tests as part of "make check". These had
+      previously been implemented, but you needed to know special
+      command-line options to enable them.
+    - We now have unit tests for our code to reject zlib "compression
+      bombs". (Fortunately, the code works fine.)
+
+
+Changes in version 0.2.8.11 - 2016-12-08
+  Tor 0.2.8.11 backports fixes for additional portability issues that
+  could prevent Tor from building correctly on OSX Sierra, or with
+  OpenSSL 1.1. Affected users should upgrade; others can safely stay
+  with 0.2.8.10.
+
+  o Minor bugfixes (portability):
+    - Avoid compilation errors when building on OSX Sierra. Sierra began
+      to support the getentropy() and clock_gettime() APIs, but created
+      a few problems in doing so. Tor 0.2.9 has a more thorough set of
+      workarounds; in 0.2.8, we are just using the /dev/urandom and mach
+      monotonic time interfaces. Fixes bug 20865. Bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix compilation with OpenSSL 1.1 and less commonly-used CPU
+      architectures. Closes ticket 20588.
+
+
+Changes in version 0.2.8.10 - 2016-12-02
+  Tor 0.2.8.10 backports a fix for a bug that would sometimes make clients
+  unusable after they left standby mode. It also backports fixes for
+  a few portability issues and a small but problematic memory leak.
+
+  o Major bugfixes (client reliability, backport from 0.2.9.5-alpha):
+    - When Tor leaves standby because of a new application request, open
+      circuits as needed to serve that request. Previously, we would
+      potentially wait a very long time. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Major bugfixes (client performance, backport from 0.2.9.5-alpha):
+    - Clients now respond to new application stream requests immediately
+      when they arrive, rather than waiting up to one second before
+      starting to handle them. Fixes part of bug 19969; bugfix
+      on 0.2.8.1-alpha.
+
+  o Minor bugfixes (portability, backport from 0.2.9.6-rc):
+    - Work around a bug in the OSX 10.12 SDK that would prevent us from
+      successfully targeting earlier versions of OSX. Resolves
+      ticket 20235.
+
+  o Minor bugfixes (portability, backport from 0.2.9.5-alpha):
+    - Fix implicit conversion warnings under OpenSSL 1.1. Fixes bug
+      20551; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (relay, backport from 0.2.9.5-alpha):
+    - Work around a memory leak in OpenSSL 1.1 when encoding public
+      keys. Fixes bug 20553; bugfix on 0.0.2pre8.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
+      Country database.
+
+
 Changes in version 0.2.8.9 - 2016-10-17
   Tor 0.2.8.9 backports a fix for a security hole in previous versions
   of Tor that would allow a remote attacker to crash a Tor client,
@@ -449,7 +2643,7 @@ Changes in version 0.2.8.6 - 2016-08-02
 
   o Minor bugfixes (build):
     - Avoid spurious failures from configure files related to calling
-      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18625; bugfix on
+      exit(0) in TOR_SEARCH_LIBRARY. Fixes bug 18626; bugfix on
       0.2.0.1-alpha. Patch from "cypherpunks".
     - Do not link the unit tests against both the testing and non-
       testing versions of the static libraries. Fixes bug 18490; bugfix
diff --git a/acinclude.m4 b/acinclude.m4
index ab12317139..49d4f14471 100644
--- a/acinclude.m4
+++ b/acinclude.m4
@@ -2,7 +2,7 @@ dnl Helper macros for Tor configure.ac
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
 dnl Copyright (c) 2007-2008, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2015, The Tor Project, Inc.
+dnl Copyright (c) 2007-2017, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_DEFUN([TOR_EXTEND_CODEPATH],
@@ -51,12 +51,12 @@ AC_DEFUN([TOR_TRY_COMPILE_WITH_CFLAGS], [
   AC_CACHE_CHECK([whether the compiler accepts $1], VAR, [
     tor_saved_CFLAGS="$CFLAGS"
     CFLAGS="$CFLAGS -pedantic -Werror $1"
-    AC_TRY_COMPILE([], [return 0;],
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                    [AS_VAR_SET(VAR,yes)],
                    [AS_VAR_SET(VAR,no)])
     if test x$2 != x; then
       AS_VAR_PUSHDEF([can_link],[tor_can_link_$1])
-      AC_TRY_LINK([], [return 0;],
+      AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                   [AS_VAR_SET(can_link,yes)],
                   [AS_VAR_SET(can_link,no)])
       AS_VAR_POPDEF([can_link])
@@ -93,7 +93,7 @@ AC_DEFUN([TOR_CHECK_LDFLAGS], [
     AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdio.h>], [fputs("", stdout)])],
                   [AS_VAR_SET(VAR,yes)],
                   [AS_VAR_SET(VAR,no)],
-	          [AC_TRY_LINK([], [return 0;],
+                  [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
                                    [AS_VAR_SET(VAR,yes)],
                                    [AS_VAR_SET(VAR,no)])])
     CFLAGS="$tor_saved_CFLAGS"
@@ -113,21 +113,21 @@ if test x$2 = xdevpkg; then
   h=" headers for"
 fi
 if test -f /etc/debian_version && test x"$tor_$1_$2_debian" != x; then
-  AC_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
+  AC_MSG_WARN([On Debian, you can install$h $1 using "apt-get install $tor_$1_$2_debian"])
   if test x"$tor_$1_$2_debian" != x"$tor_$1_devpkg_debian"; then 
-    AC_WARN([   You will probably need $tor_$1_devpkg_debian too.])
+    AC_MSG_WARN([   You will probably need $tor_$1_devpkg_debian too.])
   fi 
 fi
 if test -f /etc/fedora-release && test x"$tor_$1_$2_redhat" != x; then
-  AC_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
+  AC_MSG_WARN([On Fedora, you can install$h $1 using "dnf install $tor_$1_$2_redhat"])
   if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-    AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+    AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
   fi 
 else
   if test -f /etc/redhat-release && test x"$tor_$1_$2_redhat" != x; then
-    AC_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
+    AC_MSG_WARN([On most Redhat-based systems, you can get$h $1 by installing the $tor_$1_$2_redhat RPM package])
     if test x"$tor_$1_$2_redhat" != x"$tor_$1_devpkg_redhat"; then 
-      AC_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
+      AC_MSG_WARN([   You will probably need to install $tor_$1_devpkg_redhat too.])
     fi 
   fi
 fi
@@ -245,7 +245,10 @@ if test "$cross_compiling" != yes; then
        LDFLAGS="$tor_tryextra $orig_LDFLAGS"
      fi
      AC_RUN_IFELSE([AC_LANG_PROGRAM([$5], [$6])],
-                   [runnable=yes], [runnable=no])
+                   [runnable=yes], [runnable=no],
+                   [AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
+                                   [runnable=yes],
+                                   [runnable=no])])
      if test "$runnable" = yes; then
         tor_cv_library_$1_linker_option=$tor_tryextra
         break
diff --git a/autogen.sh b/autogen.sh
index 8c43a9798a..276dd4047c 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,12 +1,12 @@
 #!/bin/sh
 
 if [ -x "`which autoreconf 2>/dev/null`" ] ; then
-  opt="-if"
+  opt="-i -f -W all,error"
 
   for i in $@; do
     case "$i" in
       -v)
-        opt=$opt"v"
+        opt="${opt} -v"
         ;;
     esac
   done
diff --git a/changes/19974 b/changes/19974
deleted file mode 100644
index 5496143ddf..0000000000
--- a/changes/19974
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (unit tests):
-    - Fix tolerances in unit tests for monotonic time comparisons between
-      nanoseconds and microseconds. Previously, we accepted a 10 us
-      difference only, which is not realistic on every platform's
-      clock_gettime(). Fixes bug 19974; bugfix on 0.2.9.1-alpha.
diff --git a/changes/20460 b/changes/20460
deleted file mode 100644
index 9fbb4a7986..0000000000
--- a/changes/20460
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Use ECDHE ciphers instead of ECDH in tortls tests. LibreSSL has
-      removed the ECDH ciphers which caused the tests to fail on
-      platforms which use it. Fixes bug 20460; bugfix on 0.2.8.1-alpha.
diff --git a/changes/20492 b/changes/20492
deleted file mode 100644
index fdcd4d0b4b..0000000000
--- a/changes/20492
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfix (build):
-    - The current Git revision when building from a local repository is now
-      detected correctly when using git worktrees.  Fixes bug 20492; bugfix on
-      0.2.3.9-alpha.
diff --git a/changes/21359 b/changes/21359
deleted file mode 100644
index cc9b377d52..0000000000
--- a/changes/21359
+++ /dev/null
@@ -1,8 +0,0 @@
-
-  o Minor features (portability, compilationc)
-    - Support building with recent LibreSSL code that uses opaque
-      structures. Closes ticket 21359.
-    - Autoconf now check to determine if OpenSSL
-      structures are opaque, instead of explicitly checking for
-      OpenSSL version numbers. 
-      Part of ticket 21359.
diff --git a/changes/bug16082 b/changes/bug16082
new file mode 100644
index 0000000000..0f2f04fb35
--- /dev/null
+++ b/changes/bug16082
@@ -0,0 +1,4 @@
+  o Documentation:
+    - Correctly note that bandwidth accounting values are stored in the
+      state file, and the bw_accounting file is now obsolete. Closes
+      ticket 16082.
diff --git a/changes/bug17857 b/changes/bug17857
new file mode 100644
index 0000000000..6c88638231
--- /dev/null
+++ b/changes/bug17857
@@ -0,0 +1,6 @@
+  o Minor features (defensive programming):
+    - Create a pair of consensus parameters nf_pad_tor2web and
+      nf_pad_single_onion that allow us to disable netflow padding in the
+      consensus for non-anonymous connections, in case the overhead is high.
+      Closes #17857.
+
diff --git a/changes/bug18100 b/changes/bug18100
deleted file mode 100644
index cd3ba2c977..0000000000
--- a/changes/bug18100
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (linux TPROXY support):
-    - Fix a typo that had prevented TPROXY-based transparent proxying from
-      working under Linux. Fixes bug 18100; bugfix on 0.2.6.3-alpha.
-      Patch from "d4fq0fQAgoJ".
-
diff --git a/changes/bug19025 b/changes/bug19025
deleted file mode 100644
index 0f365f52ba..0000000000
--- a/changes/bug19025
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Major bugfixes (DNS):
-    - Fix a bug that prevented exit nodes from caching DNS records for more
-      than 60 seconds.
-      Fixes bug 19025; bugfix on 0.2.4.7-alpha.
diff --git a/changes/bug19418 b/changes/bug19418
new file mode 100644
index 0000000000..fb5f6ad5df
--- /dev/null
+++ b/changes/bug19418
@@ -0,0 +1,7 @@
+  o Minor bugfixes (robustness, error handling):
+    - Improve our handling of the cases where OpenSSL encounters a
+      memory error while encoding keys and certificates. We haven't
+      observed these happening in the wild, but if they do happen,
+      we now detect and respond better. Fixes bug 19418; bugfix
+      on all versions of Tor. Reported by Guido Vranken.
+
diff --git a/changes/bug19869 b/changes/bug19869
deleted file mode 100644
index 430048f161..0000000000
--- a/changes/bug19869
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (DNSPort):
-    - On DNSPort, stop logging a BUG warning on a failed hostname lookup.
-      Fixes bug 19869; bugfix on 0.2.9.1-alpha.
-
diff --git a/changes/bug19926_029_info b/changes/bug19926_029_info
deleted file mode 100644
index 93fd81b6cb..0000000000
--- a/changes/bug19926_029_info
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (logging):
-    - Downgrade a harmless log message about the pending_entry_connections
-      list from "warn" to "info". Mitigates bug 19926.
diff --git a/changes/bug19960 b/changes/bug19960
deleted file mode 100644
index 5d655859a6..0000000000
--- a/changes/bug19960
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (netbsd, unit tests):
-    - Stop expecting NetBSD unit tests to report success for ipfw;
-      on NetBSD, it's only pf that's supported.
-      Part of a fix for bug 19960; bugfix on 0.2.9.5-alpha.
diff --git a/changes/bug19968 b/changes/bug19968
deleted file mode 100644
index b285706e70..0000000000
--- a/changes/bug19968
+++ /dev/null
@@ -1,11 +0,0 @@
-  o Minor bugfixes (relay):
-    - Do not try to parallelize workers more than 16x without the
-      user explicitly configuring us to do so, even if we do detect more than
-      16 CPU cores. Fixes bug 19968; bugfix on
-      0.2.3.1-alpha.
-
-
-  o Minor bugfixes (testing):
-    - Avoid a unit test failure on systems with over 16 detectable
-      CPU cores. Fixes bug 19968; bugfix on
-      0.2.3.1-alpha.
diff --git a/changes/bug19969 b/changes/bug19969
deleted file mode 100644
index c760c6de03..0000000000
--- a/changes/bug19969
+++ /dev/null
@@ -1,10 +0,0 @@
-  o Major bugfixes (client performance):
-    - Clients now respond to new application stream requests when
-      they arrive, rather than waiting up to one second before starting
-      to handle them. Fixes part of bug 19969; bugfix on 0.2.8.1-alpha.
-
-  o Major bugfixes (clients on flaky network connections):
-    - When Tor leaves standby because of a new application request, open
-      circuits as needed to serve that request. Previously, we would
-      potentially wait a very long time. Fixes part of bug 19969; bugfix
-      on 0.2.8.1-alpha.
diff --git a/changes/bug20059 b/changes/bug20059
deleted file mode 100644
index 091fab06d1..0000000000
--- a/changes/bug20059
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (relay):
-    - Avoid a double-marked-circuit warning that can happen when we receive
-      DESTROY cells under heavy load. Fixes bug 20059; bugfix on 0.1.0.1-rc.
diff --git a/changes/bug20085 b/changes/bug20085
deleted file mode 100644
index fd10e7eeeb..0000000000
--- a/changes/bug20085
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation:
-    - Correct the minimum bandwidth value in torrc.sample, and queue a
-      corresponding change for torrc.minimal. Closes ticket 20085.
-
diff --git a/changes/bug20235 b/changes/bug20235
deleted file mode 100644
index 54026a8943..0000000000
--- a/changes/bug20235
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (compatibility):
-    - Work around a bug in the OSX 10.12 SDK that would prevent us
-      from successfully targetting earlier versions of OSX.
-      Resolves ticket 20235.
diff --git a/changes/bug20306_029 b/changes/bug20306_029
deleted file mode 100644
index ada2676b2b..0000000000
--- a/changes/bug20306_029
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (fascistfirewall):
-    - Avoid spurious warnings when ReachableAddresses or FascistFirewall
-      is set. Fixes bug 20306; bugfix on 0.2.8.2-alpha.
-
diff --git a/changes/bug20307 b/changes/bug20307
deleted file mode 100644
index 9112c9c78d..0000000000
--- a/changes/bug20307
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (circuit, hidden service)
-    - When closing a circuit, the reason for doing so was assigned from an int
-      value to a uint16_t which is quite a problem for negative values that are
-      our internal reasons (ex: END_CIRC_REASON_IP_NOW_REDUNDANT). On the HS
-      side, this was causing introduction points to be flagged as unusable
-      because the reason wasn't the right one due to the bad conversion.
-      Partially fixes bug 21056 and fixes bug 20307; Bugfix on 0.2.8.1-alpha.
diff --git a/changes/bug20401 b/changes/bug20401
deleted file mode 100644
index 85ab3c7322..0000000000
--- a/changes/bug20401
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (relay):
-    - Avoid a small memory leak when informing worker threads about rotated
-      onion keys. Fixes bug 20401; bugfix on 0.2.6.3-alpha.
-
diff --git a/changes/bug20423 b/changes/bug20423
deleted file mode 100644
index 32bdc3f081..0000000000
--- a/changes/bug20423
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes:
-    - For relays that don't know their own address, avoid attempting
-      a local hostname resolve for each descriptor we download. Also cut
-      down on the number of "Success: chose address 'x.x.x.x'" log lines.
-      Fixes bugs 20423 and 20610; bugfix on 0.2.8.1-alpha.
-
diff --git a/changes/bug20472 b/changes/bug20472
deleted file mode 100644
index 4d90c39f5b..0000000000
--- a/changes/bug20472
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (circuits):
-    - Remove a BUG warning in circuit_pick_extend_handshake. Instead, assume
-      all nodes support EXTEND2. Use ntor whenever a key is available.
-      Fixes bug 20472; bugfix on 0.2.9.3-alpha.
-
diff --git a/changes/bug20484 b/changes/bug20484
deleted file mode 100644
index 9a0b95cb39..0000000000
--- a/changes/bug20484
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (single onion services):
-    - Start correctly when creating a single onion service in a
-      directory that did not previously exist. Fixes bug 20484; bugfix on
-      0.2.9.3-alpha.
-
diff --git a/changes/bug20487 b/changes/bug20487
deleted file mode 100644
index 4435f14a95..0000000000
--- a/changes/bug20487
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation:
-    - Clarify that setting HiddenServiceNonAnonymousMode requires
-      you to also set "SOCKSPort 0". Fixes bug 20487; bugfix on
-      0.2.9.3-alpha.
diff --git a/changes/bug20509 b/changes/bug20509
deleted file mode 100644
index a39ca9f60b..0000000000
--- a/changes/bug20509
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features:
-    - Directory authorities now reject relays running versions
-      0.2.9.1-alpha through 0.2.9.4-alpha, because those relays
-      suffer from bug 20499 and don't keep their consensus cache
-      up-to-date. Resolves ticket 20509.
diff --git a/changes/bug20529 b/changes/bug20529
deleted file mode 100644
index 276be5b2b6..0000000000
--- a/changes/bug20529
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden services):
-    - When configuring hidden services, check every hidden service directory's
-      permissions. Previously, we only checked the last hidden service.
-      Fixes bug 20529; bugfix on 13942 commit 85bfad1 in 0.2.6.2-alpha.
diff --git a/changes/bug20533 b/changes/bug20533
deleted file mode 100644
index 7d1a456328..0000000000
--- a/changes/bug20533
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (consensus downloads):
-    - If a consensus expires while we are waiting for certificates to download,
-      stop waiting for certificates.
-    - If we stop waiting for certificates less than a minute after we started
-      downloading them, do not consider the certificate download failure a
-      separate failure.
-      Fixes bug 20533; bugfix on commit e0204f21 in 0.2.0.9-alpha.
diff --git a/changes/bug20534 b/changes/bug20534
deleted file mode 100644
index 49db433a01..0000000000
--- a/changes/bug20534
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Minor bugfixes (directory download scheduling):
-    - Remove the maximum delay on exponential-backoff scheduling.
-      Since we now allow an infinite number of failures (see ticket
-      20536), we must now allow the time to grow longer on each failure.
-      Fixes part of bug 20534; bugfix on 0.2.9.1-alpha.
-    - Use initial delays and decrements in download scheduling closer to
-      those from 0.2.8. Fixes another part of bug 20534; bugfix on
-      0.2.9.1-alpha.
diff --git a/changes/bug20536 b/changes/bug20536
deleted file mode 100644
index 9e0dd164bb..0000000000
--- a/changes/bug20536
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (download scheduling):
-    - When using an exponential backoff schedule, do not give up on
-      dowloading just because we have failed a bunch of times.  Since
-      each delay is longer than the last, retrying indefinitely won't
-      hurt. Fixes bug 20536; bugfix on 0.2.9.1-alpha.
-
diff --git a/changes/bug20551 b/changes/bug20551
deleted file mode 100644
index b7ec4ca7cc..0000000000
--- a/changes/bug20551
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix implicit conversion warnings under OpenSSL 1.1.
-      Fixes bug 20551; bugfix on 0.2.1.1-alpha.
diff --git a/changes/bug20553 b/changes/bug20553
deleted file mode 100644
index 12a2780303..0000000000
--- a/changes/bug20553
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (memory leak):
-    - Work around a memory leak in OpenSSL 1.1 when encoding public keys.
-      Fixes bug 20553; bugfix on 0.0.2pre8.
diff --git a/changes/bug20560 b/changes/bug20560
deleted file mode 100644
index 43d605b296..0000000000
--- a/changes/bug20560
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (portability):
-    - Run correctly when built on Windows build environments that require
-      _vcsprintf(). Fixes bug 20560; bugfix on 0.2.2.11-alpha.
-
diff --git a/changes/bug20587 b/changes/bug20587
deleted file mode 100644
index 341b001363..0000000000
--- a/changes/bug20587
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (download timing):
-    - When determining when to download a directory object, handle times
-      after 2038 if the operating system supports that.  (Someday this will be
-      important!)  Fixes bug 20587; bugfix on 0.2.8.1-alpha.
-
diff --git a/changes/bug20588 b/changes/bug20588
deleted file mode 100644
index 832ef81336..0000000000
--- a/changes/bug20588
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (portability):
-    - Fix compilation with OpenSSL 1.1 and less commonly-used
-      CPU architectures. Closes ticket 20588.
diff --git a/changes/bug20591 b/changes/bug20591
deleted file mode 100644
index deaa738f5e..0000000000
--- a/changes/bug20591
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (relay bootstrap):
-    - Ensure relays don't make multiple connections during bootstrap.
-      Fixes bug 20591; bugfix on 0.2.8.1-alpha.
diff --git a/changes/bug20593 b/changes/bug20593
deleted file mode 100644
index e9f54d317a..0000000000
--- a/changes/bug20593
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (client directory scheduling):
-    - Treat "relay too busy to answer request" as a failed request and a
-      reason to back off on our retry frequency. This is safe now that
-      exponential backups retry indefinitely, and avoids a bug where we would
-      reset our download schedule erroneously.
-      Fixes bug 20593; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug20597 b/changes/bug20597
deleted file mode 100644
index f199b63933..0000000000
--- a/changes/bug20597
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (test networks, exponential backoff):
-    - When using exponential backoff in test networks, use a lower exponent,
-      so the delays do not vary as much. This helps test networks bootstrap
-      consistently. Fixes bug 20597; bugfix on 20499; not in any released
-      version of tor.
diff --git a/changes/bug20613 b/changes/bug20613
deleted file mode 100644
index 19bb61f4e0..0000000000
--- a/changes/bug20613
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (single onion services, Tor2web):
-    - Stop logging long-term one-hop circuits deliberately created by single
-      onion services and Tor2web. These log messages are intended to diagnose
-      issue 8387, which relates to circuits hanging around forever for no
-      reason.
-      Fixes bug 20613; bugfix on 0.2.9.1-alpha. Reported by "pastly".
diff --git a/changes/bug20634 b/changes/bug20634
deleted file mode 100644
index 62fc9f4787..0000000000
--- a/changes/bug20634
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (unit tests):
-    - Stop spurious failures in the local interface address discovery unit
-      tests. Fixes bug 20634; bugfix on 0.2.8.1-alpha; patch by Neel Chauhan.
diff --git a/changes/bug20638 b/changes/bug20638
deleted file mode 100644
index 260d7d0a75..0000000000
--- a/changes/bug20638
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hidden services):
-    - Stop ignoring hidden service key anonymity when first starting tor.
-      Instead, refuse to start tor if any hidden service key has been used in
-      a different hidden service anonymity mode.
-      Fixes bug 20638; bugfix on 17178 in 0.2.9.3-alpha; reported by ahf.
diff --git a/changes/bug20710_025 b/changes/bug20710_025
deleted file mode 100644
index 12bd07536c..0000000000
--- a/changes/bug20710_025
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory leak, use-after-free, linux seccomp2 sandbox):
-    - Fix a memory leak and use-after-free error when removing entries
-      from the sandbox's getaddrinfo() cache. Fixes bug 20710; bugfix on
-      0.2.5.5-alpha. Patch from "cypherpunks".
diff --git a/changes/bug20715 b/changes/bug20715
deleted file mode 100644
index 737a560cec..0000000000
--- a/changes/bug20715
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory leak)
-    - When moving a signed descriptor object from a source to an existing
-      destination, free the allocated memory inside that destination object.
-      Bugfix on tor-0.2.8.3-alpha; Closes #20715.
diff --git a/changes/bug20716 b/changes/bug20716
deleted file mode 100644
index 37fd6feecf..0000000000
--- a/changes/bug20716
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (client, memory leak):
-    - Fix a small memory leak when receiving AF_UNIX connections on
-      a SocksPort. Fixes bug 20716; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug20810 b/changes/bug20810
deleted file mode 100644
index 5420a73175..0000000000
--- a/changes/bug20810
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (relay)
-    - When computing old Tor protocol line version in protover, we were
-      looking at 0.2.7.5 twice instead of a specific case for 0.2.9.1-alpha.
-      Bugfix on tor-0.2.9.4-alpha.
diff --git a/changes/bug20864 b/changes/bug20864
deleted file mode 100644
index 7b8c70fad6..0000000000
--- a/changes/bug20864
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (unit tests, hidden services):
-    - Remove a double-free in the single onion service unit test. Stop
-      ignoring a return value. Make future changes less error-prone.
-      Fixes bug 20864; bugfix on 0.2.9.6-rc.
diff --git a/changes/bug20875 b/changes/bug20875
deleted file mode 100644
index 6bba2cbc12..0000000000
--- a/changes/bug20875
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (download scheduling)
-    - Resolve a "bug" warning when considering a download schedule whose
-      delay had approached INT_MAX. Fixes 20875; bugfix on 0.2.9.5-alpha.
-
diff --git a/changes/bug20935 b/changes/bug20935
deleted file mode 100644
index 78068c7c06..0000000000
--- a/changes/bug20935
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (portability):
-    - Use the correct spelling of MAC_OS_X_VERSION_10_12 on configure.ac
-      Fixes bug 20935; bugfix on 0.2.9.6-rc.
diff --git a/changes/bug21018 b/changes/bug21018
deleted file mode 100644
index 49a8b47a25..0000000000
--- a/changes/bug21018
+++ /dev/null
@@ -1,11 +0,0 @@
-  o Major bugfixes (parsing, security):
-
-    - Fix a bug in parsing that could cause clients to read a single
-      byte past the end of an allocated region. This bug could be
-      used to cause hardened clients (built with
-      --enable-expensive-hardening) to crash if they tried to visit
-      a hostile hidden service.  Non-hardened clients are only
-      affected depending on the details of their platform's memory
-      allocator. Fixes bug 21018; bugfix on 0.2.0.8-alpha. Found by
-      using libFuzzer. Also tracked as TROVE-2016-12-002 and as
-      CVE-2016-1254.
diff --git a/changes/bug21035 b/changes/bug21035
deleted file mode 100644
index bbf3340787..0000000000
--- a/changes/bug21035
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (portability):
-    - Avoid crashing when Tor is built using headers that contain
-      CLOCK_MONOTONIC_COARSE, but then tries to run on an older kernel
-      without CLOCK_MONOTONIC_COARSE. Fixes bug 21035; bugfix on
-      0.2.9.1-alpha.
-
diff --git a/changes/bug21051 b/changes/bug21051
deleted file mode 100644
index 8bb4f80c8e..0000000000
--- a/changes/bug21051
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix Libevent detection on platforms without Libevent 1 headers
-      installed. Fixes bug 21051; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug21108_029 b/changes/bug21108_029
deleted file mode 100644
index 3a3f004fc6..0000000000
--- a/changes/bug21108_029
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (directory authority):
-    - During voting, when marking a node as a probable sybil, do not
-      clear its BadExit flag: sybils can still be bad in other ways
-      too. (We still clear the other flags.) Fixes bug 21108; bugfix
-      on 0.2.0.13-alpha.
-
diff --git a/changes/bug21278_extras b/changes/bug21278_extras
deleted file mode 100644
index ffdf4a047b..0000000000
--- a/changes/bug21278_extras
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (code correctness):
-    - Repair a couple of (unreachable or harmless) cases of the risky
-      comparison-by-subtraction pattern that caused bug 21278.
diff --git a/changes/bug21278_prevention b/changes/bug21278_prevention
deleted file mode 100644
index e07f0a670c..0000000000
--- a/changes/bug21278_prevention
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (directory authority):
-    - Directory authorities now reject descriptors that claim to be
-      malformed versions of Tor. Helps prevent exploitation of bug 21278.
-      
diff --git a/changes/bug21280 b/changes/bug21280
deleted file mode 100644
index e9f0bc174c..0000000000
--- a/changes/bug21280
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (tor-resolve):
-    - The tor-resolve command line tool now rejects hostnames over 255
-      characters in length. Previously, it would silently truncate
-      them, which could lead to bugs. Fixes bug 21280; bugfix on 0.0.9pre5.
-      Patch by "junglefowl".
diff --git a/changes/bug21357 b/changes/bug21357
deleted file mode 100644
index a1cb43a78a..0000000000
--- a/changes/bug21357
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Major bugfixes (IPv6 Exits):
-    - Stop rejecting all IPv6 traffic on Exits whose exit policy rejects IPv6
-      addresses. Instead, only reject a port over IPv6 if the exit policy
-      rejects that port on more than an IPv6 /16 of addresses. This bug was
-      made worse by 17027 in 0.2.8.1-alpha, which rejects a relay's own IPv6
-      address by default.
-      Fixes bug 21357; bugfix on commit 004f3f4e53 in 0.2.4.7-alpha.
diff --git a/changes/bug21450 b/changes/bug21450
deleted file mode 100644
index a1cf89ab41..0000000000
--- a/changes/bug21450
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (voting consistency):
-    - Reject version numbers with components that exceed INT32_MAX.
-      Otherwise 32-bit and 64-bit platforms would behave inconsistently.
-      Fixes bug 21450; bugfix on 0.0.8pre1.
diff --git a/changes/bug21507 b/changes/bug21507
deleted file mode 100644
index f83e291b63..0000000000
--- a/changes/bug21507
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (voting consistency):
-    - Reject version numbers with non-numeric prefixes (such as +, -, and
-      whitespace). Disallowing whitespace prevents differential version
-      parsing between POSIX-based and Windows platforms.
-      Fixes bug 21507 and part of 21508; bugfix on 0.0.8pre1.
diff --git a/changes/bug21576 b/changes/bug21576
deleted file mode 100644
index 68d8471192..0000000000
--- a/changes/bug21576
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Major bugfixes (crash, directory connections):
-    - Fix a rare crash when sending a begin cell on a circuit whose linked
-      directory connection has already been closed. Fixes bug 21576;
-      bugfix on Tor 0.2.9.3-alpha. Reported by alecmuffett.
diff --git a/changes/bug21943 b/changes/bug21943
deleted file mode 100644
index dbe2c726d9..0000000000
--- a/changes/bug21943
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (Linux seccomp2 sandbox):
-    - The getpid() system call is now permitted under the Linux seccomp2
-      sandbox, to avoid crashing with versions of OpenSSL (and other
-      libraries) that attempt to learn the process's PID by using the
-      syscall rather than the VDSO code. Fixes bug 21943; bugfix on
-      0.2.5.1-alpha.
diff --git a/changes/bug22034 b/changes/bug22034
deleted file mode 100644
index 6d9e188740..0000000000
--- a/changes/bug22034
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (control port, regression):
-    - The GETINFO extra-info/digest/<digest> command was broken because of a
-      wrong base16 decode return value check. In was introduced in a refactor
-      of that API. Fixex bug #22034; bugfix on tor-0.2.9.1-alpha.
diff --git a/changes/bug22159 b/changes/bug22159
new file mode 100644
index 0000000000..c319c7e322
--- /dev/null
+++ b/changes/bug22159
@@ -0,0 +1,7 @@
+  o Minor bugfixes (hidden service):
+    - A service is allowed to open a maximum number of circuits for a specific
+      period of time. That value was lower than it should be (8 vs 12) in the
+      normal case of 3 introduction points. Fixes bug 22159.; bugfix on
+      tor-0.3.0.5-rc.
+    - Rate limit the log if we ever go above the maximum number of allowed
+      intro circuits. Fixes bug 22159.; bugfix on tor-0.3.1.1-alpha.
diff --git a/changes/bug22212 b/changes/bug22212
new file mode 100644
index 0000000000..f92d6701d3
--- /dev/null
+++ b/changes/bug22212
@@ -0,0 +1,5 @@
+  o Minor bugfixes (netflow padding logging):
+    - Demote a warn that was caused by libevent delays to info if
+      the padding is less than 4.5 seconds late, or notice if it is more
+      (4.5 seconds is the amount of time that a netflow record might
+       be emitted after, if we chose the maximum timeout). Fixes bug #22212.
diff --git a/changes/bug22245 b/changes/bug22245
deleted file mode 100644
index 6ae18593ea..0000000000
--- a/changes/bug22245
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (bandwidth accounting):
-    - Roll over monthly accounting at the configured hour and minute,
-      rather than always at 00:00.
-      Fixes bug 22245; bugfix on 0.0.9rc1.
-      Found by Andrey Karpov with PVS-Studio.
diff --git a/changes/bug22286 b/changes/bug22286
new file mode 100644
index 0000000000..f72e8fe2c7
--- /dev/null
+++ b/changes/bug22286
@@ -0,0 +1,3 @@
+  o Minor features (tests):
+    - Add a couple more tests for compression backend initialization.
+      Closes ticket 22286.
diff --git a/changes/bug22347 b/changes/bug22347
new file mode 100644
index 0000000000..f294ba0a2d
--- /dev/null
+++ b/changes/bug22347
@@ -0,0 +1,2 @@
+  o Documentation:
+    - Add a manpage description for the key-pinning-journal file.
diff --git a/changes/bug22356 b/changes/bug22356
new file mode 100644
index 0000000000..0082b542be
--- /dev/null
+++ b/changes/bug22356
@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging, relay):
+    - Downgrade "assigned_to_cpuworker failed" message to INFO-level
+      severity. In every case that can reach it, either a better warning
+      has already been logged, or no warning is warranted. Fixes bug 22356;
+      bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug22370 b/changes/bug22370
deleted file mode 100644
index e0e87e3339..0000000000
--- a/changes/bug22370
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory handling):
-    - When directory authorities reject a router descriptor due to keypinning,
-      free the router descriptor rather than leaking the memory.
-      Fixes bug 22370; bugfix on 0.2.7.2-alpha.
diff --git a/changes/bug22400_01 b/changes/bug22400_01
new file mode 100644
index 0000000000..454c5f746f
--- /dev/null
+++ b/changes/bug22400_01
@@ -0,0 +1,4 @@
+  o Major bugfixes (entry guards):
+    - When starting with an old consensus, do not add new entry guards
+      unless the consensus is "reasonably live" (under 1 day old). Fixes
+      one root cause of bug 22400; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug22460_case2 b/changes/bug22460_case2
deleted file mode 100644
index 0a11759832..0000000000
--- a/changes/bug22460_case2
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Major bugfixes (relay, link handshake):
-
-    - When performing the v3 link handshake on a TLS connection, report that
-      we have the x509 certificate that we actually used on that connection,
-      even if we have changed certificates since that connection was first
-      opened. Previously, we would claim to have used our most recent x509
-      link certificate, which would sometimes make the link handshake fail.
-      Fixes one case of bug 22460; bugfix on 0.2.3.6-alpha.
diff --git a/changes/bug22490 b/changes/bug22490
deleted file mode 100644
index 244dd50b36..0000000000
--- a/changes/bug22490
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (correctness):
-    - Avoid undefined behavior when parsing IPv6 entries from the geoip6
-      file. Fixes bug 22490; bugfix on 0.2.4.6-alpha.
diff --git a/changes/bug22502_part1 b/changes/bug22502_part1
new file mode 100644
index 0000000000..bd95b7c7c4
--- /dev/null
+++ b/changes/bug22502_part1
@@ -0,0 +1,12 @@
+  o Major bugfixes (compression, zstd):
+    - Correctly detect a full buffer when decompessing a large
+      zstd-compressed input. Fixes bug 22628; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (compression):
+    - When compressing or decompressing a buffer, check for a failure to
+      create a compression object. Fixes bug 22626; bugfix on
+      0.3.1.1-alpha.
+
+    - When decompressing a buffer, check for extra data after the end of
+      the compressed data. Fixes bug 22629; bugfix on 0.3.1.1-alpha.
+
diff --git a/changes/bug22520 b/changes/bug22520
new file mode 100644
index 0000000000..cc14f7214c
--- /dev/null
+++ b/changes/bug22520
@@ -0,0 +1,5 @@
+  o Minor bugfixes (error reporting, windows):
+    - When formatting Windows error messages, use the English format
+      to avoid codepage issues. Fixes bug 22520; bugfix on
+      0.1.2.8-alpha. Patch from "Vort".
+
diff --git a/changes/bug22669 b/changes/bug22669
new file mode 100644
index 0000000000..804a39e781
--- /dev/null
+++ b/changes/bug22669
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compression):
+    - When serving directory votes compressed with zlib,
+      do not claim to have compressed them with zstd. Fixes bug 22669;
+      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug22670 b/changes/bug22670
new file mode 100644
index 0000000000..47403277d2
--- /dev/null
+++ b/changes/bug22670
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging, compression):
+    - When decompressing, do not warn if we fail to decompress using a
+      compression method that we merely guessed. Fixes part of
+      bug 22670; bugfix on 0.1.1.14-alpha.
diff --git a/changes/bug22670_02 b/changes/bug22670_02
new file mode 100644
index 0000000000..3e7a428faf
--- /dev/null
+++ b/changes/bug22670_02
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging, compression):
+   - When decompressing, treat mismatch between content-encoding and
+      actual compression type as a protocol warning. Fixes part of bug
+      22670; bugfix on 0.1.1.9-alpha.
diff --git a/changes/bug22670_03 b/changes/bug22670_03
new file mode 100644
index 0000000000..8a7aa49bcd
--- /dev/null
+++ b/changes/bug22670_03
@@ -0,0 +1,6 @@
+  o Minor bugfixes (compression):
+    - When decompressing an object received over an anonymous directory
+      connection, if we have already successfully decompressed it using an
+      acceptable compression method, do not reject it for looking like an
+      unacceptable compression method. Fixes part of bug 22670; bugfix on
+      0.3.1.1-alpha.
diff --git a/changes/bug22672 b/changes/bug22672
new file mode 100644
index 0000000000..ec6681149d
--- /dev/null
+++ b/changes/bug22672
@@ -0,0 +1,5 @@
+  o Minor features (compression, defensive programming):
+    - Detect and break out of infinite loops in our compression code.
+      We don't think that any such loops exist now, but it's best to be
+      safe. Closes ticket 22672.
+
diff --git a/changes/bug22702 b/changes/bug22702
new file mode 100644
index 0000000000..a2044c70bf
--- /dev/null
+++ b/changes/bug22702
@@ -0,0 +1,5 @@
+  o Major bugfixes (directory protocol):
+    - Ensure that we sent "304 Not modified" as HTTP status code when a
+      client is attempting to fetch a consensus or consensus diff that
+      matches the latest consensus we have available. Fixes bug 22702;
+      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug22719 b/changes/bug22719
new file mode 100644
index 0000000000..bfcda0a4e1
--- /dev/null
+++ b/changes/bug22719
@@ -0,0 +1,7 @@
+  o Minor bugfixes (compression):
+    - When spooling compressed data to an output buffer, don't try to
+      spool more data when there is no more data to spool and we are
+      not trying to flush the input. Previously, we would sometimes
+      launch compression requests with nothing to do, which interferes
+      with our 22672 checks. Fixes bug 22719; bugfix on 0.2.0.16-alpha.
+
diff --git a/changes/bug22720 b/changes/bug22720
new file mode 100644
index 0000000000..4893b577f0
--- /dev/null
+++ b/changes/bug22720
@@ -0,0 +1,9 @@
+  o Minor bugfixes (process behavior):
+    - When exiting because of an error, always exit with a nonzero
+      exit status. Previously, we would fail to report an error in
+      our exit status in cases related to lockfile contention,
+      __OwningControllerProcess failure, and Ed25519 key
+      initialization.  Fixes bug 22720; bugfix on versions
+      0.2.1.6-alpha, 0.2.2.28-beta, and 0.2.7.2-alpha
+      respectively. Reported by "f55jwk4f"; patch from "huyvq".
+
diff --git a/changes/bug22751 b/changes/bug22751
new file mode 100644
index 0000000000..714525c8af
--- /dev/null
+++ b/changes/bug22751
@@ -0,0 +1,5 @@
+  o Major bugfixes (compression):
+    - Fix crash in LZMA module, when the Sandbox is enabled, where
+      liblzma would allocate more than 16 MB of memory.  We solve this
+      by bumping the mprotect() limit in the Sandbox module from 16 MB
+      to 20 MB.  Fixes bug 22751; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug22752_simple b/changes/bug22752_simple
new file mode 100644
index 0000000000..7e60357052
--- /dev/null
+++ b/changes/bug22752_simple
@@ -0,0 +1,6 @@
+  o Major bugfixes (windows, directory cache):
+    - On windows, do not try to delete cached consensus documents and
+      diffs, until they unmapped from memory. Allow the diff storage
+      directory to grow larger in order to handle files that might
+      need to stay around longer.  Fixes bug 22752; bugfix on
+      0.3.1.1-alpha.
diff --git a/changes/bug22753 b/changes/bug22753
new file mode 100644
index 0000000000..32a6dfa56c
--- /dev/null
+++ b/changes/bug22753
@@ -0,0 +1,7 @@
+  o Major bugfixes (path selection, security):
+    - When choosing which guard to use for a circuit, avoid the
+      exit's family along with the exit itself. Previously, the new
+      guard selection logic avoided the exit, but did not consider
+      its family.  Fixes bug 22753; bugfix on 0.3.0.1-alpha. Tracked
+      as TROVE-2016-006 and CVE-2017-0377.
+
diff --git a/changes/bug22803 b/changes/bug22803
new file mode 100644
index 0000000000..80b4b9f589
--- /dev/null
+++ b/changes/bug22803
@@ -0,0 +1,3 @@
+  o Minor bugfixes (unit tests):
+    - Fix a memory leak in the link-handshake/certs_ok_ed25519 test.
+      Fixes bug 22803; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug22830 b/changes/bug22830
new file mode 100644
index 0000000000..123b725aff
--- /dev/null
+++ b/changes/bug22830
@@ -0,0 +1,5 @@
+ o Minor bugfixes:
+   - Fix a problem with Rust toolchains not being found when building
+     without --enable-cargo-online-mode, due to setting the $HOME
+     environment variable instead of $CARGO_HOME.  Fixes bug 22830;
+     fix by Chelsea Komlo.  Bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug22838_028 b/changes/bug22838_028
deleted file mode 100644
index 1d0a4fbfd1..0000000000
--- a/changes/bug22838_028
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (compilation, mingw, backport from 0.3.1.1-alpha):
-    - Backport a fix for an "unused variable" warning that appeared
-      in some versions of mingw. Fixes bug 22838; bugfix on
-      0.2.8.1-alpha.
-
diff --git a/changes/bug22883-config b/changes/bug22883-config
new file mode 100644
index 0000000000..d60594d9ae
--- /dev/null
+++ b/changes/bug22883-config
@@ -0,0 +1,7 @@
+  o Minor features (directory cache, consensus diff):
+    - Add a new MaxConsensusAgeForDiffs option to allow directory cache
+      operators with low-resource environments to adjust the number of
+      consensuses they'll store and generate diffs from. Most cache operators
+      should leave it unchanged. Helps to work around bug 22883.
+
+
diff --git a/changes/bug22883-priority b/changes/bug22883-priority
new file mode 100644
index 0000000000..4b3531c30b
--- /dev/null
+++ b/changes/bug22883-priority
@@ -0,0 +1,8 @@
+  o Major bugfixes (relay, performance):
+
+    - Perform circuit handshake operations at a higher priority than we use
+      for consensus diff creation and compression. This should prevent
+      circuits from starving when a relay or bridge receive a new consensus,
+      especially on lower-powered machines. Fixes bug 22883; bugfix on
+      0.3.1.1-alpha.
+
diff --git a/changes/bug22892 b/changes/bug22892
new file mode 100644
index 0000000000..9a70cb0576
--- /dev/null
+++ b/changes/bug22892
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - Compile correctly when both openssl 1.1.0 and libscrypt are detected.
+      Previously this would cause an error. Fixes bug 22892; bugfix on 
+      0.3.1.1-alpha.
diff --git a/changes/bug22927 b/changes/bug22927
new file mode 100644
index 0000000000..6e68e6ff08
--- /dev/null
+++ b/changes/bug22927
@@ -0,0 +1,6 @@
+  o Minor bugfixes (compatibility, zstd):
+    - Write zstd epilogues correctly when the epilogue requires reallocation
+      of the output buffer, even with zstd 1.3.0. (Previously,
+      we worked on 1.2.0 and failed with 1.3.0).  Fixes bug 22927; bugfix on
+      0.3.1.1-alpha.
+
diff --git a/changes/bug23053 b/changes/bug23053
new file mode 100644
index 0000000000..082e239409
--- /dev/null
+++ b/changes/bug23053
@@ -0,0 +1,5 @@
+  o Minor bugfixes (memory leak):
+    - Fix a small memory leak when validating a configuration that
+      uses two or more AF_UNIX sockets for the same port type.
+      Fixes bug 23053; bugfix on 0.2.6.3-alpha. This is CID
+      1415725.
diff --git a/changes/bug23071 b/changes/bug23071
new file mode 100644
index 0000000000..4756dd6252
--- /dev/null
+++ b/changes/bug23071
@@ -0,0 +1,5 @@
+  o Minor bugfixes (tests):
+    - Port the hs_ntor handshake test to work correctly with recent
+      versions of the pysha3 module. Fixes bug 23071; bugfix on
+      0.3.1.1-alpha.
+
diff --git a/changes/bug23077 b/changes/bug23077
new file mode 100644
index 0000000000..5ed1c56742
--- /dev/null
+++ b/changes/bug23077
@@ -0,0 +1,4 @@
+  o Minor bugfixes (unit tests):
+    - Fix a channelpadding unit test failure on extremely slow systems
+      by using mocked time instead of actual time. Fixes bug 23077; bugfix on
+      0.3.1.1-alpha. 
diff --git a/changes/bug23078 b/changes/bug23078
new file mode 100644
index 0000000000..67624007cf
--- /dev/null
+++ b/changes/bug23078
@@ -0,0 +1,7 @@
+  o Minor bugfixes (logging, relay):
+    - Remove a log_warn() that has been forgotten when an introduction point
+      successfully established a hidden service prop224 circuit with a client.
+    - Three other log_warn() for an introduction point have been changed to
+      protocol warning because they can be failure from the network and are
+      not relevant to the operator. Fixes bug 23078; bugfix on
+      tor-0.3.0.1-alpha and tor-0.3.0.2-alpha.
diff --git a/changes/bug23105-diagnostic b/changes/bug23105-diagnostic
new file mode 100644
index 0000000000..8ba4931e36
--- /dev/null
+++ b/changes/bug23105-diagnostic
@@ -0,0 +1,4 @@
+  o Minor features (diagnostic):
+    - Add a stack trace to the bug warnings that can be logged when
+      trying to send an outgoing relay cell with n_chan == 0.
+      Diagnostic attempt for bug 23105.
diff --git a/changes/bug23139 b/changes/bug23139
new file mode 100644
index 0000000000..ed63ce85e2
--- /dev/null
+++ b/changes/bug23139
@@ -0,0 +1,3 @@
+  o Minor bugfixes (directory cache):
+    - Fix a memory leak in the code that recovers space in the consensus
+      directory cache. Fixes bug 23139; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug23155 b/changes/bug23155
new file mode 100644
index 0000000000..4c24ab136c
--- /dev/null
+++ b/changes/bug23155
@@ -0,0 +1,4 @@
+  o Minor bugfixes (stability):
+    - Avoid crashing on double-free when unable to load or process
+      an included file. Fixes bug 23155; bugfix on 0.3.1.1-alpha.
+      Found with the clang static analyzer.
diff --git a/changes/bug23233 b/changes/bug23233
new file mode 100644
index 0000000000..689a99a2a8
--- /dev/null
+++ b/changes/bug23233
@@ -0,0 +1,4 @@
+  o Minor bugfixes (hidden service):
+    - Fix a BUG alert during HSv3 descriptor decoding that could trigger with a
+      specially crafted descriptor. Fixes bug #23233; bugfix on 0.3.0.1-alpha.
+      Bug found by "haxxpop".
diff --git a/changes/bug23275 b/changes/bug23275
new file mode 100644
index 0000000000..d6c3c47743
--- /dev/null
+++ b/changes/bug23275
@@ -0,0 +1,5 @@
+  o Minor bugfixes (relay):
+    - When a relay is not running as a directory cache, it will no longer
+      generate compressed consensuses and consensus diff information.
+      Previously, this was a waste of disk and CPU.  Fixes bug 23275;
+      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug23533 b/changes/bug23533
new file mode 100644
index 0000000000..b5bfdc0ce2
--- /dev/null
+++ b/changes/bug23533
@@ -0,0 +1,4 @@
+  o Minor bugfixes (relay):
+    - Inform the geoip and rephist modules about all requests, even
+      on relays that are only fetching microdescriptors. Fixes a bug related
+      to 21585; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug23551 b/changes/bug23551
new file mode 100644
index 0000000000..2f918bfa3a
--- /dev/null
+++ b/changes/bug23551
@@ -0,0 +1,3 @@
+  o Minor bugfixes (compression):
+    - Handle a pathological case when decompressing Zstandard data when the
+      output buffer size is zero. Fixes bug 23551; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug23568 b/changes/bug23568
new file mode 100644
index 0000000000..cac4655687
--- /dev/null
+++ b/changes/bug23568
@@ -0,0 +1,4 @@
+  o Minor bugfixes (compilation):
+    - Fix a compilation warning when building with zstd support
+      on 32-bit platforms. Fixes bug 23568; bugfix on 0.3.1.1-alpha.
+      Found and fixed by Andreas Stieger.
diff --git a/changes/bug23608 b/changes/bug23608
new file mode 100644
index 0000000000..16cf88aa3d
--- /dev/null
+++ b/changes/bug23608
@@ -0,0 +1,4 @@
+  o Minor bugfixes (unit tests):
+    - Fix additional channelpadding unit test failures by using mocked time
+      instead of actual time for all tests. Fixes bug 23608;
+      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug23610 b/changes/bug23610
new file mode 100644
index 0000000000..f2dc8bd1a6
--- /dev/null
+++ b/changes/bug23610
@@ -0,0 +1,4 @@
+  o Minor bugfixes (hidden service, relay):
+    - Avoid a possible double close of a circuit by the intro point on error
+      of sending the INTRO_ESTABLISHED cell. Fixes ticket 23610; bugfix on
+      0.3.0.1-alpha.
diff --git a/changes/bug23693.1 b/changes/bug23693.1
new file mode 100644
index 0000000000..4b16788814
--- /dev/null
+++ b/changes/bug23693.1
@@ -0,0 +1,4 @@
+  o Minor bugfixes (relay, crash):
+    - Avoid a crash when running with DirPort set but ORPort tuned off.
+      Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha.
+
diff --git a/changes/bug23817 b/changes/bug23817
new file mode 100644
index 0000000000..4740942799
--- /dev/null
+++ b/changes/bug23817
@@ -0,0 +1,3 @@
+  o Minor bugfixes (descriptors):
+    - Don't try fetching microdescriptors from relays that have failed to
+      deliver them in the past. Fixes bug 23817; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug23862 b/changes/bug23862
new file mode 100644
index 0000000000..301ce73672
--- /dev/null
+++ b/changes/bug23862
@@ -0,0 +1,5 @@
+  o Minor bugfixes (entry guards):
+    - Tor now updates its guard state when it reads a consensus regardless of
+      whether it's missing descriptors. That makes tor use its primary guards
+      to fetch descriptors in some edge cases where it would have used fallback
+      directories in the past. Fixes bug 23862; bugfix on 0.3.0.1-alpha.
\ No newline at end of file
diff --git a/changes/bug23908 b/changes/bug23908
new file mode 100644
index 0000000000..f641b66bb9
--- /dev/null
+++ b/changes/bug23908
@@ -0,0 +1,3 @@
+  o Minor bugfixes (directory authority, backport from 0.3.2.1-alpha):
+    - Remove the length limit on HTTP status lines that authorities can send
+      in their replies. Fixes bug 23499; bugfix on 0.3.1.6-rc.
diff --git a/changes/bug24086 b/changes/bug24086
new file mode 100644
index 0000000000..2ae0b37e65
--- /dev/null
+++ b/changes/bug24086
@@ -0,0 +1,7 @@
+  o Minor bugfixes (directory cache):
+    - When a consensus diff calculation is only partially successful, only
+      record the successful parts as having succeeded. Partial success
+      can happen if (for example) one compression method fails but
+      the others succeed. Previously we misrecorded all the calculations as
+      having succeeded, which would later cause a nonfatal assertion failure.
+      Fixes bug 24086; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug24099 b/changes/bug24099
new file mode 100644
index 0000000000..dca3992664
--- /dev/null
+++ b/changes/bug24099
@@ -0,0 +1,4 @@
+  o Minor bugfixes (directory cache):
+    - Recover better from empty or corrupt files in the consensus cache
+      directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha.
+
diff --git a/changes/bug24262 b/changes/bug24262
new file mode 100644
index 0000000000..eee69512e4
--- /dev/null
+++ b/changes/bug24262
@@ -0,0 +1,3 @@
+  o Minor bugfixes (hidden service):
+    - Fix the consensus parameter "hsdir-interval" to "hsdir_interval" so it
+      matches the dir-spec.txt. Fixes bug 24262; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug24826_031 b/changes/bug24826_031
new file mode 100644
index 0000000000..3d4a66184a
--- /dev/null
+++ b/changes/bug24826_031
@@ -0,0 +1,4 @@
+  o Minor bugfixes (performance, fragile-hardening):
+    - Improve the performance of our consensus-diff application code when Tor
+      is built with the --enable-fragile-hardening option set. Fixes bug
+      24826; bugfix on  0.3.1.1-alpha.
diff --git a/changes/bug24859 b/changes/bug24859
new file mode 100644
index 0000000000..122109d650
--- /dev/null
+++ b/changes/bug24859
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging):
+    - Don't treat inability to store a cached consensus object as a
+      bug: it can happen normally when we are out of disk space.
+      Fixes bug 24859; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug24898 b/changes/bug24898
new file mode 100644
index 0000000000..f64340d71b
--- /dev/null
+++ b/changes/bug24898
@@ -0,0 +1,8 @@
+  o Major bugfixes (relays):
+    - Fix a set of false positives where relays would consider connections
+      to other relays as being client-only connections (and thus e.g.
+      deserving different link padding schemes) if those relays fell out
+      of the consensus briefly. Now we look only at the initial handshake
+      and whether the connection authenticated as a relay. Fixes bug
+      24898; bugfix on 0.3.1.1-alpha.
+
diff --git a/changes/bug25070 b/changes/bug25070
new file mode 100644
index 0000000000..c2f4e58c45
--- /dev/null
+++ b/changes/bug25070
@@ -0,0 +1,3 @@
+  o Major bugfixes (protocol versions):
+    - Add Link protocol version 5 to the supported protocols list.
+      Fixes bug 25070; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug26069 b/changes/bug26069
new file mode 100644
index 0000000000..192e97d782
--- /dev/null
+++ b/changes/bug26069
@@ -0,0 +1,5 @@
+  o Minor bugfixes (hidden service v3):
+    - When parsing the descriptor signature, look for the token plus an extra
+      white-space at the end. This is more correct but also will allow us to
+      support new fields that might start with "signature". Fixes bug 26069;
+      bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug26158 b/changes/bug26158
new file mode 100644
index 0000000000..0d74cf1167
--- /dev/null
+++ b/changes/bug26158
@@ -0,0 +1,5 @@
+  o Minor bugfixes (relay):
+     - Relays now correctly block attempts to re-extend to the previous
+       relay by Ed25519 identity. Previously they would warn in this case,
+       but not actually reject the attempt. Fixes bug 26158; bugfix on
+       0.3.0.1-alpha.
diff --git a/changes/bug26272 b/changes/bug26272
new file mode 100644
index 0000000000..9dcf42f0e1
--- /dev/null
+++ b/changes/bug26272
@@ -0,0 +1,3 @@
+  o Minor bugfixes (compilation):
+    - Silence unused-const-variable warnings in zstd.h on some gcc versions.
+      Fixes bug 26272; bugfix on 0.3.1.1-alpha.
diff --git a/changes/diagnose_22752 b/changes/diagnose_22752
new file mode 100644
index 0000000000..b5bda05ec0
--- /dev/null
+++ b/changes/diagnose_22752
@@ -0,0 +1,4 @@
+  o Minor features (bug mitigation, diagnostics, logging):
+    - Avoid an assertion failure, and log a better error message,
+      when unable to remove a file from the consensus cache on
+      Windows. Attempts to mitigate and diagnose bug 22752.
diff --git a/changes/geoip-april2017 b/changes/geoip-april2017
deleted file mode 100644
index b489eaf016..0000000000
--- a/changes/geoip-april2017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features:
-    - Update geoip and geoip6 to the April 4 2017 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/geoip-december2016 b/changes/geoip-december2016
deleted file mode 100644
index 60754ea21d..0000000000
--- a/changes/geoip-december2016
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features:
-    - Update geoip and geoip6 to the December 7 2016 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/geoip-february2017 b/changes/geoip-february2017
deleted file mode 100644
index ec54b6122a..0000000000
--- a/changes/geoip-february2017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features:
-    - Update geoip and geoip6 to the February 8 2017 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/geoip-january2017 b/changes/geoip-january2017
deleted file mode 100644
index 77bc9a5991..0000000000
--- a/changes/geoip-january2017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the January 4 2017 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/geoip-march2017 b/changes/geoip-march2017
deleted file mode 100644
index 6dc92baa2f..0000000000
--- a/changes/geoip-march2017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features:
-    - Update geoip and geoip6 to the March 7 2017 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/geoip-may2017 b/changes/geoip-may2017
deleted file mode 100644
index 4e504d7a0a..0000000000
--- a/changes/geoip-may2017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features:
-    - Update geoip and geoip6 to the May 2 2017 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/geoip-november2016 b/changes/geoip-november2016
deleted file mode 100644
index b3f9913bb1..0000000000
--- a/changes/geoip-november2016
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (ge0oip):
-    - Update geoip and geoip6 to the November 3 2016 Maxmind GeoLite2
-      Country database.
-
diff --git a/changes/more-files b/changes/more-files
new file mode 100644
index 0000000000..861d6a3143
--- /dev/null
+++ b/changes/more-files
@@ -0,0 +1,4 @@
+  o Documentation:
+    - Document more of the files in the Tor data directory, including
+      cached-extrainfo, secret_onion_key{,_ntor}.old, hidserv-stats,
+      approved-routers, sr-random, and diff-cache.
diff --git a/changes/more-threads b/changes/more-threads
new file mode 100644
index 0000000000..eae88b70fd
--- /dev/null
+++ b/changes/more-threads
@@ -0,0 +1,3 @@
+  o Minor features (relay, performance):
+    - Always start relays with at least two worker threads, to prevent
+      priority inversion on slow tasks.  Part of the fix for bug 22883.
diff --git a/changes/more_module_docs b/changes/more_module_docs
deleted file mode 100644
index 0066ddfcf0..0000000000
--- a/changes/more_module_docs
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation:
-    - Module-level documentation for several more modules. Closes tickets 
-      19287 and 
-      19290.
diff --git a/changes/multi-priority b/changes/multi-priority
new file mode 100644
index 0000000000..6f19314b53
--- /dev/null
+++ b/changes/multi-priority
@@ -0,0 +1,5 @@
+  o Minor features (relay, thread pool):
+    - Allow background work to be queued with different priorities, so
+      that a big pile of slow low-priority jobs will not starve out
+      higher priority jobs. This lays the groundwork for a fix for bug
+      22883.
diff --git a/changes/new_requirement_pkgconfig b/changes/new_requirement_pkgconfig
new file mode 100644
index 0000000000..503ff58c9e
--- /dev/null
+++ b/changes/new_requirement_pkgconfig
@@ -0,0 +1,5 @@
+  o New dependencies:
+    - To build with zstd and lzma support, Tor now requires the
+      pkg-config tool at build time. (This requirement was new in
+      0.3.1.1-alpha, but was not noted at the time. Noting it here to
+      close ticket 22623.)
diff --git a/changes/prop275-minimal b/changes/prop275-minimal
deleted file mode 100644
index 83d42f850b..0000000000
--- a/changes/prop275-minimal
+++ /dev/null
@@ -1,9 +0,0 @@
-  o Minor features (future-proofing):
-
-    - Tor no longer refuses to download microdescriptors or descriptors if
-      they are listed as "published in the future".  This change will
-      eventually allow us to stop listing meaningful "published" dates
-      in microdescriptor consensuses, and thereby allow us to reduce the
-      resources required to download consensus diffs by over 50%.
-      Implements part of ticket 21642; implements part of proposal 275.
-
diff --git a/changes/task-22207 b/changes/task-22207
new file mode 100644
index 0000000000..63544834bf
--- /dev/null
+++ b/changes/task-22207
@@ -0,0 +1,4 @@
+  o Minor features:
+    - Add "fingerprint" line to networkstatus-bridges produced by
+      bridge authorities. Implements #22207.
+
diff --git a/changes/ticket19769 b/changes/ticket19769
deleted file mode 100644
index 9fc05c3e9e..0000000000
--- a/changes/ticket19769
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Major features (security):
-    - Change the algorithm used to decide DNS TTLs on client and server side,
-      to better resist DNS-based correlation attacks like the DefecTor attack
-      of Greschbach, Pulls, Roberts, Winter, and Feamster).  Now
-      relays only return one of two possible DNS TTL values, and clients
-      are willing to believe DNS TTL values up to 3 hours long.
-      Closes ticket 19769.
diff --git a/changes/ticket20170-v3 b/changes/ticket20170-v3
deleted file mode 100644
index d634e72053..0000000000
--- a/changes/ticket20170-v3
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (fallback directory list):
-    - Replace the 81 remaining fallbacks of the 100 originally introduced
-      in Tor 0.2.8.3-alpha in March 2016, with a list of 177 fallbacks
-      (123 new, 54 existing, 27 removed) generated in December 2016.
-      Resolves ticket 20170.
diff --git a/changes/ticket21564 b/changes/ticket21564
deleted file mode 100644
index 7e01f41f8f..0000000000
--- a/changes/ticket21564
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor features (fallback directory list):
-    - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in
-      December 2016 (of which ~126 were still functional), with a list of
-      151 fallbacks (32 new, 119 existing, 58 removed) generated in
-      May 2017.
-      Resolves ticket 21564.
diff --git a/changes/ticket21953 b/changes/ticket21953
deleted file mode 100644
index 7cc84f506d..0000000000
--- a/changes/ticket21953
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor features:
-    - Enable a couple of pieces of Windows hardening: one
-      (HeapEnableTerminationOnCorruption) that has been on-by-default since
-      Windows 8, and unavailable before Windows 7, and one
-      (PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION) which we believe doesn't
-      affect us, but shouldn't do any harm. Closes ticket 21953.
diff --git a/changes/ticket22348 b/changes/ticket22348
new file mode 100644
index 0000000000..49ae94cdf3
--- /dev/null
+++ b/changes/ticket22348
@@ -0,0 +1,5 @@
+  o Minor features (directory authority):
+    - Improve the message that authorities report to relays when
+      the RSA/Ed25519 key pair they present conflicts with a previously
+      pinned key. Closes ticket 22348.
+
diff --git a/changes/ticket22870 b/changes/ticket22870
new file mode 100644
index 0000000000..07cc8a1d04
--- /dev/null
+++ b/changes/ticket22870
@@ -0,0 +1,5 @@
+  o Minor bugfixes (consensus diff):
+    - test_consdiff_base64cmp would fail on OS X because while OS X
+      follows the standard of (less than zero/zero/greater than zero),
+      it doesn't follow the convention of (-1/0/+1). Make the test
+      comply with the standard. Fixes bug 22870; bugfix on 0.3.1.1-alpha.
diff --git a/changes/ticket25323 b/changes/ticket25323
new file mode 100644
index 0000000000..836825de5d
--- /dev/null
+++ b/changes/ticket25323
@@ -0,0 +1,4 @@
+  o Code simplification and refactoring:
+    - Update the "rust dependencies" submodule to be an project-level
+      repository, rather than a user repository. Closes ticket 25323.
+
diff --git a/changes/ticket25714 b/changes/ticket25714
new file mode 100644
index 0000000000..63823fc6ca
--- /dev/null
+++ b/changes/ticket25714
@@ -0,0 +1,4 @@
+  o Minor feature (continuous integration):
+    - Update the Travis CI configuration to use the stable Rust
+      channel, now that we have decided to require that.  Closes
+      ticket 25714.
diff --git a/changes/trove-2017-001 b/changes/trove-2017-001
deleted file mode 100644
index 5187e6d5f1..0000000000
--- a/changes/trove-2017-001
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Major bugfixes (security):
-    - Downgrade the "-ftrapv" option from "always on" to "only on when
-      --enable-expensive-hardening is provided."  This hardening option, like
-      others, can turn survivable bugs into crashes--and having it on by
-      default made a (relatively harmless) integer overflow bug into a
-      denial-of-service bug. Fixes bug 21278 (TROVE-2017-001); bugfix on
-      0.2.9.1-alpha.
-
diff --git a/changes/trove-2017-001.2 b/changes/trove-2017-001.2
deleted file mode 100644
index 3ef073cf9f..0000000000
--- a/changes/trove-2017-001.2
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Major bugfixes (parsing):
-    - Fix an integer underflow bug when comparing malformed Tor versions.
-      This bug is harmless, except when Tor has been built with
-      --enable-expensive-hardening, which would turn it into a crash;
-      or on Tor 0.2.9.1-alpha through Tor 0.2.9.8, which were built with
-      -ftrapv by default.
-      Part of TROVE-2017-001. Fixes bug 21278; bugfix on
-      0.0.8pre1. Found by OSS-Fuzz.
diff --git a/changes/trove-2017-005 b/changes/trove-2017-005
deleted file mode 100644
index cebb013f86..0000000000
--- a/changes/trove-2017-005
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Major bugfixes (hidden service, relay, security):
-    - Fix an assertion failure caused by receiving a BEGIN_DIR cell on
-      a hidden service rendezvous circuit. Fixes bug 22494, tracked as
-      TROVE-2017-005 and CVE-2017-0376; bugfix on 0.2.2.1-alpha. Found
-      by armadev.
-
-
diff --git a/changes/trove-2017-012-part2 b/changes/trove-2017-012-part2
new file mode 100644
index 0000000000..ed994c5b02
--- /dev/null
+++ b/changes/trove-2017-012-part2
@@ -0,0 +1,5 @@
+  o Major bugfixes (security, relay):
+    - When running as a relay, make sure that we never ever choose ourselves
+      as a guard. Previously, this was possible. Fixes part of bug 21534;
+      bugfix on 0.3.0.1-alpha. This issue is also tracked as TROVE-2017-012
+      and CVE-2017-8822.
diff --git a/configure.ac b/configure.ac
index 279660d9b2..7bfbcfccfc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,17 +1,17 @@
 dnl Copyright (c) 2001-2004, Roger Dingledine
 dnl Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
-dnl Copyright (c) 2007-2015, The Tor Project, Inc.
+dnl Copyright (c) 2007-2017, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_PREREQ([2.63])
-AC_INIT([tor],[0.2.9.15-dev])
+AC_INIT([tor],[0.3.1.10-dev])
 AC_CONFIG_SRCDIR([src/or/main.c])
 AC_CONFIG_MACRO_DIR([m4])
 
 # "foreign" means we don't follow GNU package layout standards
 # "1.11" means we require automake version 1.11 or newer
 # "subdir-objects" means put .o files in the same directory as the .c files
-AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects])
+AM_INIT_AUTOMAKE([foreign 1.11 subdir-objects -Wall -Werror])
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_CONFIG_HEADERS([orconfig.h])
@@ -49,6 +49,16 @@ AC_ARG_ENABLE(asserts-in-tests,
    AS_HELP_STRING(--disable-asserts-in-tests, [disable tor_assert() calls in the unit tests, for branch coverage]))
 AC_ARG_ENABLE(system-torrc,
    AS_HELP_STRING(--disable-system-torrc, [don't look for a system-wide torrc file]))
+AC_ARG_ENABLE(libfuzzer,
+   AS_HELP_STRING(--enable-libfuzzer, [build extra fuzzers based on 'libfuzzer']))
+AC_ARG_ENABLE(oss-fuzz,
+   AS_HELP_STRING(--enable-oss-fuzz, [build extra fuzzers based on 'oss-fuzz' environment]))
+AC_ARG_ENABLE(memory-sentinels,
+   AS_HELP_STRING(--disable-memory-sentinels, [disable code that tries to prevent some kinds of memory access bugs. For fuzzing only.]))
+AC_ARG_ENABLE(rust,
+   AS_HELP_STRING(--enable-rust, [enable rust integration]))
+AC_ARG_ENABLE(cargo-online-mode,
+   AS_HELP_STRING(--enable-cargo-online-mode, [Allow cargo to make network requests to fetch crates. For builds with rust only.]))
 
 if test "x$enable_coverage" != "xyes" -a "x$enable_asserts_in_tests" = "xno" ; then
     AC_MSG_ERROR([Can't disable assertions outside of coverage build])
@@ -57,6 +67,9 @@ fi
 AM_CONDITIONAL(UNITTESTS_ENABLED, test "x$enable_unittests" != "xno")
 AM_CONDITIONAL(COVERAGE_ENABLED, test "x$enable_coverage" = "xyes")
 AM_CONDITIONAL(DISABLE_ASSERTS_IN_UNIT_TESTS, test "x$enable_asserts_in_tests" = "xno")
+AM_CONDITIONAL(LIBFUZZER_ENABLED, test "x$enable_libfuzzer" = "xyes")
+AM_CONDITIONAL(OSS_FUZZ_ENABLED, test "x$enable_oss_fuzz" = "xyes")
+AM_CONDITIONAL(USE_RUST, test "x$enable_rust" = "xyes")
 
 if test "$enable_static_tor" = "yes"; then
   enable_static_libevent="yes";
@@ -70,6 +83,11 @@ if test "$enable_system_torrc" = "no"; then
             [Defined if we're not going to look for a torrc in SYSCONF])
 fi
 
+if test "$enable_memory_sentinels" = "no"; then
+  AC_DEFINE(DISABLE_MEMORY_SENTINELS, 1,
+           [Defined if we're turning off memory safety code to look for bugs])
+fi
+
 AM_CONDITIONAL(USE_OPENBSD_MALLOC, test "x$enable_openbsd_malloc" = "xyes")
 
 AC_ARG_ENABLE(asciidoc,
@@ -139,8 +157,14 @@ dnl Others suggest '/gs /safeseh /nxcompat /dynamicbase' for non-gcc on Windows
 AC_ARG_ENABLE(gcc-hardening,
     AS_HELP_STRING(--disable-gcc-hardening, [disable compiler security checks]))
 
+dnl Deprecated --enable-expensive-hardening but keep it for now for backward compat.
 AC_ARG_ENABLE(expensive-hardening,
-    AS_HELP_STRING(--enable-expensive-hardening, [enable more expensive compiler hardening; makes Tor slower]))
+    AS_HELP_STRING(--enable-expensive-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower]))
+AC_ARG_ENABLE(fragile-hardening,
+    AS_HELP_STRING(--enable-fragile-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower]))
+if test "x$enable_expensive_hardening" = "xyes" || test "x$enable_fragile_hardening" = "xyes"; then
+  fragile_hardening="yes"
+fi
 
 dnl Linker hardening options
 dnl Currently these options are ELF specific - you can't use this with MacOSX
@@ -170,11 +194,25 @@ AC_ARG_ENABLE(seccomp,
 AC_ARG_ENABLE(libscrypt,
      AS_HELP_STRING(--disable-libscrypt, [do not attempt to use libscrypt]))
 
-dnl check for the correct "ar" when cross-compiling
-AN_MAKEVAR([AR], [AC_PROG_AR])
-AN_PROGRAM([ar], [AC_PROG_AR])
-AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL([AR], [ar], [ar])])
-AC_PROG_AR
+dnl Enable event tracing which are transformed to debug log statement.
+AC_ARG_ENABLE(event-tracing-debug,
+     AS_HELP_STRING(--enable-event-tracing-debug, [build with event tracing to debug log]))
+AM_CONDITIONAL([USE_EVENT_TRACING_DEBUG], [test "x$enable_event_tracing_debug" = "xyes"])
+
+if test x$enable_event_tracing_debug = xyes; then
+  AC_DEFINE([USE_EVENT_TRACING_DEBUG], [1], [Tracing framework to log debug])
+  AC_DEFINE([TOR_EVENT_TRACING_ENABLED], [1], [Compile the event tracing instrumentation])
+fi
+
+dnl check for the correct "ar" when cross-compiling.
+dnl   (AM_PROG_AR was new in automake 1.11.2, which we do not yet require,
+dnl    so kludge up a replacement for the case where it isn't there yet.)
+m4_ifdef([AM_PROG_AR],
+         [AM_PROG_AR],
+         [AN_MAKEVAR([AR], [AC_PROG_AR])
+          AN_PROGRAM([ar], [AC_PROG_AR])
+          AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL([AR], [ar], [:])])
+          AC_PROG_AR])
 
 dnl Check whether the above macro has settled for a simply named tool even
 dnl though we're cross compiling. We must do this before running AC_PROG_CC,
@@ -194,11 +232,11 @@ AC_PROG_CC
 AC_PROG_CPP
 AC_PROG_MAKE_SET
 AC_PROG_RANLIB
+AC_PROG_SED
 
-AC_PATH_PROG([PERL], [perl])
-
-dnl autoconf 2.59 appears not to support AC_PROG_SED
-AC_CHECK_PROG([SED],[sed],[sed],[/bin/false])
+AC_ARG_VAR([PERL], [path to Perl binary])
+AC_CHECK_PROGS([PERL], [perl])
+AM_CONDITIONAL(USE_PERL, [test "x$ac_cv_prog_PERL" != "x"])
 
 dnl check for asciidoc and a2x
 AC_PATH_PROG([ASCIIDOC], [asciidoc], none)
@@ -206,9 +244,6 @@ AC_PATH_PROGS([A2X], [a2x a2x.py], none)
 
 AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue")
 
-AM_CONDITIONAL(USE_FW_HELPER, test "x$natpmp" = "xtrue" || test "x$upnp" = "xtrue")
-AM_CONDITIONAL(NAT_PMP, test "x$natpmp" = "xtrue")
-AM_CONDITIONAL(MINIUPNPC, test "x$upnp" = "xtrue")
 AM_PROG_CC_C_O
 AC_PROG_CC_C99
 
@@ -223,6 +258,68 @@ if test "x$PYTHON" = "x"; then
 fi
 AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"])
 
+dnl List all external rust crates we depend on here. Include the version
+rust_crates="libc-0.2.22"
+AC_SUBST(rust_crates)
+
+if test "x$enable_rust" = "xyes"; then
+  AC_ARG_VAR([RUSTC], [path to the rustc binary])
+  AC_CHECK_PROG([RUSTC], [rustc], [rustc],[no])
+  if test "x$RUSTC" = "xno"; then
+    AC_MSG_ERROR([rustc unavailable but rust integration requested.])
+  fi
+
+  AC_ARG_VAR([CARGO], [path to the cargo binary])
+  AC_CHECK_PROG([CARGO], [cargo], [cargo],[no])
+  if test "x$CARGO" = "xno"; then
+    AC_MSG_ERROR([cargo unavailable but rust integration requested.])
+  fi
+
+  AC_DEFINE([HAVE_RUST], 1, [have Rust])
+  if test "x$enable_cargo_online_mode" = "xyes"; then
+    CARGO_ONLINE=
+    RUST_DL=#
+  else
+    CARGO_ONLINE=--frozen
+    RUST_DL=
+
+    dnl When we're not allowed to touch the network, we need crate dependencies
+    dnl locally available.
+    AC_MSG_CHECKING([rust crate dependencies])
+    AC_ARG_VAR([RUST_DEPENDENCIES], [path to directory with local crate mirror])
+    if test "x$RUST_DEPENDENCIES" = "x"; then
+      RUST_DEPENDENCIES="$srcdir/src/ext/rust/"
+      NEED_MOD=1
+    fi
+    if test ! -d "$RUST_DEPENDENCIES"; then
+      AC_MSG_ERROR([Rust dependency directory $RUST_DEPENDENCIES does not exist. Specify a dependency directory using the RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.])
+    fi
+    for dep in $rust_crates; do
+      if test ! -d "$RUST_DEPENDENCIES"/"$dep"; then
+        AC_MSG_ERROR([Failure to find rust dependency $RUST_DEPENDENCIES/$dep. Specify a dependency directory using the RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.])
+      fi
+    done
+    if test "x$NEED_MOD" = "x1"; then
+      dnl When looking for dependencies from cargo, pick right directory
+      RUST_DEPENDENCIES="../../src/ext/rust"
+    fi
+  fi
+
+  AC_SUBST(CARGO_ONLINE)
+  AC_SUBST(RUST_DL)
+
+dnl Let's check the rustc version, too
+  AC_MSG_CHECKING([rust version])
+  RUSTC_VERSION_MAJOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 1`
+  RUSTC_VERSION_MINOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 2`
+  if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then
+    AC_MSG_ERROR([rustc version couldn't be identified])
+  fi
+  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 14; then
+    AC_MSG_ERROR([rustc must be at least version 1.14])
+  fi
+fi
+
 ifdef([AC_C_FLEXIBLE_ARRAY_MEMBER], [
 AC_C_FLEXIBLE_ARRAY_MEMBER
 ], [
@@ -432,7 +529,7 @@ AC_CHECK_FUNCS(
 # checks. So we should only probe for those functions if we are sure that we
 # are not targetting OSX 10.11 or earlier.
 AC_MSG_CHECKING([for a pre-Sierra OSX build target])
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #ifdef __APPLE__
 #  include <AvailabilityMacros.h>
 #  ifndef MAC_OS_X_VERSION_10_12
@@ -444,7 +541,7 @@ AC_TRY_COMPILE([
 #    endif
 #  endif
 #endif
-], [],
+]], [[]])],
    [on_macos_pre_10_12=no ; AC_MSG_RESULT([no])],
    [on_macos_pre_10_12=yes; AC_MSG_RESULT([yes])])
 
@@ -644,16 +741,16 @@ LIBS="$TOR_OPENSSL_LIBS $LIBS"
 LDFLAGS="$TOR_LDFLAGS_openssl $LDFLAGS"
 CPPFLAGS="$TOR_CPPFLAGS_openssl $CPPFLAGS"
 
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
 #if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x1000100fL
 #error "too old"
 #endif
-   ], [],
+   ]], [[]])],
    [ : ],
-   [ AC_ERROR([OpenSSL is too old. We require 1.0.1 or later. You can specify a path to a newer one with --with-openssl-dir.]) ])
+   [ AC_MSG_ERROR([OpenSSL is too old. We require 1.0.1 or later. You can specify a path to a newer one with --with-openssl-dir.]) ])
 
-AC_TRY_COMPILE([
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #if defined(OPENSSL_NO_EC) || defined(OPENSSL_NO_ECDH) || defined(OPENSSL_NO_ECDSA)
@@ -662,9 +759,9 @@ AC_TRY_COMPILE([
 #if !defined(NID_X9_62_prime256v1) || !defined(NID_secp224r1)
 #error "curves unavailable"
 #endif
-   ], [],
+   ]], [[]])],
    [ : ],
-   [ AC_ERROR([OpenSSL is built without full ECC support, including curves P256 and P224. You can specify a path to one with ECC support with --with-openssl-dir.]) ])
+   [ AC_MSG_ERROR([OpenSSL is built without full ECC support, including curves P256 and P224. You can specify a path to one with ECC support with --with-openssl-dir.]) ])
 
 AC_CHECK_MEMBERS([struct ssl_method_st.get_cipher_by_char], , ,
 [#include <openssl/ssl.h>
@@ -717,6 +814,70 @@ else
 fi
 AC_SUBST(TOR_ZLIB_LIBS)
 
+dnl ------------------------------------------------------
+dnl Where we do we find lzma?
+
+AC_ARG_ENABLE(lzma,
+      AS_HELP_STRING(--enable-lzma, [enable support for the LZMA compression scheme.]),
+      [case "${enableval}" in
+        "yes") lzma=true ;;
+        "no")  lzma=false ;;
+        * ) AC_MSG_ERROR(bad value for --enable-lzma) ;;
+      esac], [lzma=auto])
+
+if test "x$enable_lzma" = "xno"; then
+    have_lzma=no;
+else
+    PKG_CHECK_MODULES([LZMA],
+                      [liblzma],
+                      have_lzma=yes,
+                      have_lzma=no)
+
+    if test "x$have_lzma" = "xno" ; then
+        AC_MSG_WARN([Unable to find liblzma.])
+    fi
+fi
+
+if test "x$have_lzma" = "xyes"; then
+    AC_DEFINE(HAVE_LZMA,1,[Have LZMA])
+    TOR_LZMA_CFLAGS="${LZMA_CFLAGS}"
+    TOR_LZMA_LIBS="${LZMA_LIBS}"
+fi
+AC_SUBST(TOR_LZMA_CFLAGS)
+AC_SUBST(TOR_LZMA_LIBS)
+
+dnl ------------------------------------------------------
+dnl Where we do we find zstd?
+
+AC_ARG_ENABLE(zstd,
+      AS_HELP_STRING(--enable-zstd, [enable support for the Zstandard compression scheme.]),
+      [case "${enableval}" in
+        "yes") zstd=true ;;
+        "no")  zstd=false ;;
+        * ) AC_MSG_ERROR(bad value for --enable-zstd) ;;
+      esac], [zstd=auto])
+
+if test "x$enable_zstd" = "xno"; then
+    have_zstd=no;
+else
+    PKG_CHECK_MODULES([ZSTD],
+                      [libzstd >= 1.1],
+                      have_zstd=yes,
+                      have_zstd=no)
+
+    if test "x$have_zstd" = "xno" ; then
+        AC_MSG_WARN([Unable to find libzstd.])
+    fi
+fi
+
+if test "x$have_zstd" = "xyes"; then
+    AC_DEFINE(HAVE_ZSTD,1,[Have Zstd])
+    TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}"
+    TOR_ZSTD_LIBS="${ZSTD_LIBS}"
+fi
+AC_SUBST(TOR_ZSTD_CFLAGS)
+AC_SUBST(TOR_ZSTD_LIBS)
+
 dnl ----------------------------------------------------------------------
 dnl Check if libcap is available for capabilities.
 
@@ -744,6 +905,7 @@ CFLAGS_FWRAPV=
 CFLAGS_ASAN=
 CFLAGS_UBSAN=
 
+
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
 #if !defined(__clang__)
 #error
@@ -767,21 +929,21 @@ m4_ifdef([AS_VAR_IF],[
     AS_VAR_POPDEF([can_compile])
     TOR_CHECK_CFLAGS(-Wstack-protector)
     TOR_CHECK_CFLAGS(--param ssp-buffer-size=1)
-    if test "$bwin32" = "false"; then
+    if test "$bwin32" = "false" && test "$enable_libfuzzer" != "yes" && test "$enable_oss_fuzz" != "yes"; then
        TOR_CHECK_CFLAGS(-fPIE)
        TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check")
     fi
     TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true)
 fi
 
-if test "x$enable_expensive_hardening" = "xyes"; then
+if test "$fragile_hardening" = "yes"; then
     TOR_TRY_COMPILE_WITH_CFLAGS(-ftrapv, also_link, CFLAGS_FTRAPV="-ftrapv", true)
    if test "$tor_cv_cflags__ftrapv" = "yes" && test "$tor_can_link__ftrapv" != "yes"; then
       AC_MSG_WARN([The compiler supports -ftrapv, but for some reason I was not able to link with -ftrapv. Are you missing run-time support? Run-time hardening will not work as well as it should.])
    fi
 
    if test "$tor_cv_cflags__ftrapv" != "yes"; then
-     AC_MSG_ERROR([You requested expensive hardening, but the compiler does not seem to support -ftrapv.])
+     AC_MSG_ERROR([You requested fragile hardening, but the compiler does not seem to support -ftrapv.])
    fi
 
    TOR_TRY_COMPILE_WITH_CFLAGS([-fsanitize=address], also_link, CFLAGS_ASAN="-fsanitize=address", true)
@@ -863,7 +1025,7 @@ saved_CFLAGS="$CFLAGS"
 TOR_CHECK_CFLAGS(-fomit-frame-pointer)
 F_OMIT_FRAME_POINTER=''
 if test "$saved_CFLAGS" != "$CFLAGS"; then
-  if test "x$enable_expensive_hardening" != "xyes"; then
+  if test "$fragile_hardening" = "yes"; then
     F_OMIT_FRAME_POINTER='-fomit-frame-pointer'
   fi
 fi
@@ -1142,10 +1304,6 @@ if test "x$linux_netfilter_ipv6_ip6_tables" = "x1"; then
 fi
 if test "x$transparent_ok" = "x1"; then
   AC_DEFINE(USE_TRANSPARENT, 1, "Define to enable transparent proxy support")
-  case "$host" in
-    *-*-openbsd* | *-*-bitrig*)
-      AC_DEFINE(OPENBSD, 1, "Define to handle pf on OpenBSD properly") ;;
-  esac
 else
   AC_MSG_NOTICE([Transparent proxy support enabled, but missing headers.])
 fi
@@ -1449,6 +1607,14 @@ AC_CHECK_DECLS([mlockall], , , [
 #include <sys/mman.h>
 #endif])
 
+# Some MinGW environments don't have getpagesize in unistd.h. We don't use
+# AC_CHECK_FUNCS(getpagesize), because other environments rename getpagesize
+# using macros
+AC_CHECK_DECLS([getpagesize], , , [
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif])
+
 # Allow user to specify an alternate syslog facility
 AC_ARG_WITH(syslog-facility,
 AS_HELP_STRING(--with-syslog-facility=LOG, [syslog facility to use (default=LOG_DAEMON)]),
@@ -1468,9 +1634,9 @@ AC_CHECK_FUNC(gethostbyname_r, [
   AC_MSG_CHECKING([how many arguments gethostbyname_r() wants])
   OLD_CFLAGS=$CFLAGS
   CFLAGS="$CFLAGS $MY_CPPFLAGS $MY_THREAD_CPPFLAGS $MY_CFLAGS"
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([
+  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-  ], [[
+  ]], [[
     char *cp1, *cp2;
     struct hostent *h1, *h2;
     int i1, i2;
@@ -1481,27 +1647,27 @@ AC_CHECK_FUNC(gethostbyname_r, [
      [Define this if gethostbyname_r takes 6 arguments])
     AC_MSG_RESULT(6)
   ], [
-    AC_TRY_COMPILE([
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-    ], [
+    ]], [[
       char *cp1, *cp2;
       struct hostent *h1;
       int i1, i2;
       (void)gethostbyname_r(cp1,h1,cp2,i1,&i2);
-    ], [
+    ]])], [
       AC_DEFINE(HAVE_GETHOSTBYNAME_R)
       AC_DEFINE(HAVE_GETHOSTBYNAME_R_5_ARG, 1,
         [Define this if gethostbyname_r takes 5 arguments])
       AC_MSG_RESULT(5)
    ], [
-      AC_TRY_COMPILE([
+      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <netdb.h>
-     ], [
+     ]], [[
        char *cp1;
        struct hostent *h1;
        struct hostent_data hd;
        (void) gethostbyname_r(cp1,h1,&hd);
-     ], [
+     ]])], [
        AC_DEFINE(HAVE_GETHOSTBYNAME_R)
        AC_DEFINE(HAVE_GETHOSTBYNAME_R_3_ARG, 1,
          [Define this if gethostbyname_r takes 3 arguments])
@@ -1933,6 +2099,7 @@ AC_CONFIG_FILES([
         contrib/dist/tor.service
         src/config/torrc.sample
         src/config/torrc.minimal
+        src/rust/.cargo/config
         scripts/maint/checkOptionDocs.pl
         scripts/maint/updateVersions.pl
 ])
@@ -1955,4 +2122,19 @@ if test "x$asciidoc" = "xtrue" && test "$ASCIIDOC" = "none"; then
   done
 fi
 
+if test "$fragile_hardening" = "yes"; then
+  AC_MSG_WARN([
+
+============
+Warning!  Building Tor with --enable-fragile-hardening (also known as
+--enable-expensive-hardening) makes some kinds of attacks harder, but makes
+other kinds of attacks easier. A Tor instance build with this option will be
+somewhat less vulnerable to remote code execution, arithmetic overflow, or
+out-of-bounds read/writes... but at the cost of becoming more vulnerable to
+denial of service attacks. For more information, see
+https://trac.torproject.org/projects/tor/wiki/doc/TorFragileHardening
+============
+  ])
+fi
+
 AC_OUTPUT
diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in
index 3a45741d8d..7ddb21d8a7 100644
--- a/contrib/win32build/tor-mingw.nsi.in
+++ b/contrib/win32build/tor-mingw.nsi.in
@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.2.9.15-dev"
+!define VERSION "0.3.1.10-dev"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"
diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md
index f1c65850a4..c7787a72cc 100644
--- a/doc/HACKING/CodingStandards.md
+++ b/doc/HACKING/CodingStandards.md
@@ -4,9 +4,10 @@ Coding conventions for Tor
 tl;dr:
 
    - Run configure with `--enable-fatal-warnings`
-   - Run `make check-spaces` to catch whitespace errors
    - Document your functions
    - Write unit tests
+   - Run `make test-full` to test against all unit and integration tests.
+   - Run `make distcheck` to ensure the distribution works
    - Add a file in `changes` for your branch.
 
 Patch checklist
@@ -22,10 +23,12 @@ preference)
 Did you remember...
 
    - To build your code while configured with `--enable-fatal-warnings`?
-   - To run `make check-spaces` on your code?
    - To run `make check-docs` to see whether all new options are on
      the manpage?
    - To write unit tests, as possible?
+   - To run `make test-full` to test against all unit and integration tests (or
+     `make test-full-online` if you have a working connection to the internet)?
+   - To test that the distribution will actually work via `make distcheck`?
    - To base your code on the appropriate branch?
    - To include a file in the `changes` directory as appropriate?
 
@@ -93,6 +96,10 @@ What needs a changes file?
    rewrites.  Anything about which somebody might plausibly wonder "when
    did that happen, and/or why did we do that" 6 months down the line.
 
+What does not need a changes file?
+
+   * Bugfixes for code that hasn't shipped in any released version of Tor
+
 Why use changes files instead of Git commit messages?
 
    * Git commit messages are written for developers, not users, and they
diff --git a/doc/HACKING/Fuzzing.md b/doc/HACKING/Fuzzing.md
new file mode 100644
index 0000000000..2039d6a4c0
--- /dev/null
+++ b/doc/HACKING/Fuzzing.md
@@ -0,0 +1,123 @@
+= Fuzzing Tor
+
+== The simple version (no fuzzing, only tests)
+
+Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place
+where you checked it out.
+
+To run the fuzzing test cases in a deterministic fashion, use:
+      make test-fuzz-corpora
+
+This won't actually fuzz Tor!  It will just run all the fuzz binaries
+on our existing set of testcases for the fuzzer.
+
+
+== Different kinds of fuzzing
+
+Right now we support three different kinds of fuzzer.
+
+First, there's American Fuzzy Lop (AFL), a fuzzer that works by forking
+a target binary and passing it lots of different inputs on stdin.  It's the
+trickiest one to set up, so I'll be describing it more below.
+
+Second, there's libFuzzer, a llvm-based fuzzer that you link in as a library,
+and it runs a target function over and over.  To use this one, you'll need to
+have a reasonably recent clang and libfuzzer installed.  At that point, you
+just build with --enable-expensive-hardening and --enable-libfuzzer.  That
+will produce a set of binaries in src/test/fuzz/lf-fuzz-* .  These programs
+take as input a series of directories full of fuzzing examples.  For more
+information on libfuzzer, see http://llvm.org/docs/LibFuzzer.html
+
+Third, there's Google's OSS-Fuzz infrastructure, which expects to get all of
+its.  For more on this, see https://github.com/google/oss-fuzz and the
+projects/tor subdirectory.  You'll need to mess around with Docker a bit to
+test this one out; it's meant to run on Google's infrastructure.
+
+In all cases, you'll need some starting examples to give the fuzzer when it
+starts out.  There's a set in the "fuzzing-corpora" git repository.  Try
+setting TOR_FUZZ_CORPORA to point to a checkout of that repository
+
+== Writing Tor fuzzers
+
+A tor fuzzing harness should have:
+* a fuzz_init() function to set up any necessary global state.
+* a fuzz_main() function to receive input and pass it to a parser.
+* a fuzz_cleanup() function to clear global state.
+
+Most fuzzing frameworks will produce many invalid inputs - a tor fuzzing
+harness should rejecting invalid inputs without crashing or behaving badly.
+
+But the fuzzing harness should crash if tor fails an assertion, triggers a
+bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect
+"interesting" cases.
+
+
+== Guided Fuzzing with AFL
+
+There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so
+its integrity can't be verified. That said, you really shouldn't fuzz on a
+machine you care about, anyway.
+
+To Build:
+  Get AFL from http://lcamtuf.coredump.cx/afl/ and unpack it
+  cd afl
+  make
+  cd ../tor
+  PATH=$PATH:../afl/ CC="../afl/afl-gcc" ./configure --enable-expensive-hardening
+  AFL_HARDEN=1 make clean fuzzers
+
+To Find The ASAN Memory Limit: (64-bit only)
+
+On 64-bit platforms, afl needs to know how much memory ASAN uses,
+because ASAN tends to allocate a ridiculous amount of virtual memory,
+and then not actually use it.
+
+Read afl/docs/notes_for_asan.txt for more details.
+
+  Download recidivm from http://jwilk.net/software/recidivm
+  Download the signature
+  Check the signature
+  tar xvzf recidivm*.tar.gz
+  cd recidivm*
+  make
+  /path/to/recidivm -v src/test/fuzz/fuzz-http
+  Use the final "ok" figure as the input to -m when calling afl-fuzz
+  (Normally, recidivm would output a figure automatically, but in some cases,
+  the fuzzing harness will hang when the memory limit is too small.)
+
+You could also just say "none" instead of the memory limit below, if you
+don't care about memory limits.
+
+
+To Run:
+  mkdir -p src/test/fuzz/fuzz_http_findings
+  ../afl/afl-fuzz -i ${TOR_FUZZ_CORPORA}/http -o src/test/fuzz/fuzz_http_findings -m <asan-memory-limit> -- src/test/fuzz/fuzz-http
+
+
+AFL has a multi-core mode, check the documentation for details.
+You might find the included fuzz-multi.sh script useful for this.
+
+macOS (OS X) requires slightly more preparation, including:
+* using afl-clang (or afl-clang-fast from the llvm directory)
+* disabling external crash reporting (AFL will guide you through this step)
+
+== Triaging Issues
+
+Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code.
+
+Hangs might be interesting, but they might also be spurious machine slowdowns.
+Check if a hang is reproducible before reporting it. Sometimes, processing
+valid inputs may take a second or so, particularly with the fuzzer and
+sanitizers enabled.
+
+To see what fuzz-http is doing with a test case, call it like this:
+  src/test/fuzz/fuzz-http --debug < /path/to/test.case
+
+(Logging is disabled while fuzzing to increase fuzzing speed.)
+
+== Reporting Issues
+
+Please report any issues discovered using the process in Tor's security issue
+policy:
+
+https://trac.torproject.org/projects/tor/wiki/org/meetings/2016SummerDevMeeting/Notes/SecurityIssuePolicy
diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md
index a7f36e6c7e..b8ba2aa408 100644
--- a/doc/HACKING/HelpfulTools.md
+++ b/doc/HACKING/HelpfulTools.md
@@ -142,6 +142,12 @@ run `make test-network`.
 We also have scripts to run integration tests using Stem.  To try them, set
 `STEM_SOURCE_DIR` to your Stem source directory, and run `test-stem`.
 
+Profiling Tor
+-------------
+
+Ongoing notes about Tor profiling can be found at
+https://pad.riseup.net/p/profiling-tor
+
 Profiling Tor with oprofile
 ---------------------------
 
@@ -168,20 +174,62 @@ Here are some basic instructions
    * `opreport -l that_dir/*`
  - Profit
 
+Profiling Tor with perf
+-----------------------
+
+This works with a running Tor, and requires root.
+
+1. Decide how long you want to profile for. Start with (say) 30 seconds. If that
+   works, try again with longer times.
+
+2. Find the PID of your running tor process.
+
+3. Run `perf record --call-graph dwarf -p <PID> sleep <SECONDS>`
+
+   (You may need to do this as root.)
+
+   You might need to add `-e cpu-clock` as an option to the perf record line
+   above, if you are on an older CPU without access to hardware profiling
+   events, or in a VM, or something.
+
+4. Now you have a perf.data file. Have a look at it with `perf report
+   --no-children --sort symbol,dso` or `perf report --no-children --sort
+   symbol,dso --stdio --header`. How does it look?
+
+5a. Once you have a nice big perf.data file, you can compress it, encrypt it,
+    and send it to your favorite Tor developers.
+
+5b. Or maybe you'd rather not send a nice big perf.data file. Who knows what's
+    in that!? It's kinda scary. To generate a less scary file, you can use `perf
+    report -g > <FILENAME>.out`. Then you can compress that and put it somewhere
+    public.
+
+Profiling Tor with gperftools aka Google-performance-tools
+----------------------------------------------------------
+
+This should work on nearly any unixy system. It doesn't seem to be compatible
+with RunAsDaemon though.
+
+Beforehand, install google-perftools.
+
+1. You need to rebuild Tor, hack the linking steps to add `-lprofiler` to the
+   libs. You can do this by adding `LIBS=-lprofiler` when you call `./configure`.
+
+Now you can run Tor with profiling enabled, and use the pprof utility to look at
+performance! See the gperftools manual for more info, but basically:
+
+2. Run `env CPUPROFILE=/tmp/profile src/or/tor -f <path/torrc>`. The profile file
+   is not written to until Tor finishes execuction.
+
+3. Run `pprof src/or/tor /tm/profile` to start the REPL.
+
 Generating and analyzing a callgraph
 ------------------------------------
 
-1. Run `./scripts/maint/generate_callgraph.sh`.  This will generate a
-   bunch of files in a new ./callgraph directory.
-
-2. Run `./scripts/maint/analyze_callgraph.py callgraph/src/*/*`.  This
-   will do a lot of graph operations and then dump out a new
-   `callgraph.pkl` file, containing data in Python's 'pickle' format.
+0. Build Tor on linux or mac, ideally with -O0 or -fno-inline.
 
-3. Run `./scripts/maint/display_callgraph.py`.  It will display:
-    - the number of functions reachable from each function.
-    - all strongly-connnected components in the Tor callgraph
-    - the largest bottlenecks in the largest SCC in the Tor callgraph.
+1. Clone 'https://gitweb.torproject.org/user/nickm/calltool.git/' .
+   Follow the README in that repository.
 
 Note that currently the callgraph generator can't detect calls that pass
 through function pointers.
diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md
index 7595398241..4ece4d7a1d 100644
--- a/doc/HACKING/ReleasingTor.md
+++ b/doc/HACKING/ReleasingTor.md
@@ -26,9 +26,7 @@ new Tor release:
 
    What about Coverity Scan?
 
-   Is make check-spaces happy?
-
-   Does 'make distcheck' compain?
+   Does 'make distcheck' complain?
 
    How about 'make test-stem' and 'make test-network'?
 
@@ -44,21 +42,16 @@ new Tor release:
    of them and reordering to focus on what users and funders would find
    interesting and understandable.
 
-   1. Make sure that everything that wants a bug number has one.
-      Make sure that everything which is a bugfix says what version
-      it was a bugfix on.
-
-   2. Concatenate them.
-
-   3. Sort them by section. Within each section, sort by "version it's
-      a bugfix on", else by numerical ticket order.
+   To do this, first run `./scripts/maint/lintChanges.py changes/*` and
+   fix as many warnings as you can.  Then run `./scripts/maint/sortChanges.py
+   changes/* > changelog.in` to combine headings and sort the entries.
+   After that, it's time to hand-edit and fix the issues that lintChanges
+   can't find:
 
-   4. Clean them up:
+   1. Within each section, sort by "version it's a bugfix on", else by
+      numerical ticket order.
 
-      Standard idioms:
-      `Fixes bug 9999; bugfix on 0.3.3.3-alpha.`
-
-      One space after a period.
+   2. Clean them up:
 
       Make stuff very terse
 
@@ -86,19 +79,23 @@ new Tor release:
       maint-0.2.1), be sure to make the stanzas identical (so people can
       distinguish if these are the same change).
 
-   5. Merge them in.
+   3. Clean everything one last time.
 
-   6. Clean everything one last time.
+   4. Run `./scripts/maint/format_changelog.py --inplace` to make it prettier
 
-   7. Run `./scripts/maint/format_changelog.py` to make it prettier.
 
 2. Compose a short release blurb to highlight the user-facing
    changes. Insert said release blurb into the ChangeLog stanza. If it's
    a stable release, add it to the ReleaseNotes file too. If we're adding
-   to a release-0.2.x branch, manually commit the changelogs to the later
+   to a release-* branch, manually commit the changelogs to the later
    git branches too.
 
-3.  If you're doing the first stable release in a series, you need to
+3. If there are changes that require or suggest operator intervention
+   before or during the update, mail operators (either dirauth or relays
+   list) with a headline that indicates that an action is required or
+   appreciated.
+
+4. If you're doing the first stable release in a series, you need to
    create a ReleaseNotes for the series as a whole.  To get started
    there, copy all of the Changelog entries from the series into a new
    file, and run `./scripts/maint/sortChanges.py` on it.  That will
@@ -111,38 +108,40 @@ new Tor release:
 
 === III. Making the source release.
 
-1. In `maint-0.2.x`, bump the version number in `configure.ac` and run
+1. In `maint-0.?.x`, bump the version number in `configure.ac` and run
    `scripts/maint/updateVersions.pl` to update version numbers in other
-   places, and commit.  Then merge `maint-0.2.x` into `release-0.2.x`.
+   places, and commit.  Then merge `maint-0.?.x` into `release-0.?.x`.
 
    (NOTE: To bump the version number, edit `configure.ac`, and then run
    either `make`, or `perl scripts/maint/updateVersions.pl`, depending on
    your version.)
 
-2. Make distcheck, put the tarball up somewhere, and tell `#tor` about
-   it. Wait a while to see if anybody has problems building it. Try to
-   get Sebastian or somebody to try building it on Windows.
+2. Make distcheck, put the tarball up in somewhere (how about your
+   homedir on your homedir on people.torproject.org?) , and tell `#tor`
+   about it. Wait a while to see if anybody has problems building it.
+   (Though jenkins is usually pretty good about catching these things.)
 
 === IV. Commit, upload, announce
 
 1. Sign the tarball, then sign and push the git tag:
 
         gpg -ba <the_tarball>
-        git tag -u <keyid> tor-0.2.x.y-status
-        git push origin tag tor-0.2.x.y-status
+        git tag -u <keyid> tor-0.3.x.y-status
+        git push origin tag tor-0.3.x.y-status
 
 2. scp the tarball and its sig to the dist website, i.e.
    `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want
    it to go live, you run "static-update-component dist.torproject.org"
    on dist-master.
 
-   Edit `include/versions.wmi` and `Makefile` to note the new version.
+   In the webwml.git repository, `include/versions.wmi` and `Makefile`
+   to note the new version.
 
    (NOTE: Due to #17805, there can only be one stable version listed at
    once.  Nonetheless, do not call your version "alpha" if it is stable,
    or people will get confused.)
 
-3. Email the packagers (cc'ing tor-assistants) that a new tarball is up.
+3. Email the packagers (cc'ing tor-team) that a new tarball is up.
    The current list of packagers is:
 
        - {weasel,gk,mikeperry} at torproject dot org
@@ -151,11 +150,7 @@ new Tor release:
        - {lfleischer} at archlinux dot org
        - {Nathan} at freitas dot net
        - {mike} at tig dot as
-       - {tails-rm} at boum dot org (for pre-release announcments)
-
-
-       - {tails-dev} at boum dot org (for at-release announcements)
-
+       - {tails-rm} at boum dot org
 
 4. Add the version number to Trac.  To do this, go to Trac, log in,
     select "Admin" near the top of the screen, then select "Versions" from
@@ -164,22 +159,26 @@ new Tor release:
     0.2.2.23-alpha" (or whatever the version is), and we select the date as
     the date in the ChangeLog.
 
-5. Wait up to a day or two (for a development release), or until most
-    packages are up (for a stable release), and mail the release blurb and
-    changelog to tor-talk or tor-announce.
+5. Mail the release blurb and ChangeLog to tor-talk (development release) or
+   tor-announce (stable).
+
+   Post the changelog on the the blog as well. You can generate a
+   blog-formatted version of the changelog with the -B option to
+   format-changelog.
 
-   (We might be moving to faster announcements, but don't announce until
-   the website is at least updated.)
+   When you post, include an estimate of when the next TorBrowser
+   releases will come out that include this Tor release.  This will
+   usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it
+   can vary.
 
 
 === V. Aftermath and cleanup
 
-1. If it's a stable release, bump the version number in the `maint-x.y.z`
-    branch to "newversion-dev", and do a `merge -s ours` merge to avoid
-    taking that change into master.  Do a similar `merge -s theirs`
-    merge to get the change (and only that change) into release.  (Some
-    of the build scripts require that maint merge cleanly into release.)
+1. If it's a stable release, bump the version number in the
+    `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours`
+    merge to avoid taking that change into master.
 
 2. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
 
+3. Keep an eye on the blog post, to moderate comments and answer questions.
 
diff --git a/doc/HACKING/Tracing.md b/doc/HACKING/Tracing.md
new file mode 100644
index 0000000000..a5fb5165e2
--- /dev/null
+++ b/doc/HACKING/Tracing.md
@@ -0,0 +1,91 @@
+# Tracing #
+
+This document describes how the event tracing subsystem works in tor so
+developers can add events to the code base but also hook them to an event
+tracing framework.
+
+## Basics ###
+
+Event tracing is seperated in two concepts, trace events and a tracer. The
+tracing subsystem can be found in `src/trace`. The `events.h` header file is
+the main file that maps the different tracers to trace events.
+
+### Events ###
+
+A trace event is basically a function from which we can pass any data that
+we want to collect. In addition, we specify a context for the event such as
+a subsystem and an event name.
+
+A trace event in tor has the following standard format:
+
+	tor_trace(subsystem, event\_name, args...)
+
+The `subsystem` parameter is the name of the subsytem the trace event is in.
+For example that could be "scheduler" or "vote" or "hs". The idea is to add
+some context to the event so when we collect them we know where it's coming
+from. The `event_name` is the name of the event which helps a lot with
+adding some semantic to the event. Finally, `args` is any number of
+arguments we want to collect.
+
+Here is an example of a possible tracepoint in main():
+
+	tor_trace(main, init_phase, argc)
+
+The above is a tracepoint in the `main` subsystem with `init_phase` as the
+event name and the `int argc` is passed to the event as well.
+
+How `argc` is collected or used has nothing to do with the instrumentation
+(adding trace events to the code). It is the work of the tracer so this is why
+the trace events and collection framework (tracer) are decoupled. You _can_
+have trace events without a tracer.
+
+### Tracer ###
+
+In `src/trace/events.h`, we map the `tor_trace()` function to the right
+tracer. A tracer support is only enabled at compile time. For instance, the
+file `src/trace/debug.h` contains the mapping of the generic tracing function
+`tor_trace()` to the `log_debug()` function. More specialized function can be
+mapped depending on the tracepoint.
+
+## Build System ##
+
+This section describes how it is integrated into the build system of tor.
+
+By default, every tracing events are disabled in tor that is `tor_trace()`
+is a NOP.
+
+To enable a tracer, there is a configure option on the form of:
+
+	--enable-tracing-<tracer>
+
+We have an option that will send every trace events to a `log_debug()` (as
+mentionned above) which will print you the subsystem and name of the event but
+not the arguments for technical reasons. This is useful if you want to quickly
+see if your trace event is being hit or well written. To do so, use this
+configure option:
+
+	--enable-tracing-debug
+
+## Instrument Tor ##
+
+This is pretty easy. Let's say you want to add a trace event in
+`src/or/rendcache.c`, you only have to add this include statement:
+
+	#include "trace/events.h"
+
+Once done, you can add as many as you want `tor_trace()` that you need.
+Please use the right subsystem (here it would be `hs`) and a unique name that
+tells what the event is for. For example:
+
+	tor_trace(hs, store_desc_as_client, desc, desc_id);
+
+If you look in `src/trace/events.h`, you'll see that if tracing is enabled it
+will be mapped to a function called:
+
+	tor_trace_hs_store_desc_as_client(desc, desc_id)
+
+And the point of all this is for that function to be defined in a new file
+that you might want to add named `src/trace/hs.{c|h}` which would defined how
+to collect the data for the `tor_trace_hs_store_desc_as_client()` function
+like for instance sending it to a `log_debug()` or do more complex operations
+or use a userspace tracer like LTTng (https://lttng.org).
diff --git a/doc/HACKING/WritingTests.md b/doc/HACKING/WritingTests.md
index de80bbdef2..4dae41e922 100644
--- a/doc/HACKING/WritingTests.md
+++ b/doc/HACKING/WritingTests.md
@@ -48,7 +48,7 @@ isolation, you just run `./src/test/test-memwipe`.
 To run tests within the unit test programs, you can specify the name
 of the test.  The string ".." can be used as a wildcard at the end of the
 test name.  For example, to run all the cell format tests, enter
-`./src/test/test cellfmt/..`.  To run
+`./src/test/test cellfmt/..`.
 
 Many tests that need to mess with global state run in forked subprocesses in
 order to keep from contaminating one another.  But when debugging a failing test,
diff --git a/doc/include.am b/doc/include.am
index 7164a4b2a0..0e8de231e1 100644
--- a/doc/include.am
+++ b/doc/include.am
@@ -12,17 +12,11 @@
 # part of the source distribution, so that people without asciidoc can
 # just use the .1 and .html files.
 
-base_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
-all_mans = $(base_mans)
-if USE_FW_HELPER
-install_mans = $(all_mans)
-else
-install_mans = $(base_mans)
-endif
+all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify
 
 if USE_ASCIIDOC
-nodist_man1_MANS = $(install_mans:=.1)
-doc_DATA = $(install_mans:=.html)
+nodist_man1_MANS = $(all_mans:=.1)
+doc_DATA = $(all_mans:=.html)
 html_in = $(all_mans:=.html.in)
 man_in = $(all_mans:=.1.in)
 txt_in = $(all_mans:=.1.txt)
diff --git a/doc/tor.1.txt b/doc/tor.1.txt
index a7ee7d11ca..a6b4f2fc4b 100644
--- a/doc/tor.1.txt
+++ b/doc/tor.1.txt
@@ -153,6 +153,13 @@ values. To split one configuration entry into multiple lines, use a single
 backslash character (\) before the end of the line.  Comments can be used in
 such multiline entries, but they must start at the beginning of a line.
 
+Configuration options can be imported from files or folders using the %include
+option with the value being a path. If the path is a file, the options from the
+file will be parsed as if they were written where the %include option is. If
+the path is a folder, all files on that folder will be parsed following lexical
+order. Files starting with a dot are ignored. Files on subfolders are ignored.
+The %include option can be used recursively.
+
 By default, an option on the command line overrides an option found in the
 configuration file, and an option in a configuration file overrides one in
 the defaults file.
@@ -176,7 +183,7 @@ forward slash (/) in the configuration file and on the command line.
 GENERAL OPTIONS
 ---------------
 
-[[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[BandwidthRate]] **BandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     A token bucket limits the average incoming bandwidth usage on this node
     to the specified number of bytes per second, and the average outgoing
     bandwidth usage to that same value.  If you want to run a relay in the
@@ -185,6 +192,9 @@ GENERAL OPTIONS
     course, more is better; we recommend at least 250 KBytes (2 mbits) if
     possible.  (Default: 1 GByte) +
  +
+    Note that this option, and other bandwidth-limiting options, apply to TCP
+    data only: They do not count TCP headers or DNS traffic. +
+ +
     With this option, and in other options that take arguments in bytes,
     KBytes, and so on, other formats are also supported. Notably, "KBytes" can
     also be written as "kilobytes" or "kb"; "MBytes" can be written as
@@ -195,35 +205,35 @@ GENERAL OPTIONS
     To avoid confusion, we recommend writing "bytes" or "bits" explicitly,
     since it's easy to forget that "B" means bytes, not bits.
 
-[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Limit the maximum token bucket size (also known as the burst) to the given
     number of bytes in each direction. (Default: 1 GByte)
 
-[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[MaxAdvertisedBandwidth]] **MaxAdvertisedBandwidth** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If set, we will not advertise more than this amount of bandwidth for our
     BandwidthRate. Server operators who want to reduce the number of clients
     who ask to build circuits through them (since this is proportional to
     advertised bandwidth rate) can thus reduce the CPU demands on their server
     without impacting network performance.
 
-[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[RelayBandwidthRate]] **RelayBandwidthRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If not 0, a separate token bucket limits the average incoming bandwidth
     usage for \_relayed traffic_ on this node to the specified number of bytes
     per second, and the average outgoing bandwidth usage to that same value.
     Relayed traffic currently is calculated to include answers to directory
     requests, but that may change in future versions. (Default: 0)
 
-[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[RelayBandwidthBurst]] **RelayBandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If not 0, limit the maximum token bucket size (also known as the burst) for
     \_relayed traffic_ to the given number of bytes in each direction.
     (Default: 0)
 
-[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[PerConnBWRate]] **PerConnBWRate** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If set, do separate rate limiting for each connection from a non-relay.
     You should never need to change this value, since a network-wide value is
     published in the consensus and your relay will use that value. (Default: 0)
 
-[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[PerConnBWBurst]] **PerConnBWBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     If set, do separate rate limiting for each connection from a non-relay.
     You should never need to change this value, since a network-wide value is
     published in the consensus and your relay will use that value. (Default: 0)
@@ -338,14 +348,6 @@ GENERAL OPTIONS
         Unix domain sockets only: Do not insist that the directory
         that holds the socket be read-restricted.
 
-[[ControlListenAddress]] **ControlListenAddress** __IP__[:__PORT__]::
-    Bind the controller listener to this address. If you specify a port, bind
-    to this port rather than the one specified in ControlPort. We strongly
-    recommend that you leave this alone unless you know what you're doing,
-    since giving attackers access to your control listener is really
-    dangerous. This directive can be specified multiple
-    times to bind to multiple addresses/ports.  (Default: 127.0.0.1)
-
 [[ControlSocket]] **ControlSocket** __Path__::
     Like ControlPort, but listens on a Unix domain socket, rather than a TCP
     socket. '0' disables ControlSocket (Unix and Unix-like systems only.)
@@ -390,7 +392,10 @@ GENERAL OPTIONS
     file readable by the default GID. (Default: 0)
 
 [[DataDirectory]] **DataDirectory** __DIR__::
-    Store working data in DIR (Default: @LOCALSTATEDIR@/lib/tor)
+    Store working data in DIR. Can not be changed while tor is running.
+    (Default: ~/.tor if your home directory is not /; otherwise,
+    @LOCALSTATEDIR@/lib/tor.  On Windows, the default is
+    your ApplicationData folder.)
 
 [[DataDirectoryGroupReadable]] **DataDirectoryGroupReadable** **0**|**1**::
     If this option is set to 0, don't allow the filesystem group to read the
@@ -465,7 +470,8 @@ GENERAL OPTIONS
     not supported. We believe that this feature works on modern Gnu/Linux
     distributions, and that it should work on *BSD systems (untested). This
     option requires that you start your Tor as root, and you should use the
-    **User** option to properly reduce Tor's privileges. (Default: 0)
+    **User** option to properly reduce Tor's privileges.
+    Can not be changed while tor is running. (Default: 0)
 
 [[DisableDebuggerAttachment]] **DisableDebuggerAttachment** **0**|**1**::
    If set to 1, Tor will attempt to prevent basic debugging attachment attempts
@@ -505,11 +511,13 @@ GENERAL OPTIONS
     (Default: 1)
 
 [[FetchUselessDescriptors]] **FetchUselessDescriptors** **0**|**1**::
-    If set to 1, Tor will fetch every non-obsolete descriptor from the
-    authorities that it hears about. Otherwise, it will avoid fetching useless
-    descriptors, for example for routers that are not running. This option is
-    useful if you're using the contributed "exitlist" script to enumerate Tor
-    nodes that exit to certain addresses. (Default: 0)
+    If set to 1, Tor will fetch every consensus flavor, descriptor, and
+    certificate that it hears about. Otherwise, it will avoid fetching useless
+    descriptors: flavors that it is not using to build circuits, and authority
+    certificates it does not trust. This option is useful if you're using a
+    tor client with an external parser that uses a full consensus.
+    This option fetches all documents, **DirCache** fetches and serves
+    all documents. (Default: 0)
 
 [[HTTPProxy]] **HTTPProxy** __host__[:__port__]::
     Tor will make all its directory requests through this host:port (or host:80
@@ -538,7 +546,20 @@ GENERAL OPTIONS
 [[Sandbox]] **Sandbox** **0**|**1**::
     If set to 1, Tor will run securely through the use of a syscall sandbox.
     Otherwise the sandbox will be disabled. The option is currently an
-    experimental feature. (Default: 0)
+    experimental feature. Can not be changed while tor is running.
+
+    When the Sandbox is 1, the following options can not be changed when tor
+    is running:
+    Address
+    ConnLimit
+    CookieAuthFile
+    DirPortFrontPage
+    ExtORPortCookieAuthFile
+    Logs
+    ServerDNSResolvConfFile
+    Tor must remain in client or server mode (some changes to ClientOnly and
+    ORPort are not allowed).
+    (Default: 0)
 
 [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]::
     Tor will make all OR connections through the SOCKS 4 proxy at host:port
@@ -609,7 +630,7 @@ GENERAL OPTIONS
     message currently has at least one domain; most currently have exactly
     one.  This doesn't affect controller log messages. (Default: 0)
 
-[[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**::
+[[MaxUnparseableDescSizeToLog]] **MaxUnparseableDescSizeToLog** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**::
     Unparseable descriptors (e.g. for votes, consensuses, routers) are logged
     in separate files by hash, up to the specified size in total.  Note that
     only files logged during the lifetime of this Tor process count toward the
@@ -624,24 +645,34 @@ GENERAL OPTIONS
     This setting will be ignored for connections to the loopback addresses
     (127.0.0.0/8 and ::1).
 
+[[OutboundBindAddressOR]] **OutboundBindAddressOR** __IP__::
+    Make all outbound non-exit (relay and other) connections
+    originate from the IP address specified. This option overrides
+    **OutboundBindAddress** for the same IP version. This option may
+    be used twice, once with an IPv4 address and once with an IPv6
+    address. This setting will be ignored for connections to the loopback
+    addresses (127.0.0.0/8 and ::1).
+
+[[OutboundBindAddressExit]] **OutboundBindAddressExit** __IP__::
+    Make all outbound exit connections originate from the IP address
+    specified. This option overrides **OutboundBindAddress** for the
+    same IP version. This option may be used twice, once with an IPv4
+    address and once with an IPv6 address. This setting will be ignored
+    for connections to the loopback addresses (127.0.0.0/8 and ::1).
+
 [[PidFile]] **PidFile** __FILE__::
     On startup, write our PID to FILE. On clean shutdown, remove
-    FILE.
+    FILE. Can not be changed while tor is running.
 
 [[ProtocolWarnings]] **ProtocolWarnings** **0**|**1**::
     If 1, Tor will log with severity \'warn' various cases of other parties not
     following the Tor specification. Otherwise, they are logged with severity
     \'info'. (Default: 0)
 
-[[PredictedPortsRelevanceTime]] **PredictedPortsRelevanceTime** __NUM__::
-    Set how long, after the client has made an anonymized connection to a
-    given port, we will try to make sure that we build circuits to
-    exits that support that port. The maximum value for this option is 1
-    hour. (Default: 1 hour)
-
 [[RunAsDaemon]] **RunAsDaemon** **0**|**1**::
     If 1, Tor forks and daemonizes to the background. This option has no effect
     on Windows; instead you should use the --service command-line option.
+    Can not be changed while tor is running.
     (Default: 0)
 
 [[LogTimeGranularity]] **LogTimeGranularity** __NUM__::
@@ -658,7 +689,8 @@ GENERAL OPTIONS
 
 [[SyslogIdentityTag]] **SyslogIdentityTag** __tag__::
     When logging to syslog, adds a tag to the syslog identity such that
-    log entries are marked with "Tor-__tag__".  (Default: none)
+    log entries are marked with "Tor-__tag__". Can not be changed while tor is
+    running. (Default: none)
 
 [[SafeLogging]] **SafeLogging** **0**|**1**|**relay**::
     Tor can scrub potentially sensitive strings from log messages (e.g.
@@ -673,6 +705,7 @@ GENERAL OPTIONS
 
 [[User]] **User** __Username__::
     On startup, setuid to this user and setgid to their primary group.
+    Can not be changed while tor is running.
 
 [[KeepBindCapabilities]] **KeepBindCapabilities** **0**|**1**|**auto**::
     On Linux, when we are started as root and we switch our identity using
@@ -680,20 +713,23 @@ GENERAL OPTIONS
     try to retain our ability to bind to low ports.  If this value is 1, we
     try to keep the capability; if it is 0 we do not; and if it is **auto**,
     we keep the capability only if we are configured to listen on a low port.
+    Can not be changed while tor is running.
     (Default: auto.)
 
 [[HardwareAccel]] **HardwareAccel** **0**|**1**::
     If non-zero, try to use built-in (static) crypto hardware acceleration when
-    available. (Default: 0)
+    available. Can not be changed while tor is running. (Default: 0)
 
 [[AccelName]] **AccelName** __NAME__::
     When using OpenSSL hardware crypto acceleration attempt to load the dynamic
     engine of this name. This must be used for any dynamic hardware engine.
-    Names can be verified with the openssl engine command.
+    Names can be verified with the openssl engine command. Can not be changed
+    while tor is running.
 
 [[AccelDir]] **AccelDir** __DIR__::
     Specify this option if using dynamic hardware acceleration and the engine
     implementation library resides somewhere other than the OpenSSL default.
+    Can not be changed while tor is running.
 
 [[AvoidDiskWrites]] **AvoidDiskWrites** **0**|**1**::
     If non-zero, try to write to disk less frequently than we would otherwise.
@@ -718,28 +754,19 @@ GENERAL OPTIONS
     127.0.0.1 or 10.0.0.1.  This is mostly useful for debugging
     rate-limiting.  (Default: 0)
 
+[[ExtendByEd25519ID]] **ExtendByEd25519ID** **0**|**1**|**auto**::
+    If this option is set to 1, we always try to include a relay's Ed25519 ID
+    when telling the proceeding relay in a circuit to extend to it.
+    If this option is set to 0, we never include Ed25519 IDs when extending
+    circuits.  If the option is set to "default", we obey a
+    parameter in the consensus document. (Default: auto)
+
 CLIENT OPTIONS
 --------------
 
 The following options are useful only for clients (that is, if
 **SocksPort**, **TransPort**, **DNSPort**, or **NATDPort** is non-zero):
 
-[[AllowInvalidNodes]] **AllowInvalidNodes** **entry**|**exit**|**middle**|**introduction**|**rendezvous**|**...**::
-    If some Tor servers are obviously not working right, the directory
-    authorities can manually mark them as invalid, meaning that it's not
-    recommended you use them for entry or exit positions in your circuits. You
-    can opt to use them in some circuit positions, though. The default is
-    "middle,rendezvous", and other choices are not advised.
-
-[[ExcludeSingleHopRelays]] **ExcludeSingleHopRelays** **0**|**1**::
-    This option controls whether circuits built by Tor will include relays with
-    the AllowSingleHopExits flag set to true. If ExcludeSingleHopRelays is set
-    to 0, these relays will be included. Note that these relays might be at
-    higher risk of being seized or observed, so they are not normally
-    included.  Also note that relatively few clients turn off this option,
-    so using these relays might make your client stand out.
-    (Default: 1)
-
 [[Bridge]] **Bridge** [__transport__] __IP__:__ORPort__ [__fingerprint__]::
     When set along with UseBridges, instructs Tor to use the relay at
     "IP:ORPort" as a "bridge" relaying into the Tor network. If "fingerprint"
@@ -753,7 +780,12 @@ The following options are useful only for clients (that is, if
     rather than connecting to the bridge directly. Some transports use a
     transport-specific method to work out the remote address to connect to.
     These transports typically ignore the "IP:ORPort" specified in the bridge
-    line.
+    line.  +
+ +
+    Tor passes any "key=val" settings to the pluggable transport proxy as
+    per-connection arguments when connecting to the bridge.  Consult
+    the documentation of the pluggable transport for details of what
+    arguments it supports.
 
 [[LearnCircuitBuildTimeout]] **LearnCircuitBuildTimeout** **0**|**1**::
     If 0, CircuitBuildTimeout adaptive learning is disabled. (Default: 1)
@@ -766,13 +798,15 @@ The following options are useful only for clients (that is, if
     LearnCircuitBuildTimeout is 0, this value is the only value used.
     (Default: 60 seconds)
 
-[[CircuitIdleTimeout]] **CircuitIdleTimeout** __NUM__::
-    If we have kept a clean (never used) circuit around for NUM seconds, then
-    close it. This way when the Tor client is entirely idle, it can expire all
-    of its circuits, and then expire its TLS connections. Also, if we end up
-    making a circuit that is not useful for exiting any of the requests we're
-    receiving, it won't forever take up a slot in the circuit list. (Default: 1
-    hour)
+[[CircuitsAvailableTimeout]] **CircuitsAvailableTimeout** __NUM__::
+    Tor will attempt to keep at least one open, unused circuit available for
+    this amount of time. This option governs how long idle circuits are kept
+    open, as well as the amount of time Tor will keep a circuit open to each
+    of the recently used ports. This way when the Tor client is entirely
+    idle, it can expire all of its circuits, and then expire its TLS
+    connections. Note that the actual timeout value is uniformly randomized
+    from the specified value to twice that amount. (Default: 30 minutes;
+    Max: 24 hours)
 
 [[CircuitStreamTimeout]] **CircuitStreamTimeout** __NUM__::
     If non-zero, this option overrides our internal timeout schedule for how
@@ -789,6 +823,22 @@ The following options are useful only for clients (that is, if
     and fast enough. The current behavior is simply that Tor is a client
     unless ORPort, ExtORPort, or DirPort are configured.) (Default: 0)
 
+[[ConnectionPadding]] **ConnectionPadding** **0**|**1**|**auto**::
+    This option governs Tor's use of padding to defend against some forms of
+    traffic analysis. If it is set to 'auto', Tor will send padding only
+    if both the client and the relay support it. If it is set to 0, Tor will
+    not send any padding cells. If it is set to 1, Tor will still send padding
+    for client connections regardless of relay support. Only clients may set
+    this option. This option should be offered via the UI to mobile users
+    for use where bandwidth may be expensive.
+    (Default: auto)
+
+[[ReducedConnectionPadding]] **ReducedConnectionPadding** **0**|**1**::
+    If set to 1, Tor will not not hold OR connections open for very long,
+    and will send less padding on these connections. Only clients may set
+    this option. This option should be offered via the UI to mobile users
+    for use where bandwidth may be expensive. (Default: 0)
+
 [[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to avoid when building a circuit. Country codes are
@@ -817,7 +867,7 @@ The following options are useful only for clients (that is, if
 [[ExcludeExitNodes]] **ExcludeExitNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to never use when picking an exit node---that is, a
-    node that delivers traffic for you outside the Tor network.   Note that any
+    node that delivers traffic for you *outside* the Tor network.   Note that any
     node listed in ExcludeNodes is automatically considered to be part of this
     list too.  See
     the **ExcludeNodes** option for more information on how to specify
@@ -834,7 +884,7 @@ The following options are useful only for clients (that is, if
 [[ExitNodes]] **ExitNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to use as exit node---that is, a
-    node that delivers traffic for you outside the Tor network.  See
+    node that delivers traffic for you *outside* the Tor network.  See
     the **ExcludeNodes** option for more information on how to specify nodes. +
  +
     Note that if you list too few nodes here, or if you exclude too many exit
@@ -842,7 +892,7 @@ The following options are useful only for clients (that is, if
     if none of the exits you list allows traffic on port 80 or 443, you won't
     be able to browse the web. +
  +
-    Note also that not every circuit is used to deliver traffic outside of
+    Note also that not every circuit is used to deliver traffic *outside* of
     the Tor network.  It is normal to see non-exit circuits (such as those
     used to connect to hidden services, those that do directory fetches,
     those used for relay reachability self-tests, and so on) that end
@@ -868,16 +918,16 @@ The following options are useful only for clients (that is, if
     the **ExcludeNodes** option for more information on how to specify nodes.
 
 [[StrictNodes]] **StrictNodes** **0**|**1**::
-    If StrictNodes is set to 1, Tor will treat the ExcludeNodes option as a
-    requirement to follow for all the circuits you generate, even if doing so
-    will break functionality for you.  If StrictNodes is set to 0, Tor will
+    If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
+    as a requirement to follow for all the circuits you generate, even if
+    doing so will break functionality for you (StrictNodes applies to neither
+    ExcludeExitNodes nor to ExitNodes).  If StrictNodes is set to 0, Tor will
     still try to avoid nodes in the ExcludeNodes list, but it will err on the
-    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells
-    Tor that it is okay to use an excluded node when it is *necessary* to
-    perform relay reachability self-tests, connect to
-    a hidden service, provide a hidden service to a client, fulfill a .exit
-    request, upload directory information, or download directory information.
-    (Default: 0)
+    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells Tor
+    that it is okay to use an excluded node when it is *necessary* to perform
+    relay reachability self-tests, connect to a hidden service, provide a
+    hidden service to a client, fulfill a .exit request, upload directory
+    information, or download directory information.  (Default: 0)
 
 [[FascistFirewall]] **FascistFirewall** **0**|**1**::
     If 1, Tor will only create outgoing connections to ORs running on ports
@@ -931,24 +981,6 @@ The following options are useful only for clients (that is, if
     services can be configured to require authorization using the
     **HiddenServiceAuthorizeClient** option.
 
-[[CloseHSClientCircuitsImmediatelyOnTimeout]] **CloseHSClientCircuitsImmediatelyOnTimeout** **0**|**1**::
-    If 1, Tor will close unfinished hidden service client circuits
-    which have not moved closer to connecting to their destination
-    hidden service when their internal state has not changed for the
-    duration of the current circuit-build timeout.  Otherwise, such
-    circuits will be left open, in the hope that they will finish
-    connecting to their destination hidden services.  In either case,
-    another set of introduction and rendezvous circuits for the same
-    destination hidden service will be launched. (Default: 0)
-
-[[CloseHSServiceRendCircuitsImmediatelyOnTimeout]] **CloseHSServiceRendCircuitsImmediatelyOnTimeout** **0**|**1**::
-    If 1, Tor will close unfinished hidden-service-side rendezvous
-    circuits after the current circuit-build timeout.  Otherwise, such
-    circuits will be left open, in the hope that they will finish
-    connecting to their destinations.  In either case, another
-    rendezvous circuit for the same destination client will be
-    launched. (Default: 0)
-
 [[LongLivedPorts]] **LongLivedPorts** __PORTS__::
     A list of ports for services that tend to have long-running connections
     (e.g. chat and interactive shells). Circuits for streams that use these
@@ -1007,7 +1039,8 @@ The following options are useful only for clients (that is, if
     but never attach a new stream to a circuit that is too old.  For hidden
     services, this applies to the __last__ time a circuit was used, not the
     first. Circuits with streams constructed with SOCKS authentication via
-    SocksPorts that have **KeepAliveIsolateSOCKSAuth** ignore this value.
+    SocksPorts that have **KeepAliveIsolateSOCKSAuth** also remain alive
+    for MaxCircuitDirtiness seconds after carrying the last such stream.
     (Default: 10 minutes)
 
 [[MaxClientCircuitsPending]] **MaxClientCircuitsPending** __NUM__::
@@ -1069,8 +1102,9 @@ The following options are useful only for clients (that is, if
         Don't share circuits with streams targeting a different
         destination address.
     **KeepAliveIsolateSOCKSAuth**;;
-        If **IsolateSOCKSAuth** is enabled, keep alive circuits that have
-        streams with SOCKS authentication set indefinitely.
+        If **IsolateSOCKSAuth** is enabled, keep alive circuits while they have
+        at least one stream with SOCKS authentication active. After such a circuit
+        is idle for more than MaxCircuitDirtiness seconds, it can be closed.
     **SessionGroup=**__INT__;;
         If no other isolation rules would prevent it, allow streams
         on this port to share circuits with streams from every other
@@ -1142,20 +1176,11 @@ The following options are useful only for clients (that is, if
         authentication" when IsolateSOCKSAuth is disabled, or when this
         option is set.
 
+[[SocksPortFlagsMisc]]::
     Flags are processed left to right. If flags conflict, the last flag on the
     line is used, and all earlier flags are ignored. No error is issued for
     conflicting flags.
 
-[[SocksListenAddress]] **SocksListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for connections from Socks-speaking
-    applications. (Default: 127.0.0.1) You can also specify a port (e.g.
-    192.168.0.1:9100). This directive can be specified multiple times to bind
-    to multiple addresses/ports.  (DEPRECATED: As of 0.2.3.x-alpha, you can
-    now use multiple SocksPort entries, and provide addresses for SocksPort
-    entries, so SocksListenAddress no longer has a purpose.  For backward
-    compatibility, SocksListenAddress is only allowed when SocksPort is just
-    a port number.)
-
 [[SocksPolicy]] **SocksPolicy** __policy__,__policy__,__...__::
     Set an entrance policy for this server, to limit who can connect to the
     SocksPort and DNSPort ports. The policies have the same form as exit
@@ -1172,7 +1197,8 @@ The following options are useful only for clients (that is, if
     NUM must be between 1 and 1000, inclusive.  Note that the configured
     bandwidth limits are still expressed in bytes per second: this
     option only affects the frequency with which Tor checks to see whether
-    previously exhausted connections may read again. (Default: 100 msec)
+    previously exhausted connections may read again.
+    Can not be changed while tor is running. (Default: 100 msec)
 
 [[TrackHostExits]] **TrackHostExits** __host__,__.domain__,__...__::
     For each value in the comma separated list, Tor will track recent
@@ -1209,15 +1235,6 @@ The following options are useful only for clients (that is, if
     Authorities, Single Onion Services, and Tor2web clients. In these cases,
     the this option is ignored. (Default: 1)
 
-[[UseEntryGuardsAsDirGuards]] **UseEntryGuardsAsDirGuards** **0**|**1**::
-    If this option is set to 1, and UseEntryGuards is also set to 1,
-    we try to use our entry guards as directory
-    guards, and failing that, pick more nodes to act as our directory guards.
-    This helps prevent an adversary from enumerating clients. It's only
-    available for clients (non-relay, non-bridge) that aren't configured to
-    download any non-default directory material.  It doesn't currently
-    do anything when we lack a live consensus. (Default: 1)
-
 [[GuardfractionFile]] **GuardfractionFile** __FILENAME__::
     V3 authoritative directories only. Configures the location of the
     guardfraction file which contains information about how long relays
@@ -1231,16 +1248,16 @@ The following options are useful only for clients (that is, if
 
 [[NumEntryGuards]] **NumEntryGuards** __NUM__::
     If UseEntryGuards is set to 1, we will try to pick a total of NUM routers
-    as long-term entries for our circuits. If NUM is 0, we try to learn
-    the number from the NumEntryGuards consensus parameter, and default
-    to 3 if the consensus parameter isn't set. (Default: 0)
+    as long-term entries for our circuits. If NUM is 0, we try to learn the
+    number from the guard-n-primary-guards-to-use consensus parameter, and
+    default to 1 if the consensus parameter isn't set. (Default: 0)
 
 [[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__::
-    If UseEntryGuardsAsDirectoryGuards is enabled, we try to make sure we
-    have at least NUM routers to use as directory guards. If this option
-    is set to 0, use the value from the NumDirectoryGuards consensus
-    parameter, falling back to the value from NumEntryGuards if the
-    consensus parameter is 0 or isn't set. (Default: 0)
+    If UseEntryGuardsAsDirectoryGuards is enabled, we try to make sure we have
+    at least NUM routers to use as directory guards. If this option is set to
+    0, use the value from the guard-n-primary-dir-guards-to-use consensus
+    parameter, and default to 3 if the consensus parameter isn't set.
+    (Default: 0)
 
 [[GuardLifetime]] **GuardLifetime**  __N__ **days**|**weeks**|**months**::
     If nonzero, and UseEntryGuards is set, minimum time to keep a guard before
@@ -1262,12 +1279,6 @@ The following options are useful only for clients (that is, if
     helps to determine whether an application using Tor is possibly leaking
     DNS requests. (Default: 0)
 
-[[WarnUnsafeSocks]] **WarnUnsafeSocks** **0**|**1**::
-    When this option is enabled, Tor will warn whenever a request is
-    received that only contains an IP address instead of a hostname. Allowing
-    applications to do DNS resolves themselves is usually a bad idea and
-    can leak your location to attackers. (Default: 1)
-
 [[VirtualAddrNetworkIPv4]] **VirtualAddrNetworkIPv4** __Address__/__bits__ +
 
 [[VirtualAddrNetworkIPv6]] **VirtualAddrNetworkIPv6** [__Address__]/__bits__::
@@ -1299,18 +1310,6 @@ The following options are useful only for clients (that is, if
     the node "foo". Disabled by default since attacking websites and exit
     relays can use it to manipulate your path selection. (Default: 0)
 
-[[FastFirstHopPK]] **FastFirstHopPK** **0**|**1**|**auto**::
-    When this option is disabled, Tor uses the public key step for the first
-    hop of creating circuits. Skipping it is generally safe since we have
-    already used TLS to authenticate the relay and to establish forward-secure
-    keys. Turning this option off makes circuit building a little
-    slower. Setting this option to "auto" takes advice from the authorities
-    in the latest consensus about whether to use this feature.  +
- +
-    Note that Tor will always use the public key step for the first hop if it's
-    operating as a relay, and it will never use the public key step if it
-    doesn't yet know the onion key of the first hop. (Default: auto)
-
 [[TransPort]] **TransPort**  \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for transparent proxy connections.  Set this to
     0 if you don't want to allow transparent proxy connections.  Set the port
@@ -1321,41 +1320,29 @@ The following options are useful only for clients (that is, if
     TransPort requires OS support for transparent proxies, such as BSDs' pf or
     Linux's IPTables. If you're planning to use Tor as a transparent proxy for
     a network, you'll want to examine and change VirtualAddrNetwork from the
-    default setting. You'll also want to set the TransListenAddress option for
-    the network you'd like to proxy. (Default: 0)
-
-[[TransListenAddress]] **TransListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for transparent proxy connections. (Default:
-    127.0.0.1). This is useful for exporting a transparent proxy server to an
-    entire network. (DEPRECATED: As of 0.2.3.x-alpha, you can
-    now use multiple TransPort entries, and provide addresses for TransPort
-    entries, so TransListenAddress no longer has a purpose.  For backward
-    compatibility, TransListenAddress is only allowed when TransPort is just
-    a port number.)
+    default setting. (Default: 0)
 
 [[TransProxyType]] **TransProxyType** **default**|**TPROXY**|**ipfw**|**pf-divert**::
     TransProxyType may only be enabled when there is transparent proxy listener
-    enabled.
+    enabled. +
  +
     Set this to "TPROXY" if you wish to be able to use the TPROXY Linux module
     to transparently proxy connections that are configured using the TransPort
-    option. This setting lets the listener on the TransPort accept connections
-    for all addresses, even when the TransListenAddress is configured for an
-    internal address. Detailed information on how to configure the TPROXY
+    option. Detailed information on how to configure the TPROXY
     feature can be found in the Linux kernel source tree in the file
-    Documentation/networking/tproxy.txt.
+    Documentation/networking/tproxy.txt. +
  +
-    Set this option to "ipfw" to use the FreeBSD ipfw interface.
+    Set this option to "ipfw" to use the FreeBSD ipfw interface. +
  +
     On *BSD operating systems when using pf, set this to "pf-divert" to take
     advantage of +divert-to+ rules, which do not modify the packets like
     +rdr-to+ rules do. Detailed information on how to configure pf to use
     +divert-to+ rules can be found in the pf.conf(5) manual page. On OpenBSD,
     +divert-to+ is available to use on versions greater than or equal to
-    OpenBSD 4.4.
+    OpenBSD 4.4. +
  +
     Set this to "default", or leave it unconfigured, to use regular IPTables
-    on Linux, or to use pf +rdr-to+ rules on *BSD systems.
+    on Linux, or to use pf +rdr-to+ rules on *BSD systems. +
  +
     (Default: "default".)
 
@@ -1369,13 +1356,6 @@ The following options are useful only for clients (that is, if
  +
     This option is only for people who cannot use TransPort. (Default: 0)
 
-[[NATDListenAddress]] **NATDListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for NATD connections. (DEPRECATED: As of
-    0.2.3.x-alpha, you can now use multiple NATDPort entries, and provide
-    addresses for NATDPort entries, so NATDListenAddress no longer has a
-    purpose.  For backward compatibility, NATDListenAddress is only allowed
-    when NATDPort is just a port number.)
-
 [[AutomapHostsOnResolve]] **AutomapHostsOnResolve** **0**|**1**::
     When this option is enabled, and we get a request to resolve an address
     that ends with one of the suffixes in **AutomapHostsSuffixes**, we map an
@@ -1396,13 +1376,6 @@ The following options are useful only for clients (that is, if
     addresses/ports. See SocksPort for an explanation of isolation
     flags. (Default: 0)
 
-[[DNSListenAddress]] **DNSListenAddress** __IP__[:__PORT__]::
-    Bind to this address to listen for DNS connections. (DEPRECATED: As of
-    0.2.3.x-alpha, you can now use multiple DNSPort entries, and provide
-    addresses for DNSPort entries, so DNSListenAddress no longer has a
-    purpose.  For backward compatibility, DNSListenAddress is only allowed
-    when DNSPort is just a port number.)
-
 [[ClientDNSRejectInternalAddresses]] **ClientDNSRejectInternalAddresses** **0**|**1**::
     If true, Tor does not believe any anonymously retrieved DNS answer that
     tells it that an address resolves to an internal address (like 127.0.0.1 or
@@ -1413,7 +1386,8 @@ The following options are useful only for clients (that is, if
     If true, Tor does not try to fulfill requests to connect to an internal
     address (like 127.0.0.1 or 192.168.0.1) __unless a exit node is
     specifically requested__ (for example, via a .exit hostname, or a
-    controller request).  (Default: 1)
+    controller request).  If true, multicast DNS hostnames for machines on the
+    local network (of the form *.local) are also rejected.  (Default: 1)
 
 [[DownloadExtraInfo]] **DownloadExtraInfo** **0**|**1**::
     If true, Tor downloads and caches "extra-info" documents. These documents
@@ -1431,11 +1405,6 @@ The following options are useful only for clients (that is, if
     Like WarnPlaintextPorts, but instead of warning about risky port uses, Tor
     will instead refuse to make the connection. (Default: None)
 
-[[AllowSingleHopCircuits]] **AllowSingleHopCircuits** **0**|**1**::
-    When this option is set, the attached Tor controller can use relays
-    that have the **AllowSingleHopExits** option turned on to build
-    one-hop Tor connections.  (Default: 0)
-
 [[OptimisticData]] **OptimisticData** **0**|**1**|**auto**::
     When this option is set, and Tor is using an exit node that supports
     the feature, it will try optimistically to send data to the exit node
@@ -1462,11 +1431,11 @@ The following options are useful only for clients (that is, if
     (Example:
     Tor2webRendezvousPoints Fastyfasty, ABCD1234CDEF5678ABCD1234CDEF5678ABCD1234, \{cc}, 255.254.0.0/8) +
  +
-    This feature can only be used if Tor2webMode is also enabled.
+    This feature can only be used if Tor2webMode is also enabled. +
  +
     ExcludeNodes have higher priority than Tor2webRendezvousPoints,
     which means that nodes specified in ExcludeNodes will not be
-    picked as RPs.
+    picked as RPs. +
  +
     If no nodes in Tor2webRendezvousPoints are currently available for
     use, Tor will choose a random node when building HS circuits.
@@ -1494,7 +1463,7 @@ The following options are useful only for clients (that is, if
     These options override the default behavior of Tor's (**currently
     experimental**) path bias detection algorithm. To try to find broken or
     misbehaving guard nodes, Tor looks for nodes where more than a certain
-    fraction of circuits through that guard fail to get built.
+    fraction of circuits through that guard fail to get built. +
  +
     The PathBiasCircThreshold option controls how many circuits we need to build
     through a guard before we make these checks.  The PathBiasNoticeRate,
@@ -1520,14 +1489,14 @@ The following options are useful only for clients (that is, if
 
 [[PathBiasScaleUseThreshold]] **PathBiasScaleUseThreshold** __NUM__::
     Similar to the above options, these options override the default behavior
-    of Tor's (**currently experimental**) path use bias detection algorithm.
+    of Tor's (**currently experimental**) path use bias detection algorithm. +
  +
     Where as the path bias parameters govern thresholds for successfully
     building circuits, these four path use bias parameters govern thresholds
     only for circuit usage. Circuits which receive no stream usage
     are not counted by this detection algorithm. A used circuit is considered
     successful if it is capable of carrying streams or otherwise receiving
-    well-formed responses to RELAY cells.
+    well-formed responses to RELAY cells. +
  +
     By default, or if a negative value is provided for one of these options,
     Tor uses reasonable defaults from the networkstatus consensus document.
@@ -1628,13 +1597,6 @@ is non-zero):
     Tor client binds to.  To bind to a different address, use the
     *ListenAddress and OutboundBindAddress options.
 
-[[AllowSingleHopExits]] **AllowSingleHopExits** **0**|**1**::
-    This option controls whether clients can use this server as a single hop
-    proxy. If set to 1, clients can use this server as an exit even if it is
-    the only hop in the circuit.  Note that most clients will refuse to use
-    servers that set this option, since most clients have
-    ExcludeSingleHopRelays set.  (Default: 0)
-
 [[AssumeReachable]] **AssumeReachable** **0**|**1**::
     This option is used when bootstrapping a new Tor network. If set to 1,
     don't do self-reachability testing; just upload your server descriptor
@@ -1661,7 +1623,7 @@ is non-zero):
     Tells Tor whether to run as an exit relay.  If Tor is running as a
     non-bridge server, and ExitRelay is set to 1, then Tor allows traffic to
     exit according to the ExitPolicy option (or the default ExitPolicy if
-    none is specified).
+    none is specified). +
  +
     If ExitRelay is set to 0, no traffic is allowed to
     exit, and the ExitPolicy option is ignored. +
@@ -1739,6 +1701,7 @@ is non-zero):
        reject *:6881-6999
        accept *:*
 
+[[ExitPolicyDefault]]::
     Since the default exit policy uses accept/reject *, it applies to both
     IPv4 and IPv6 addresses.
 
@@ -1768,14 +1731,18 @@ is non-zero):
     If we have more onionskins queued for processing than we can process in
     this amount of time, reject new ones. (Default: 1750 msec)
 
-[[MyFamily]] **MyFamily** __node__,__node__,__...__::
-    Declare that this Tor server is controlled or administered by a group or
-    organization identical or similar to that of the other servers, defined by
-    their identity fingerprints. When two servers both declare
-    that they are in the same \'family', Tor clients will not use them in the
-    same circuit. (Each server only needs to list the other servers in its
-    family; it doesn't need to list itself, but it won't hurt.) Do not list
-    any bridge relay as it would compromise its concealment.
+[[MyFamily]] **MyFamily** __fingerprint__,__fingerprint__,...::
+    Declare that this Tor relay is controlled or administered by a group or
+    organization identical or similar to that of the other relays, defined by
+    their (possibly $-prefixed) identity fingerprints.
+    This option can be repeated many times, for
+    convenience in defining large families: all fingerprints in all MyFamily
+    lines are merged into one list.
+    When two relays both declare that they are in the
+    same \'family', Tor clients will not use them in the same circuit. (Each
+    relay only needs to list the other servers in its family; it doesn't need to
+    list itself, but it won't hurt if it does.) Do not list any bridge relay as it would
+    compromise its concealment. +
  +
     When listing a node, it's better to list it by fingerprint than by
     nickname: fingerprints are more reliable.
@@ -1793,38 +1760,30 @@ is non-zero):
     Advertise this port to listen for connections from Tor clients and
     servers.  This option is required to be a Tor server.
     Set it to "auto" to have Tor pick a port for you. Set it to 0 to not
-    run an ORPort at all. This option can occur more than once. (Default: 0)
-+
+    run an ORPort at all. This option can occur more than once. (Default: 0) +
+ +
     Tor recognizes these flags on each ORPort:
-    **NoAdvertise**::
+    **NoAdvertise**;;
         By default, we bind to a port and tell our users about it. If
         NoAdvertise is specified, we don't advertise, but listen anyway.  This
         can be useful if the port everybody will be connecting to (for
         example, one that's opened on our firewall) is somewhere else.
-    **NoListen**::
+    **NoListen**;;
         By default, we bind to a port and tell our users about it. If
         NoListen is specified, we don't bind, but advertise anyway.  This
         can be useful if something else  (for example, a firewall's port
         forwarding configuration) is causing connections to reach us.
-    **IPv4Only**::
+    **IPv4Only**;;
         If the address is absent, or resolves to both an IPv4 and an IPv6
         address, only listen to the IPv4 address.
-    **IPv6Only**::
+    **IPv6Only**;;
         If the address is absent, or resolves to both an IPv4 and an IPv6
         address, only listen to the IPv6 address.
-+
+
+[[ORPortFlagsExclusive]]::
     For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and
     IPv4Only and IPv6Only are mutually exclusive.
 
-[[ORListenAddress]] **ORListenAddress** __IP__[:__PORT__]::
-    Bind to this IP address to listen for connections from Tor clients and
-    servers. If you specify a port, bind to this port rather than the one
-    specified in ORPort. (Default: 0.0.0.0) This directive can be specified
-    multiple times to bind to multiple addresses/ports.
-+
-    This option is deprecated; you can get the same behavior with ORPort now
-    that it supports NoAdvertise and explicit addresses.
-
 [[PortForwarding]] **PortForwarding** **0**|**1**::
     Attempt to automatically forward the DirPort and ORPort on a NAT router
     connecting this Tor server to the Internet. If set, Tor will try both
@@ -1840,7 +1799,7 @@ is non-zero):
 [[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**::
     This option specifies which descriptors Tor will publish when acting as
     a relay. You can
-    choose multiple arguments, separated by commas.
+    choose multiple arguments, separated by commas. +
  +
     If this option is set to 0, Tor will not publish its
     descriptors to any directories. (This is useful if you're testing
@@ -1868,7 +1827,7 @@ is non-zero):
     to 0 will disable the heartbeat. Otherwise, it must be at least 30
     minutes. (Default: 6 hours)
 
-[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**|**TBytes**::
+[[AccountingMax]] **AccountingMax** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Limits the max number of bytes sent and received within a set time period
     using a given calculation rule (see: AccountingStart, AccountingRule).
     Useful if you need to stay under a specific bandwidth. By default, the
@@ -1976,12 +1935,6 @@ is non-zero):
 [[GeoIPv6File]] **GeoIPv6File** __filename__::
     A filename containing IPv6 GeoIP data, for use with by-country statistics.
 
-[[TLSECGroup]] **TLSECGroup** **P224**|**P256**::
-    What EC group should we try to use for incoming TLS connections?
-    P224 is faster, but makes us stand out more. Has no effect if
-    we're a client, or if our OpenSSL version lacks support for ECDHE.
-    (Default: P256)
-
 [[CellStatistics]] **CellStatistics** **0**|**1**::
     Relays only.
     When this option is enabled, Tor collects statistics about cell
@@ -1992,6 +1945,14 @@ is non-zero):
     If ExtraInfoStatistics is enabled, it will published as part of
     extra-info document. (Default: 0)
 
+[[PaddingStatistics]] **PaddingStatistics** **0**|**1**::
+    Relays only.
+    When this option is enabled, Tor collects statistics for padding cells
+    sent and received by this relay, in addition to total cell counts.
+    These statistics are rounded, and omitted if traffic is low. This
+    information is important for load balancing decisions related to padding.
+    (Default: 1)
+
 [[DirReqStatistics]] **DirReqStatistics** **0**|**1**::
     Relays and bridges only.
     When this option is enabled, a Tor directory writes statistics on the
@@ -2082,8 +2043,9 @@ is non-zero):
 DIRECTORY SERVER OPTIONS
 ------------------------
 
-The following options are useful only for directory servers (that is,
-if DirPort is non-zero):
+The following options are useful only for directory servers. (Relays with
+enough bandwidth automatically become directory servers; see DirCache for
+details.)
 
 [[DirPortFrontPage]] **DirPortFrontPage** __FILENAME__::
     When this option is set, it takes an HTML file and publishes it as "/" on
@@ -2095,19 +2057,10 @@ if DirPort is non-zero):
     If this option is nonzero, advertise the directory service on this port.
     Set it to "auto" to have Tor pick a port for you.  This option can occur
     more than once, but only one advertised DirPort is supported: all
-    but one DirPort must have the **NoAdvertise** flag set. (Default: 0)
-+
+    but one DirPort must have the **NoAdvertise** flag set. (Default: 0) +
+ +
     The same flags are supported here as are supported by ORPort.
 
-[[DirListenAddress]] **DirListenAddress** __IP__[:__PORT__]::
-    Bind the directory service to this address. If you specify a port, bind to
-    this port rather than the one specified in DirPort.  (Default: 0.0.0.0)
-    This directive can be specified multiple times  to bind to multiple
-    addresses/ports.
-+
-    This option is deprecated; you can get the same behavior with DirPort now
-    that it supports NoAdvertise and explicit addresses.
-
 [[DirPolicy]] **DirPolicy** __policy__,__policy__,__...__::
     Set an entrance policy for this server, to limit who can connect to the
     directory ports. The policies have the same form as exit policies above,
@@ -2120,6 +2073,16 @@ if DirPort is non-zero):
     because clients connect via the ORPort by default. Setting either DirPort
     or BridgeRelay and setting DirCache to 0 is not supported.  (Default: 1)
 
+[[MaxConsensusAgeForDiffs]] **MaxConsensusAgeForDiffs**  __N__ **minutes**|**hours**|**days**|**weeks**::
+    When this option is nonzero, Tor caches will not try to generate
+    consensus diffs for any consensus older than this amount of time.
+    If this option is set to zero, Tor will pick a reasonable default from
+    the current networkstatus document.  You should not set this
+    option unless your cache is severely low on disk space or CPU.
+    If you need to set it, keeping it above 3 or 4 hours will help clients
+    much more than setting it to zero.
+    (Default: 0)
+
 
 DIRECTORY AUTHORITY SERVER OPTIONS
 ----------------------------------
@@ -2199,7 +2162,7 @@ on the public Tor network.
 [[AuthDirBadExit]] **AuthDirBadExit** __AddressPattern...__::
     Authoritative directories only. A set of address patterns for servers that
     will be listed as bad exits in any network status document this authority
-    publishes, if **AuthDirListBadExits** is set.
+    publishes, if **AuthDirListBadExits** is set. +
  +
     (The address pattern syntax here and in the options below
     is the same as for exit policies, except that you don't need to say
@@ -2237,26 +2200,22 @@ on the public Tor network.
     list as acceptable on a single IP address. Set this to "0" for "no limit".
     (Default: 2)
 
-[[AuthDirMaxServersPerAuthAddr]] **AuthDirMaxServersPerAuthAddr** __NUM__::
-    Authoritative directories only. Like AuthDirMaxServersPerAddr, but applies
-    to addresses shared with directory authorities. (Default: 5)
-
-[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Authoritative directories only. If non-zero, always vote the
     Fast flag for any relay advertising this amount of capacity or
     more. (Default: 100 KBytes)
 
-[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[AuthDirGuardBWGuarantee]] **AuthDirGuardBWGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Authoritative directories only. If non-zero, this advertised capacity
     or more is always sufficient to satisfy the bandwidth requirement
-    for the Guard flag. (Default: 250 KBytes)
+    for the Guard flag. (Default: 2 MBytes)
 
 [[AuthDirPinKeys]] **AuthDirPinKeys** **0**|**1**::
     Authoritative directories only. If non-zero, do not allow any relay to
     publish a descriptor if any other relay has reserved its <Ed25519,RSA>
     identity keypair. In all cases, Tor records every keypair it accepts
     in a journal if it is new, or if it differs from the most recently
-    accepted pinning for one of the keys it contains. (Default: 0)
+    accepted pinning for one of the keys it contains. (Default: 1)
 
 [[AuthDirSharedRandomness]] **AuthDirSharedRandomness** **0**|**1**::
     Authoritative directories only. Switch for the shared random protocol.
@@ -2264,6 +2223,13 @@ on the public Tor network.
     (default), the flag "shared-rand-participate" is added to the authority
     vote indicating participation in the protocol. (Default: 1)
 
+[[AuthDirTestEd25519LinkKeys]] **AuthDirTestEd25519LinkKeys**  **0**|**1**::
+    Authoritative directories only. If this option is set to 0, then we treat
+    relays as "Running" if their RSA key is correct when we probe them,
+    regardless of their Ed25519 key. We should only ever set this option to 0
+    if there is some major bug in Ed25519 link authentication that causes us
+    to label all the relays as not Running.  (Default: 1)
+
 [[BridgePassword]] **BridgePassword** __Password__::
     If set, contains an HTTP authenticator that tells a bridge authority to
     serve all requested bridge information. Used by the (only partially
@@ -2335,9 +2301,9 @@ The following options are used to configure a hidden service.
 [[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__::
     Store data files for a hidden service in DIRECTORY. Every hidden service
     must have a separate directory. You may use this option  multiple times to
-    specify multiple services. DIRECTORY must be an existing directory.
+    specify multiple services. If DIRECTORY does not exist, Tor will create it.
     (Note: in current versions of Tor, if DIRECTORY is a relative path,
-    it will be relative to current
+    it will be relative to the current
     working directory of Tor instance, not to its DataDirectory.  Do not
     rely on this behavior; it is not guaranteed to remain the same in future
     versions.)
@@ -2384,8 +2350,8 @@ The following options are used to configure a hidden service.
 
 [[HiddenServiceMaxStreams]] **HiddenServiceMaxStreams** __N__::
    The maximum number of simultaneous streams (connections) per rendezvous
-   circuit. (Setting this to 0 will allow an unlimited number of simultanous
-   streams.) (Default: 0)
+   circuit. The maximum value allowed is 65535. (Setting this to 0 will allow
+   an unlimited number of simultanous streams.) (Default: 0)
 
 [[HiddenServiceMaxStreamsCloseCircuit]] **HiddenServiceMaxStreamsCloseCircuit** **0**|**1**::
    If set to 1, then exceeding **HiddenServiceMaxStreams** will cause the
@@ -2394,8 +2360,9 @@ The following options are used to configure a hidden service.
 
 [[RendPostPeriod]] **RendPostPeriod** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**::
     Every time the specified period elapses, Tor uploads any rendezvous
-    service descriptors to the directory servers. This information  is also
-    uploaded whenever it changes. (Default: 1 hour)
+    service descriptors to the directory servers. This information is also
+    uploaded whenever it changes. Minimum value allowed is 10 minutes and
+    maximum is 3.5 days. (Default: 1 hour)
 
 [[HiddenServiceDirGroupReadable]] **HiddenServiceDirGroupReadable** **0**|**1**::
     If this option is set to 1, allow the filesystem group to read the
@@ -2417,20 +2384,20 @@ The following options are used to configure a hidden service.
     Single Onion Service. One-hop circuits make Single Onion servers easily
     locatable, but clients remain location-anonymous. However, the fact that a
     client is accessing a Single Onion rather than a Hidden Service may be
-    statistically distinguishable.
-
+    statistically distinguishable. +
+ +
     **WARNING:** Once a hidden service directory has been used by a tor
     instance in HiddenServiceSingleHopMode, it can **NEVER** be used again for
     a hidden service. It is best practice to create a new hidden service
     directory, key, and address for each new Single Onion Service and Hidden
     Service. It is not possible to run Single Onion Services and Hidden
     Services from the same tor instance: they should be run on different
-    servers with different IP addresses.
-
+    servers with different IP addresses. +
+ +
     HiddenServiceSingleHopMode requires HiddenServiceNonAnonymousMode to be set
     to 1. Since a Single Onion service is non-anonymous, you can not configure
     a SOCKSPort on a tor instance that is running in
-    **HiddenServiceSingleHopMode**.
+    **HiddenServiceSingleHopMode**. Can not be changed while tor is running.
     (Default: 0)
 
 [[HiddenServiceNonAnonymousMode]] **HiddenServiceNonAnonymousMode** **0**|**1**::
@@ -2438,8 +2405,8 @@ The following options are used to configure a hidden service.
     non-anonymous HiddenServiceSingleHopMode. Enables direct connections in the
     server-side hidden service protocol.  If you are using this option,
     you need to disable all client-side services on your Tor instance,
-    including setting SOCKSPort to "0".
-    (Default: 0)
+    including setting SOCKSPort to "0". Can not be changed while tor is
+    running. (Default: 0)
 
 DENIAL OF SERVICE MITIGATION OPTIONS
 ------------------------------------
@@ -2620,7 +2587,7 @@ The following options are used for running a testing Tor network.
     time. Changing this requires that **TestingTorNetwork** is set. (Default:
     10 minutes)
 
-[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[TestingMinFastFlagThreshold]] **TestingMinFastFlagThreshold** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Minimum value for the Fast flag.  Overrides the ordinary minimum taken
     from the consensus when TestingTorNetwork is set. (Default: 0.)
 
@@ -2679,7 +2646,7 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints, country codes, and
     address patterns of nodes to vote Exit for regardless of their
     uptime, bandwidth, or exit policy. See the **ExcludeNodes**
-    option for more information on how to specify nodes.
+    option for more information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set. See the **ExcludeNodes** option for more
@@ -2688,7 +2655,7 @@ The following options are used for running a testing Tor network.
 [[TestingDirAuthVoteExitIsStrict]] **TestingDirAuthVoteExitIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Exit flag unless it is specified
     in the **TestingDirAuthVoteExit** list, regardless of its uptime, bandwidth,
-    or exit policy.
+    or exit policy. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2697,14 +2664,14 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote Guard for regardless of their
     uptime and bandwidth. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+    information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
 
 [[TestingDirAuthVoteGuardIsStrict]] **TestingDirAuthVoteGuardIsStrict** **0**|**1** ::
     If True (1), a node will never receive the Guard flag unless it is specified
-    in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth.
+    in the **TestingDirAuthVoteGuard** list, regardless of its uptime and bandwidth. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2713,14 +2680,14 @@ The following options are used for running a testing Tor network.
     A list of identity fingerprints and country codes and
     address patterns of nodes to vote HSDir for regardless of their
     uptime and DirPort. See the **ExcludeNodes** option for more
-    information on how to specify nodes.
+    information on how to specify nodes. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     must be set.
 
 [[TestingDirAuthVoteHSDirIsStrict]] **TestingDirAuthVoteHSDirIsStrict** **0**|**1** ::
     If True (1), a node will never receive the HSDir flag unless it is specified
-    in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort.
+    in the **TestingDirAuthVoteHSDir** list, regardless of its uptime and DirPort. +
  +
     In order for this option to have any effect, **TestingTorNetwork**
     has to be set.
@@ -2740,7 +2707,7 @@ The following options are used for running a testing Tor network.
     events.  Changing this requires that **TestingTorNetwork** is set.
     (Default: 0)
 
-[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**::
+[[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold**  __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**::
     Sets a lower-bound for assigning an exit flag when running as an
     authority on a testing network. Overrides the usual default lower bound
     of 4 KB. (Default: 0)
@@ -2764,6 +2731,19 @@ The following options are used for running a testing Tor network.
     we replace it and issue a new key?
     (Default: 3 hours for link and auth; 1 day for signing.)
 
+NON-PERSISTENT OPTIONS
+----------------------
+
+These options are not saved to the torrc file by the "SAVECONF" controller
+command.  Other options of this type are documented in control-spec.txt,
+section 5.4. End-users should mostly ignore them.
+
+[[UnderscorePorts]] **\_\_ControlPort**, **\_\_DirPort**, **\_\_DNSPort**, **\_\_ExtORPort**, **\_\_NATDPort**, **\_\_ORPort**, **\_\_SocksPort**, **\_\_TransPort**::
+    These underscore-prefixed options are variants of the regular Port
+    options.  They behave the same, except they are not saved to the
+    torrc file by the controller's SAVECONF command.
+
+
 SIGNALS
 -------
 
@@ -2813,7 +2793,8 @@ FILES
 __DataDirectory__**/cached-status/**::
     The most recently downloaded network status document for each authority.
     Each file holds one such document; the filenames are the hexadecimal
-    identity key fingerprints of the directory authorities.  Mostly obsolete.
+    identity key fingerprints of the directory authorities.  Obsolete;
+    no longer in use.
 
 __DataDirectory__**/cached-certs**::
     This file holds downloaded directory key certificates that are used to
@@ -2829,6 +2810,13 @@ __DataDirectory__**/cached-descriptors** and **cached-descriptors.new**::
     a given router. The ".new" file is an append-only journal; when it gets
     too large, all entries are merged into a new cached-descriptors file.
 
+__DataDirectory__**/cached-extrainfo** and **cached-extrainfo.new**::
+   As "cached-descriptors", but holds optionally-downloaded "extra-info"
+   documents. Relays use these documents to send inessential information
+   about statistics, bandwidth history, and network health to the
+   authorities. They aren't fetched by default; see the DownloadExtraInfo
+   option for more info.
+
 __DataDirectory__**/cached-microdescs** and **cached-microdescs.new**::
     These files hold downloaded microdescriptors.  Lines beginning with
     @-signs are annotations that contain more information about a given
@@ -2843,18 +2831,27 @@ __DataDirectory__**/state**::
     A set of persistent key-value mappings. These are documented in
     the file. These include:
             - The current entry guards and their status.
-            - The current bandwidth accounting values (unused so far; see
-            below).
+            - The current bandwidth accounting values.
             - When the file was last written
             - What version of Tor generated the state file
             - A short history of bandwidth usage, as produced in the server
             descriptors.
 
+__DataDirectory__**/sr-state**::
+    Authority only. State file used to record information about the current
+    status of the shared-random-value voting state.
+
+__DataDirectory__**/diff-cache**::
+    Directory cache only. Holds older consensuses, and diffs from older
+    consensuses to the most recent consensus of each type, compressed
+    in various ways. Each file contains a set of key-value arguments
+    decribing its contents, followed by a single NUL byte, followed by the
+    main file contents.
+
 __DataDirectory__**/bw_accounting**::
     Used to track bandwidth accounting values (when the current period starts
     and ends; how much has been read and written so far this period). This file
-    is obsolete, and the data is now stored in the \'state' file as well. Only
-    used when bandwidth accounting is enabled.
+    is obsolete, and the data is now stored in the \'state' file instead.
 
 __DataDirectory__**/control_auth_cookie**::
     Used for cookie authentication with the controller. Location can be
@@ -2867,6 +2864,13 @@ __DataDirectory__**/lock**::
     directory. If access to this file is locked, data directory is already
     in use by Tor.
 
+__DataDirectory__**/key-pinning-journal**::
+    Used by authorities. A line-based file that records mappings between
+    RSA1024 identity keys and Ed25519 identity keys. Authorities enforce
+    these mappings, so that once a relay has picked an Ed25519 key, stealing
+    or factoring the RSA1024 key will no longer let an attacker impersonate
+    the relay.
+
 __DataDirectory__**/keys/***::
     Only used by servers. Holds identity keys and onion keys.
 
@@ -2917,13 +2921,17 @@ __DataDirectory__**/keys/ed25519_signing_cert**::
     The certificate which authenticates "ed25519_signing_secret_key" as
     having been signed by the Ed25519 master key.
 
-__DataDirectory__**/keys/secret_onion_key**::
+__DataDirectory__**/keys/secret_onion_key** and **secret_onion_key.old**::
     A relay's RSA1024 short-term onion key. Used to decrypt old-style ("TAP")
-    circuit extension requests.
+    circuit extension requests. The ".old" file holds the previously
+    generated key, which the relay uses to handle any requests that were
+    made by clients that didn't have the new one.
 
-__DataDirectory__**/keys/secret_onion_key_ntor**::
+__DataDirectory__**/keys/secret_onion_key_ntor** and **secret_onion_key_ntor.old**::
     A relay's Curve25519 short-term onion key. Used to handle modern ("ntor")
-    circuit extension requests.
+    circuit extension requests. The ".old" file holds the previously
+    generated key, which the relay uses to handle any requests that were
+    made by clients that didn't have the new one.
 
 __DataDirectory__**/fingerprint**::
     Only used by servers. Holds the fingerprint of the server's identity key.
@@ -2978,11 +2986,20 @@ __DataDirectory__**/stats/conn-stats**::
     Only used by servers. This file is used to collect approximate connection
     history (number of active connections over time).
 
+__DataDirectory__**/stats/hidserv-stats**::
+    Only used by servers. This file is used to collect approximate counts
+    of what fraction of the traffic is hidden service rendezvous traffic, and
+    approximately how many hidden services the relay has seen.
+
 __DataDirectory__**/networkstatus-bridges**::
     Only used by authoritative bridge directories. Contains information
     about bridges that have self-reported themselves to the bridge
     authority.
 
+__DataDirectory__**/approved-routers**::
+    Authorities only. This file is used to configure which relays are
+    known to be valid, invalid, and so forth.
+
 __HiddenServiceDirectory__**/hostname**::
     The <base32-encoded-fingerprint>.onion domain name for this hidden service.
     If the hidden service is restricted to authorized clients only, this file
diff --git a/doc/torify.1.txt b/doc/torify.1.txt
index 8dca901317..7e49081cfc 100644
--- a/doc/torify.1.txt
+++ b/doc/torify.1.txt
@@ -17,25 +17,23 @@ SYNOPSIS
 
 DESCRIPTION
 -----------
-**torify** is a simple wrapper that attempts to find the best underlying Tor
-wrapper available on a system. It calls torsocks with a tor specific
-configuration file. +
+**torify** is a simple wrapper that calls torsocks with a tor-specific
+configuration file.
 
-torsocks is an improved wrapper that explicitly rejects UDP, safely resolves DNS
-lookups and properly socksifies your TCP connections. +
-
-Please note that since both method use LD_PRELOAD, torify cannot be applied to
-suid binaries.
+It is provided for backward compatibility; instead you should use torsocks.
 
 WARNING
 -------
-When used with torsocks, torify should not leak DNS requests or UDP data. +
+When used with torsocks, torify should not leak DNS requests or UDP data.
+
+torify can leak ICMP data.
 
-Both will leak ICMP data.
+torify will not ensure that different requests are processed on
+different circuits.
 
 SEE ALSO
 --------
-**tor**(1), **tor-resolve**(1), **torsocks**(1)
+**tor**(1), **torsocks**(1)
 
 AUTHORS
 -------
diff --git a/doc/torrc_format.txt b/doc/torrc_format.txt
index 7a809901d9..b175b9fb15 100644
--- a/doc/torrc_format.txt
+++ b/doc/torrc_format.txt
@@ -175,7 +175,7 @@ and\
 friends
 
 # Backslashes in the middle of a line are included as-is.  The key of
-# this one is "Too" and the value is "Many\\Backsl\ashes here" (with
+# this one is "Too" and the value is "Many\\Backsl\ashes \here" (with
 # backslashes in that last string as-is)
 Too \
 Many\\\
@@ -185,7 +185,7 @@ here
 # And here's the really yucky part. If a comment appears in a multi-line
 # entry, the entry is still able to continue on the next line, as in the
 # following, where the key is "This" and the value is
-# "entry   and some        are  silly"
+# "entry        and some        are  silly"
 This entry      \
  # has comments \
  and some       \
diff --git a/m4/ax_check_sign.m4 b/m4/ax_check_sign.m4
index 104b17014c..d67e114dba 100644
--- a/m4/ax_check_sign.m4
+++ b/m4/ax_check_sign.m4
@@ -41,8 +41,8 @@ AU_ALIAS([VL_CHECK_SIGN], [AX_CHECK_SIGN])
 AC_DEFUN([AX_CHECK_SIGN], [
  typename=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g"`
  AC_CACHE_CHECK([whether $1 is signed], ax_cv_decl_${typename}_signed, [
-   AC_TRY_COMPILE([$4],
-     [ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ],
+   AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$4]],
+     [[ int foo @<:@ 1 - 2 * !((($1) -1) < 0) @:>@ ]])],
      [ eval "ax_cv_decl_${typename}_signed=\"yes\"" ],
      [ eval "ax_cv_decl_${typename}_signed=\"no\"" ])])
  symbolname=`echo $1 | sed "s/@<:@^a-zA-Z0-9_@:>@/_/g" | tr "a-z" "A-Z"`
diff --git a/m4/pc_from_ucontext.m4 b/m4/pc_from_ucontext.m4
index 8a9dc459e6..9b66bf752c 100644
--- a/m4/pc_from_ucontext.m4
+++ b/m4/pc_from_ucontext.m4
@@ -79,29 +79,29 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      if ! $pc_field_found; then
        # Prefer sys/ucontext.h to ucontext.h, for OS X's sake.
        if test "x$ac_cv_header_cygwin_signal_h" = xyes; then
-         AC_TRY_COMPILE([#include <cygwin/signal.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <cygwin/signal.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_sys_ucontext_h" = xyes; then
-         AC_TRY_COMPILE([#include <sys/ucontext.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/ucontext.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        elif test "x$ac_cv_header_ucontext_h" = xyes; then
-         AC_TRY_COMPILE([#include <ucontext.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ucontext.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
                         pc_field_found=true)
        else     # hope some standard header gives it to us
-         AC_TRY_COMPILE([],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
@@ -114,8 +114,8 @@ AC_DEFUN([AC_PC_FROM_UCONTEXT],
      pc_fields="$pc_fields sc_rip"  # OpenBSD (x86_64)
      for pc_field in $pc_fields; do
        if ! $pc_field_found; then
-         AC_TRY_COMPILE([#include <signal.h>],
-                        [ucontext_t u; return u.$pc_field == 0;],
+         AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <signal.h>]],
+                        [[ucontext_t u; return u.$pc_field == 0;]])],
                         AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field,
                                            How to access the PC from a struct ucontext)
                         AC_MSG_RESULT([$pc_field])
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
new file mode 100755
index 0000000000..6e45c21926
--- /dev/null
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -0,0 +1,157 @@
+#!/usr/bin/python
+
+FUZZERS = """
+	consensus
+	descriptor
+	diff
+	diff-apply
+	extrainfo
+	hsdescv2
+	http
+	iptsv2
+	microdesc
+	vrs
+"""
+
+
+PREAMBLE = r"""
+FUZZING_CPPFLAGS = \
+	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+FUZZING_CFLAGS = \
+	$(AM_CFLAGS) $(TEST_CFLAGS)
+FUZZING_LDFLAG = \
+	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+FUZZING_LIBS = \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIBEVENT_LIBS@ \
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	@TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
+
+oss-fuzz-prereqs: \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a
+
+noinst_HEADERS += \
+	src/test/fuzz/fuzzing.h
+
+LIBFUZZER = -lFuzzer
+LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
+LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
+LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
+
+LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
+"""
+
+POSTAMBLE = r"""
+noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
+noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
+oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
+fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
+
+test-fuzz-corpora: $(FUZZERS)
+	$(top_srcdir)/src/test/fuzz_static_testcases.sh
+"""
+
+########### No user serviceable parts will follow.
+
+PREAMBLE = PREAMBLE.strip()
+POSTAMBLE = POSTAMBLE.strip()  # If I use it, it's a word!
+FUZZERS = FUZZERS.split()
+FUZZERS.sort()
+
+WARNING = """
+# This file was generated by fuzzing_include_am.py; do not hand-edit unless
+# you enjoy having your changes erased.
+""".strip()
+
+print(WARNING)
+
+print(PREAMBLE)
+
+print("\n# ===== AFL fuzzers")
+
+def get_id_name(s):
+    return s.replace("-", "_")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+src_test_fuzz_fuzz_{name}_SOURCES = \\
+	src/test/fuzz/fuzzing_common.c \\
+	src/test/fuzz/fuzz_{name}.c
+src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS)
+""".format(name=idname))
+
+print("FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("\n# ===== libfuzzer")
+print("\nif LIBFUZZER_ENABLED")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+src_test_fuzz_lf_fuzz_{name}_SOURCES = \\
+	$(src_test_fuzz_fuzz_{name}_SOURCES)
+src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS)
+""".format(name=idname))
+
+print("LIBFUZZER_FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/lf-fuzz-{name}".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("""
+else
+LIBFUZZER_FUZZERS =
+endif""")
+
+print("\n# ===== oss-fuzz\n")
+print("if OSS_FUZZ_ENABLED")
+
+for fuzzer in FUZZERS:
+    idname = get_id_name(fuzzer)
+    print("""\
+src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\
+	$(src_test_fuzz_fuzz_{name}_SOURCES)
+src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+""".format(name=idname))
+
+print("OSS_FUZZ_FUZZERS = \\")
+print(" \\\n".join("\tsrc/test/fuzz/liboss-fuzz-{name}.a".format(name=fuzzer)
+                   for fuzzer in FUZZERS))
+
+print("""
+else
+OSS_FUZZ_FUZZERS =
+endif""")
+
+print("")
+
+print(POSTAMBLE)
diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py
index 0dca8a6734..7ea39c540d 100755
--- a/scripts/codegen/gen_server_ciphers.py
+++ b/scripts/codegen/gen_server_ciphers.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc
+# Copyright 2014-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 # This script parses openssl headers to find ciphersuite names, determines
@@ -13,13 +13,13 @@ import sys
 
 EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ]
 BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_",
-              "_SEED_", "_CAMELLIA_", "_NULL" ]
+              "_SEED_", "_CAMELLIA_", "_NULL",
+              "_CCM_8", "_DES_", ]
 
 # these never get #ifdeffed.
 MANDATORY = [
     "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA",
     "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA",
-    "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA",
 ]
 
 def find_ciphers(filename):
@@ -48,15 +48,23 @@ def usable_cipher(ciph):
 # All fields we sort on, in order of priority.
 FIELDS = [ 'cipher', 'fwsec', 'mode',  'digest', 'bitlength' ]
 # Map from sorted fields to recognized value in descending order of goodness
-FIELD_VALS = { 'cipher' : [ 'AES', 'DES'],
+FIELD_VALS = { 'cipher' : [ 'AES', 'CHACHA20' ],
                'fwsec' : [ 'ECDHE', 'DHE' ],
-               'mode' : [ 'GCM', 'CBC' ],
-               'digest' : [ 'SHA384', 'SHA256', 'SHA' ],
+               'mode' : [ 'POLY1305', 'GCM', 'CCM', 'CBC', ],
+               'digest' : [ 'n/a', 'SHA384', 'SHA256', 'SHA', ],
                'bitlength' : [ '256', '128', '192' ],
 }
 
 class Ciphersuite(object):
     def __init__(self, name, fwsec, cipher, bitlength, mode, digest):
+        if fwsec == 'EDH':
+            fwsec = 'DHE'
+
+        if mode in [ '_CBC3', '_CBC', '' ]:
+            mode = 'CBC'
+        elif mode == '_GCM':
+            mode = 'GCM'
+
         self.name = name
         self.fwsec = fwsec
         self.cipher = cipher
@@ -74,42 +82,50 @@ class Ciphersuite(object):
 def parse_cipher(ciph):
     m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph)
 
-    if not m:
-        print "/* Couldn't parse %s ! */"%ciph
-        return None
+    if m:
+        fwsec, cipher, bits, mode, digest = m.groups()
+        return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
-    fwsec, cipher, bits, mode, digest = m.groups()
-    if fwsec == 'EDH':
-        fwsec = 'DHE'
+    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)_CCM', ciph)
+    if m:
+        fwsec, cipher, bits = m.groups()
+        return Ciphersuite(ciph, fwsec, cipher, bits, "CCM", "n/a")
 
-    if mode in [ '_CBC3', '_CBC', '' ]:
-        mode = 'CBC'
-    elif mode == '_GCM':
-        mode = 'GCM'
+    m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_CHACHA20_POLY1305', ciph)
+    if m:
+        fwsec, = m.groups()
+        return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a")
+
+    print "/* Couldn't parse %s ! */"%ciph
+    return None
 
-    return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest)
 
 ALL_CIPHERS = []
 
 for fname in sys.argv[1:]:
-    ALL_CIPHERS += (parse_cipher(c)
-                           for c in find_ciphers(fname)
-                           if usable_cipher(c) )
+    for c in find_ciphers(fname):
+        if usable_cipher(c):
+            parsed = parse_cipher(c)
+            if parsed != None:
+                ALL_CIPHERS.append(parsed)
 
 ALL_CIPHERS.sort(key=Ciphersuite.sort_key)
 
+indent = " "*7
+
 for c in ALL_CIPHERS:
     if c is ALL_CIPHERS[-1]:
-        colon = ';'
+        colon = ''
     else:
         colon = ' ":"'
 
     if c.name in MANDATORY:
-        print "       /* Required */"
-        print '       %s%s'%(c.name,colon)
+        print "%s/* Required */"%indent
+        print '%s%s%s'%(indent,c.name,colon)
     else:
         print "#ifdef %s"%c.name
-        print '       %s%s'%(c.name,colon)
+        print '%s%s%s'%(indent,c.name,colon)
         print "#endif"
 
+print '%s;'%indent
 
diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py
index e673ec7dc6..946957ac77 100644..100755
--- a/scripts/codegen/get_mozilla_ciphers.py
+++ b/scripts/codegen/get_mozilla_ciphers.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # coding=utf-8
-# Copyright 2011-2015, The Tor Project, Inc
+# Copyright 2011-2017, The Tor Project, Inc
 # original version by Arturo Filastò
 # See LICENSE for licensing information
 
@@ -127,9 +127,9 @@ for k, v in enabled_ciphers.items():
 #oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h',
 #               '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h',
 #               '/usr/include/openssl/tls1.h')
-oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h',
-               'ssl/ssl2.h', 'ssl/ssl23.h',
-               'ssl/tls1.h')
+oSSLinclude = ['ssl3.h', 'ssl.h'
+               'ssl2.h', 'ssl23.h',
+               'tls1.h']
 
 #####
 # This reads the hex code for the ciphers that are used by firefox.
@@ -155,9 +155,12 @@ for x in used_ciphers:
 openssl_macro_by_hex = {}
 all_openssl_macros = {}
 for fl in oSSLinclude:
-    fp = open(ossl(fl), 'r')
+    fname = ossl("include/openssl/"+fl)
+    if not os.path.exists(fname):
+        continue
+    fp = open(fname, 'r')
     for line in fp.readlines():
-        m = re.match('#define\s+(\S+)\s+(\S+)', line)
+        m = re.match('# *define\s+(\S+)\s+(\S+)', line)
         if m:
             value,key = m.groups()
             if key.startswith('0x') and "_CK_" in value:
diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py
index d4ba21efae..8d9d4edaaf 100644
--- a/scripts/codegen/makedesc.py
+++ b/scripts/codegen/makedesc.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc.
+# Copyright 2014-2017, The Tor Project, Inc.
 # See LICENSE for license information
 
 # This is a kludgey python script that uses ctypes and openssl to sign
diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh
index d2669931e9..428804342a 100755
--- a/scripts/codegen/run_trunnel.sh
+++ b/scripts/codegen/run_trunnel.sh
@@ -5,7 +5,13 @@ if test "x$TRUNNEL_PATH" != "x"; then
   export PYTHONPATH
 fi
 
-python -m trunnel --require-version=1.4 ./src/trunnel/*.trunnel
+OPTIONS="--require-version=1.5.1"
 
-python -m trunnel --require-version=1.4 --write-c-files --target-dir=./src/ext/trunnel/
+# Get all .trunnel files recursively from that directory so we can support
+# multiple sub-directories.
+for file in `find ./src/trunnel/ -name '*.trunnel'`; do
+  python -m trunnel ${OPTIONS} $file
+done
+
+python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/
 
diff --git a/scripts/maint/analyze_callgraph.py b/scripts/maint/analyze_callgraph.py
deleted file mode 100755
index 8ce5827f07..0000000000
--- a/scripts/maint/analyze_callgraph.py
+++ /dev/null
@@ -1,259 +0,0 @@
-#!/usr/bin/python
-
-import re
-import sys
-import copy
-import cPickle
-import os
-
-class Parser:
-  def __init__(self):
-    self.calls = {}
-    self.definedIn = {}
-
-  def enter_func(self, name):
-    if self.infunc and not self.extern and self.calledfns:
-      if self.infunc in self.definedIn:
-        #print "{}: {} or {}?".format(
-        #  self.infunc, self.definedIn[self.infunc], self.module)
-        self.definedIn[self.infunc] = 'nil'
-      else:
-        self.definedIn[self.infunc] = self.module
-      self.calls.setdefault(self.infunc, set()).update( self.calledfns )
-
-    self.calledfns = set()
-    self.infunc = name
-    self.extern = False
-
-  def parse_callgraph_file(self, inp, module):
-    self.infunc = None
-    self.extern = False
-    self.calledfns = set()
-    self.module = module
-
-    for line in inp:
-       m = re.match(r"Call graph node for function: '([^']+)'", line)
-       if m:
-           self.enter_func(m.group(1))
-           continue
-       m = re.match(r"  CS<[^>]+> calls external node", line)
-       if m:
-           self.extern = True
-       m = re.match(r"  CS<[^>]+> calls function '([^']+)'", line)
-       if m:
-           self.calledfns.add(m.group(1))
-    self.enter_func(None)
-
-  def extract_callgraph(self):
-    c = self.calls
-    self.calls = {}
-    return c
-
-
-def transitive_closure(g):
-    passno = 0
-    changed = True
-    g = copy.deepcopy(g)
-    import random
-    while changed:
-      passno += 1
-      changed = False
-      keys = g.keys()
-      idx = 0
-      for k in keys:
-         idx += 1
-         print "Pass %d/?: %d/%d\r" %(passno, idx, len(keys)),
-         sys.stdout.flush()
-         newset = g[k].copy()
-         for fn in g[k]:
-            newset.update(g.get(fn, set()))
-         if len(newset) != len(g[k]):
-            g[k].update( newset )
-            changed = True
-
-      print
-
-    return g
-
-def strongly_connected_components(g):
-  # From https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm, done stupidly.
-  index_of = {}
-  index = [ 0 ]
-  lowlink = {}
-  S = []
-  onStack = set()
-
-  all_sccs = []
-
-  def strongconnect(fn):
-    index_of[fn] = index[0]
-    lowlink[fn] = index[0]
-    index[0] += 1
-    S.append(fn)
-    onStack.add(fn)
-
-    for w in g.get(fn, []):
-      if w not in index_of:
-        strongconnect(w)
-        lowlink[fn] = min(lowlink[fn], lowlink[w])
-      elif w in onStack:
-        lowlink[fn] = min(lowlink[fn], index_of[w])
-
-    if lowlink[fn] == index_of[fn]:
-      this_scc = []
-      all_sccs.append(this_scc)
-      while True:
-        w = S.pop()
-        onStack.remove(w)
-        this_scc.append(w)
-        if w == fn:
-          break
-
-  for v in g.keys():
-    if v not in index_of:
-      strongconnect(v)
-
-  return all_sccs
-
-def biggest_component(sccs):
-  return max(len(c) for c in sccs)
-
-def connection_bottlenecks(callgraph):
-
-  callers = {}
-  for fn in callgraph:
-    for fn2 in callgraph[fn]:
-      callers.setdefault(fn2, set()).add(fn)
-
-  components = strongly_connected_components(callgraph)
-  components.sort(key=len)
-  big_component_fns = components[-1]
-  size = len(big_component_fns)
-
-  function_bottlenecks = fn_results = []
-
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    idx += 1
-    print "Pass 1/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-    cg2 = copy.deepcopy(callgraph)
-    del cg2[fn]
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn) )
-
-  print
-  bcf_set = set(big_component_fns)
-
-  call_bottlenecks = fn_results = []
-  result_set = set()
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    fn_callers = callers[fn].intersection(bcf_set)
-    idx += 1
-    if len(fn_callers) != 1:
-      continue
-
-    print "Pass 2/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-
-    caller = fn_callers.pop()
-    assert len(fn_callers) == 0
-    cg2 = copy.deepcopy(callgraph)
-    cg2[caller].remove(fn)
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn, "called by", caller) )
-    result_set.add( (caller, fn) )
-
-  print
-
-  total = len(big_component_fns)
-  idx = 0
-  for fn in big_component_fns:
-    fn_calls = callgraph[fn].intersection(bcf_set)
-    idx += 1
-    if len(fn_calls) != 1:
-      continue
-
-    print "Pass 3/3: %d/%d\r"%(idx, total),
-    sys.stdout.flush()
-
-    callee = fn_calls.pop()
-    if (fn, callee) in result_set:
-      continue
-
-    assert len(fn_calls) == 0
-    cg2 = copy.deepcopy(callgraph)
-    cg2[fn].remove(callee)
-
-    fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), callee, "called by", fn) )
-
-  print
-
-
-  return (function_bottlenecks, call_bottlenecks)
-
-if __name__ == '__main__':
-    p = Parser()
-    for fname in sys.argv[1:]:
-      modname = re.sub(r'.*/', '', fname).replace('.callgraph', '.c')
-      with open(fname, 'r') as f:
-        p.parse_callgraph_file(f, modname)
-
-    sys.stdout.flush()
-
-    print "Building callgraph"
-    callgraph = p.extract_callgraph()
-    inModule = p.definedIn
-
-    print "Deriving module callgraph"
-    modCallgraph = {}
-    for fn in callgraph:
-      fnMod = inModule[fn]
-      for called in callgraph[fn]:
-        try:
-          calledMod = inModule[called]
-        except KeyError:
-            continue
-        modCallgraph.setdefault(fnMod, set()).add(calledMod)
-    del modCallgraph['nil']
-
-    print "Finding strongly connected components"
-    sccs = strongly_connected_components(callgraph)
-
-    print "Finding the transitive closure of the callgraph.."
-    closure = transitive_closure(callgraph)
-
-    print "Finding bottlenecks..."
-    bottlenecks = connection_bottlenecks(callgraph)
-
-    print "Finding module SCCs"
-    modSCCS = strongly_connected_components(modCallgraph)
-
-    print "Finding module TC"
-    modTC = transitive_closure(modCallgraph)
-
-    print "Finding module bottlenecks"
-    modB = connection_bottlenecks(modCallgraph)
-
-    data = {
-      'callgraph' : callgraph,
-      'sccs' : sccs,
-      'closure' : closure,
-      'bottlenecks' : bottlenecks,
-      'modules' : p.definedIn,
-      'modItems' : {
-        'callgraph' : modCallgraph,
-        'sccs' : modSCCS,
-        'closure' : modTC,
-        'bottlenecks' : modB,
-      }
-    }
-
-    with open('callgraph.pkl', 'w') as f:
-      cPickle.dump(data, f)
-
-
-
diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl
index 5af95a27e5..3043eb1aaf 100755
--- a/scripts/maint/checkSpace.pl
+++ b/scripts/maint/checkSpace.pl
@@ -1,48 +1,59 @@
-#!/usr/bin/perl -w
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+my $found = 0;
+sub msg {
+  $found = 1;
+  print "$_[0]";
+}
+
+my $C = 0;
 
 if ($ARGV[0] =~ /^-/) {
-    $lang = shift @ARGV;
+    my $lang = shift @ARGV;
     $C = ($lang eq '-C');
-#    $TXT = ($lang eq '-txt');
 }
 
-for $fn (@ARGV) {
+for my $fn (@ARGV) {
     open(F, "$fn");
-    $lastnil = 0;
-    $lastline = "";
-    $incomment = 0;
+    my $lastnil = 0;
+    my $lastline = "";
+    my $incomment = 0;
+    my $in_func_head = 0;
     while (<F>) {
         ## Warn about windows-style newlines.
-	#    (We insist on lines that end with a single LF character, not
-	#    CR LF.)
+        #    (We insist on lines that end with a single LF character, not
+        #    CR LF.)
         if (/\r/) {
-            print "       CR:$fn:$.\n";
+            msg "       CR:$fn:$.\n";
         }
         ## Warn about tabs.
-	#    (We only use spaces)
+        #    (We only use spaces)
         if (/\t/) {
-            print "      TAB:$fn:$.\n";
+            msg "      TAB:$fn:$.\n";
         }
         ## Warn about labels that don't have a space in front of them
-	#    (We indent every label at least one space)
+        #    (We indent every label at least one space)
         if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) {
-            print "nosplabel:$fn:$.\n";
+            msg "nosplabel:$fn:$.\n";
         }
         ## Warn about trailing whitespace.
-	#    (We don't allow whitespace at the end of the line; make your
-	#    editor highlight it for you so you can stop adding it in.)
+        #    (We don't allow whitespace at the end of the line; make your
+        #    editor highlight it for you so you can stop adding it in.)
         if (/ +$/) {
-            print "Space\@EOL:$fn:$.\n";
+            msg "Space\@EOL:$fn:$.\n";
         }
         ## Warn about control keywords without following space.
-	#    (We put a space after every 'if', 'while', 'for', 'switch', etc)
+        #    (We put a space after every 'if', 'while', 'for', 'switch', etc)
         if ($C && /\s(?:if|while|for|switch)\(/) {
-            print "      KW(:$fn:$.\n";
+            msg "      KW(:$fn:$.\n";
         }
         ## Warn about #else #if instead of #elif.
         #    (We only allow #elif)
         if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) {
-            print " #else#if:$fn:$.\n";
+            msg " #else#if:$fn:$.\n";
         }
         ## Warn about some K&R violations
         #    (We use K&R-style C, where open braces go on the same line as
@@ -53,23 +64,23 @@ for $fn (@ARGV) {
         #            other stuff;
         #          }
         if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and
-	    $lastline !~ /\{$/) {
-            print "non-K&R {:$fn:$.\n";
-	}
+            $lastline !~ /\{$/) {
+            msg "non-K&R {:$fn:$.\n";
+        }
         if (/^\s*else/ and $lastline =~ /\}$/) {
-	    print "  }\\nelse:$fn:$.\n";
-	}
+            msg "  }\\nelse:$fn:$.\n";
+        }
         $lastline = $_;
         ## Warn about unnecessary empty lines.
         #   (Don't put an empty line before a line that contains nothing
         #   but a closing brace.)
         if ($lastnil && /^\s*}\n/) {
-            print "  UnnecNL:$fn:$.\n";
+            msg "  UnnecNL:$fn:$.\n";
         }
         ## Warn about multiple empty lines.
         #   (At most one blank line in a row.)
         if ($lastnil && /^$/) {
-            print " DoubleNL:$fn:$.\n";
+            msg " DoubleNL:$fn:$.\n";
         } elsif (/^$/) {
             $lastnil = 1;
         } else {
@@ -79,7 +90,7 @@ for $fn (@ARGV) {
         ## accept double-line lines.
         #   (Don't make lines wider than 80 characters, including newline.)
         if (/^.{80}/) {
-            print "     Wide:$fn:$.\n";
+            msg "     Wide:$fn:$.\n";
         }
         ### Juju to skip over comments and strings, since the tests
         ### we're about to do are okay there.
@@ -102,28 +113,28 @@ for $fn (@ARGV) {
             s!"(?:[^\"]+|\\.)*"!"X"!g;
             next if /^\#/;
             ## Warn about C++-style comments.
-	    #   (Use C style comments only.)
+            #   (Use C style comments only.)
             if (m!//!) {
-                #    print "       //:$fn:$.\n";
+                #    msg "       //:$fn:$.\n";
                 s!//.*!!;
             }
             ## Warn about unquoted braces preceded by non-space.
-	    #   (No character except a space should come before a {)
+            #   (No character except a space should come before a {)
             if (/([^\s'])\{/) {
-                print "       $1\{:$fn:$.\n";
+                msg "       $1\{:$fn:$.\n";
             }
             ## Warn about multiple internal spaces.
             #if (/[^\s,:]\s{2,}[^\s\\=]/) {
-            #    print "     X  X:$fn:$.\n";
+            #    msg "     X  X:$fn:$.\n";
             #}
             ## Warn about { with stuff after.
             #s/\s+$//;
             #if (/\{[^\}\\]+$/) {
-            #    print "     {X:$fn:$.\n";
+            #    msg "     {X:$fn:$.\n";
             #}
             ## Warn about function calls with space before parens.
-	    #   (Don't put a space between the name of a function and its
-	    #   arguments.)
+            #   (Don't put a space between the name of a function and its
+            #   arguments.)
             if (/(\w+)\s\(([A-Z]*)/) {
                 if ($1 ne "if" and $1 ne "while" and $1 ne "for" and
                     $1 ne "switch" and $1 ne "return" and $1 ne "int" and
@@ -131,19 +142,19 @@ for $fn (@ARGV) {
                     $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and
                     $1 ne "size_t" and $1 ne "double" and
                     $1 ne "workqueue_reply_t") {
-                    print "     fn ():$fn:$.\n";
+                    msg "     fn ():$fn:$.\n";
                 }
             }
             ## Warn about functions not declared at start of line.
-	    #    (When you're declaring functions, put "static" and "const"
-	    #    and the return type on one line, and the function name at
-	    #    the start of a new line.)
+            #    (When you're declaring functions, put "static" and "const"
+            #    and the return type on one line, and the function name at
+            #    the start of a new line.)
             if ($in_func_head ||
                 ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ &&
                  ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ &&
                  ! /= *\{$/ && ! /;$/)) {
                 if (/.\{$/){
-                    print "fn() {:$fn:$.\n";
+                    msg "fn() {:$fn:$.\n";
                     $in_func_head = 0;
                 } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) {
                     $in_func_head = -1; # started with tp fn
@@ -151,31 +162,33 @@ for $fn (@ARGV) {
                     $in_func_head = 0;
                 } elsif (/\{/) {
                     if ($in_func_head == -1) {
-                        print "tp fn():$fn:$.\n";
+                        msg "tp fn():$fn:$.\n";
                     }
                     $in_func_head = 0;
                 }
             }
 
-	    ## Check for forbidden functions except when they are
-	    # explicitly permitted
-	    if (/\bassert\(/ && not /assert OK/) {
-		print "assert :$fn:$.   (use tor_assert)\n";
-	    }
-	    if (/\bmemcmp\(/ && not /memcmp OK/) {
-		print "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
-	    }
-	    # always forbidden.
-	    if (not / OVERRIDE /) {
-		if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
-		    print "$& :$fn:$.\n";
-		}
-		if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
-		    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
-		    print "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
-		}
-	    }
+            ## Check for forbidden functions except when they are
+            # explicitly permitted
+            if (/\bassert\(/ && not /assert OK/) {
+                msg "assert :$fn:$.   (use tor_assert)\n";
+            }
+            if (/\bmemcmp\(/ && not /memcmp OK/) {
+                msg "memcmp :$fn:$.   (use {tor,fast}_mem{eq,neq,cmp}\n";
+            }
+            # always forbidden.
+            if (not /\ OVERRIDE\ /) {
+                if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) {
+                    msg "$& :$fn:$.\n";
+                }
+                if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or
+                    /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) {
+                    msg "$& :$fn:$.    (use tor_malloc, tor_free, etc)\n";
+                }
+            }
         }
     }
     close(F);
 }
+
+exit $found;
diff --git a/scripts/maint/display_callgraph.py b/scripts/maint/display_callgraph.py
deleted file mode 100755
index c9001c6d96..0000000000
--- a/scripts/maint/display_callgraph.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/python
-
-import cPickle
-
-data = cPickle.load(open("callgraph.pkl"))
-
-# data = data['modItems']
-
-callgraph = data['callgraph']
-closure = data['closure']
-sccs = data['sccs']
-fn_bottle, call_bottle = data['bottlenecks']
-
-for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems())):
-   print "%s can reach %s other functions." %(fn, n_reachable)
-
-
-c = [ (len(component), component) for component in sccs ]
-c.sort()
-
-print "\n================================"
-
-for n, component in c:
-   if n < 2:
-      continue
-   print "Strongly connected component of size %d:"%n
-   print component
-
-
-print "\n================================"
-
-print "====== Number of functions pulled into blob, by function in blob."
-fn_bottle.sort()
-for n, fn in fn_bottle[-30:]:
-   print "%3d: %s"%(n, fn)
-
-print "====== Number of functions pulled into blob, by call in blob."
-call_bottle.sort()
-for n, fn1, _, fn2 in call_bottle[-30:]:
-   print "%3d: %s -> %s "%(n, fn2, fn1)
-
diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist
index c9fd8a9236..1417a13a98 100644
--- a/scripts/maint/fallback.blacklist
+++ b/scripts/maint/fallback.blacklist
@@ -27,11 +27,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html
 80.82.215.199:80 orport=443 id=3BEFAB76461B6B99DCF34C285E933562F5712AE4 ipv6=[2001:4ba0:cafe:a18::1]:443
 
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008515.html
-# later opt-out in
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008521.html
-5.9.158.75:80 orport=443 id=F1BE15429B3CE696D6807F4D4A58B1BFEC45C822 ipv6=[2a01:4f8:190:514a::2]:443
-
 # Email sent directly to teor, verified using relay contact info
 5.34.183.168:80 orport=443 id=601C92108A568742A7A6D9473FE3A414F7149070
 217.12.199.208:8080 orport=22 id=BCFB0933367D626715DA32A147F417194A5D48D6
@@ -132,7 +127,6 @@
 85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001
 148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001
 62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001
-212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB
 
 # Email sent directly to teor, verified using relay contact info
 74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0
@@ -227,3 +221,54 @@ id=9C8A123081EFBE022EF795630F447839DDFDDDEC
 
 # Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6
 37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
+
+# Fallback was on 0.2.8.6 list, but changed IPv4 before 0.2.9
+195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64
+# Same operator, not on 0.2.8.6 list, also changed IPv4
+51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132
+
+# Fallback was on 0.2.8.6 list, but changed IPv4 before 0.2.9
+5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33
+
+# Fallbacks were on 0.2.8.6 list, but went down before 0.2.9
+194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C
+94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073
+195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197
+176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22
+
+# Fallback was on 0.2.8.6 list, but opted-out before 0.2.9
+144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
+# no dirport
+86.107.110.34:0 orport=9001 id=A0E3D30A660DB70CA0B6D081BA54D094DED6F28D
+94.242.59.147:80 orport=9001 id=674DCBB0D9C1C4C4DBFB4A9AE024AF59FE4E7F46 ipv6=[2a00:1838:35:42::b648]:9001
+
+# Email sent directly to teor, verified using relay contact info
+167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
+
+# Email sent directly to teor, verified using relay contact info
+163.172.35.245:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
+
+# Email sent directly to teor, verified using relay contact info
+104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
+
+# Relay changed IPv4 address, operator uncontactable
+138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
+
+# Emails sent directly to teor, verified using relay contact info
+217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 ipv6=[2a02:27a8:0:2::7e]:443
+
+# Emails sent directly to teor, verified using relay contact info
+195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
+195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
+
+# Email sent directly to teor, verified using relay contact info
+5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
+
+#​https://lists.torproject.org/pipermail/tor-relays/2017-May/012281.html
+62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
+62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
index c801e46b15..0620d6b5fe 100644
--- a/scripts/maint/fallback.whitelist
+++ b/scripts/maint/fallback.whitelist
@@ -37,8 +37,6 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001
-62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101
 212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
 212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
 
@@ -49,18 +47,13 @@
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
 167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
 
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html
-170.130.1.7:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD
-
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
 144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
 # Email sent directly to teor, verified using relay contact info
-91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html
-5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33
+91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F ipv6=[2001:41d0:1:8989::1]:4051
+91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334 ipv6=[2001:41d0:1:8989::1]:4052
 
 # https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
 # Sent additional email to teor with more relays
@@ -98,18 +91,12 @@
 # https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
 178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
 
-# Emails sent directly to teor, verified using relay contact info
-217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05
-
 # Email sent directly to teor, verified using relay contact info
 94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
 
-# https://twitter.com/binarytenshi/status/717952514327453697
-94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073
-
 # Email sent directly to teor, verified using relay contact info
-171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
 171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
+171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
 171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576
 171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
 # OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576
@@ -118,9 +105,9 @@
 #171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
 
 # Email sent directly to teor, verified using relay contact info
-212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68
+212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001
 93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
-46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001
+46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001
 
 # Email sent directly to teor, verified using relay contact info
 85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE
@@ -154,31 +141,19 @@
 178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
 
 # Email sent directly to teor, verified using relay contact info
-195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64
-51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132
 5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
 
 # Email sent directly to teor, verified using relay contact info
-109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
-109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
-109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
-77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
-109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
-77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
-109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
-77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
-109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
+109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52
+109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C
+109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24
+109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18
+109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281
+109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9
 62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
-109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B
-109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722
-109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA
 77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D
-109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6
 77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107
-109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8
 77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69
-109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A
-62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
 
 # https://twitter.com/biotimylated/status/718994247500718080
 212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
@@ -215,9 +190,7 @@
 
 # Email sent directly to teor, verified using relay contact info
 86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
-
-# Email sent directly to teor, verified using relay contact info
-144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907
+86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B
 
 # Email sent directly to teor, verified using relay contact info
 193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
@@ -278,8 +251,8 @@
 
 # Email sent directly to teor, verified using relay contact info
 178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
-188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
-198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050
+188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
+198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050
 159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
 
 # Email sent directly to teor, verified using relay contact info
@@ -301,9 +274,6 @@
 212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
 
 # Email sent directly to teor, verified using relay contact info
-176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22
-
-# Email sent directly to teor, verified using relay contact info
 192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
 192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
 51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A
@@ -318,7 +288,7 @@
 151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6
+5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443
 
 # Email sent directly to teor, verified using relay contact info
 176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
@@ -332,10 +302,7 @@
 212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C
 
 # Email sent directly to teor, verified using relay contact info
-195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197
-
-# Email sent directly to teor, verified using relay contact info
-62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
+212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
 
 # Email sent directly to teor, verified using relay contact info
 37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2
@@ -356,9 +323,6 @@
 37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE
-
-# Email sent directly to teor, verified using relay contact info
 198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
 
 # Email sent directly to teor, verified using relay contact info
@@ -405,12 +369,12 @@
 91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
 
 # Email sent directly to teor, verified using relay contact info
-# Suitable, check with operator before adding
-#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
-212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60
+212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
+# Ok, but on the same machine as 2A4C448784F5A83AFE6C78DA357D5E31F7989DEB
+#212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60
 163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443
-# Suitable, check with operator before adding
-#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942
+# Ok, but on the same machine as AD253B49E303C6AB1E048B014392AC569E8A7DAE
+#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=[2001:bc8:4400:2100::2:1009]:993
 
 # Email sent directly to teor, verified using relay contact info
 46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
@@ -444,9 +408,6 @@
 188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484
-
-# Email sent directly to teor, verified using relay contact info
 46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
 
 # Email sent directly to teor, verified using relay contact info
@@ -456,12 +417,10 @@
 46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
 46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
 46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451
+163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
 
 # Email sent directly to teor, verified using relay contact info
 176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
-195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D
-195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C
 
 # Email sent directly to teor, verified using relay contact info
 46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
@@ -485,11 +444,10 @@
 5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
 
 # Email sent directly to teor, verified using relay contact info
-167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443
-
-# Email sent directly to teor, verified using relay contact info
-192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
-167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+# Updated details from atlas based on ticket #20010
+163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
+163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
+163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2
 
 # Email sent directly to teor, verified using relay contact info
 204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
@@ -498,13 +456,10 @@
 151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
 
 # Email sent directly to teor, verified using relay contact info
-194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C
-
-# Email sent directly to teor, verified using relay contact info
 185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
 
 # Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A
+5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
 
 # Email sent directly to teor, verified using relay contact info
 89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
@@ -513,7 +468,7 @@
 92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
 
 # Email sent directly to teor, verified using relay contact info
-80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001
+80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001
 
 # Email sent directly to teor, verified using relay contact info
 83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
@@ -522,7 +477,7 @@
 95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
 
 # Email sent directly to teor, verified using relay contact info
-176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C
+176.31.191.26:80 orport=443 id=7350AB9ED7568F22745198359373C04AC783C37C
 
 # Email sent directly to teor, verified using relay contact info
 128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
@@ -540,16 +495,17 @@
 80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
 
 # Email sent directly to teor, verified using relay contact info
-185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A
+185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088
 
 # Email sent directly to teor, verified using relay contact info
 149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76
 
 # Email sent directly to teor, verified using relay contact info
-81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243
+81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80
 
 # Email sent directly to teor, verified using relay contact info
-195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993
+# IPv6 address unreliable
+195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993
 138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
 81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
 134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
@@ -563,7 +519,7 @@
 217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
 81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
 46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
-91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993
+91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993
 163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
 188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
 81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
@@ -587,11 +543,10 @@
 91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
 
 # Email sent directly to teor, verified using relay contact info
-# Awaiting confirmation of new ORPort from relay operator
 80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
 
 # Email sent directly to teor, verified using relay contact info
-163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D
+163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D ipv6=[2001:bc8:4400:2100::1:3]:443
 
 # Email sent directly to teor, verified using relay contact info
 97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
@@ -603,10 +558,11 @@
 178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
 
 # Email sent directly to teor, verified using relay contact info
-195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796
+163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6
 
 # Email sent directly to teor, verified using relay contact info
 185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
+185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
 
 # Email sent directly to teor, verified using relay contact info
 164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
@@ -617,10 +573,11 @@
 178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
 
 # Email sent directly to teor, verified using relay contact info
-91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
+# Very low bandwidth, stale consensues, excluded to cut down on warnings
+#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
 
 # Email sent directly to teor, verified using relay contact info
-167.114.113.48:9030 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
+167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
 
 # Email sent directly to teor, verified using relay contact info
 79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6
@@ -675,20 +632,18 @@
 46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
 
 # Email sent directly to teor, verified using relay contact info
-78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257
-
-# Email sent directly to teor, verified using relay contact info
 185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
 
 # Email sent directly to teor, verified using relay contact info
 108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
 
 # Email sent directly to teor, verified using relay contact info
-91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99
+91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6
 
 # Email sent directly to teor, verified using relay contact info
 185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
 192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
+128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D
 
 # Email sent directly to teor, verified using relay contact info
 185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
@@ -719,7 +674,7 @@
 166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
 
 # Emails sent directly to teor, verified using relay contact info
-#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
+69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
 
 # Email sent directly to teor, verified using relay contact info
 213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
@@ -749,7 +704,6 @@
 # Email sent directly to teor, verified using relay contact info
 163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A
 163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4
-163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24
 
 # Email sent directly to teor, verified using relay contact info
 64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2
@@ -768,3 +722,108 @@
 
 # Email sent directly to teor, verified using relay contact info
 62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE
+
+# Email sent directly to teor, verified using relay contact info
+51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63
+
+# Email sent directly to teor, verified using relay contact info
+163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001
+
+# Email sent directly to teor, verified using relay contact info
+5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443
+
+# Email sent directly to teor, verified using relay contact info
+46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001
+
+# Email sent directly to teor, verified using relay contact info
+199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3
+199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6
+199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29
+199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813
+199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338
+199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060
+199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C
+199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616
+199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971
+199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241
+199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0
+199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD
+199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887
+199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B
+199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A
+199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F
+199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2
+199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0
+199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51
+199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
+86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022
+
+# Email sent directly to teor, verified using relay contact info
+178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443
+
+# Email sent directly to teor, verified using relay contact info
+#31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
+# OK, but on same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
+31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D
+#31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1
+#31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148
+
+# Email sent directly to teor, verified using relay contact info
+185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135
+
+# Email sent directly to teor, verified using relay contact info
+94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443
+
+# Email sent directly to teor, verified using relay contact info
+185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001
+
+# Email sent directly to teor, verified using relay contact info
+# The e84 part of the IPv6 address does not have a leading 0 in the consensus
+81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001
+
+# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A
+
+# Email sent directly to teor, verified using relay contact info
+217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001
+
+# Email sent directly to teor, verified using relay contact info
+212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20
+163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24
+163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7
+
+# Email sent directly to teor, verified using relay contact info
+163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438
+195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6
+
+# Email sent directly to teor, verified using relay contact info
+185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
+185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270
+31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F
+
+# Email sent directly to teor, verified using relay contact info
+89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001
+
+# Email sent directly to teor, verified using relay contact info
+95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
+
+# Email sent directly to teor, verified using relay contact info
+85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
+
+# Email sent directly to teor, verified using relay contact info
+72.52.75.27:9030 orport=9001 id=1220F0F20E80D348244C5F3B6D126DAA0A446DFD
+
+# Email sent directly to teor, verified using relay contact info
+5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
+5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
+
+# Email sent directly to teor, verified using relay contact info
+5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
index e909fc550a..c5a0cfc81b 100755
--- a/scripts/maint/format_changelog.py
+++ b/scripts/maint/format_changelog.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2015, The Tor Project, Inc.
+# Copyright (c) 2014-2017, The Tor Project, Inc.
 # See LICENSE for licensing information
 #
 # This script reformats a section of the changelog to wrap everything to
@@ -205,6 +205,8 @@ def head_score(s):
         score = -300
     elif lw.startswith("deprecated version"):
         score = -200
+    elif lw.startswith("directory auth"):
+        score = -150
     elif (('new' in lw and 'requirement' in lw) or
           ('new' in lw and 'dependenc' in lw) or
           ('build' in lw and 'requirement' in lw) or
diff --git a/scripts/maint/generate_callgraph.sh b/scripts/maint/generate_callgraph.sh
deleted file mode 100755
index c6b33c0aea..0000000000
--- a/scripts/maint/generate_callgraph.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-C_FILES=`echo src/common/*.c src/or/*.c src/tools/*.c`
-CFLAGS="-Isrc/ext/trunnel -Isrc/trunnel -I. -Isrc/ext -Isrc/common -DLOCALSTATEDIR=\"\" -DSHARE_DATADIR=\"\" -Dinline="
-
-mkdir -p callgraph/src/common
-mkdir -p callgraph/src/or
-mkdir -p callgraph/src/tools
-
-for fn in $C_FILES; do
-  echo $fn
-  clang $CFLAGS  -S -emit-llvm -fno-inline -o - $fn  | \
-    opt -analyze -print-callgraph >/dev/null 2> "callgraph/${fn}allgraph"
-done
diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py
index b63a4eb3a1..bf06064fa8 100755
--- a/scripts/maint/lintChanges.py
+++ b/scripts/maint/lintChanges.py
@@ -7,7 +7,7 @@ import re
 import os
 
 
-KNOWN_GROUPS=set([
+KNOWN_GROUPS = set([
     "Minor bugfix",
     "Minor bugfixes",
     "Major bugfix",
@@ -22,6 +22,7 @@ KNOWN_GROUPS=set([
     "Code simplification and refactoring",
     "Removed features"])
 
+
 def lintfile(fname):
     have_warned = []
 
@@ -43,17 +44,15 @@ def lintfile(fname):
     if bugnum and bugnum not in contents:
         warn("bug number {} does not appear".format(bugnum))
 
-    lines = contents.split("\n")
-
     m = re.match(r'^[ ]{2}o ([^\(:]*)([^:]*):', contents)
     if not m:
         warn("header not in format expected")
     elif m.group(1).strip() not in KNOWN_GROUPS:
-        warn("Weird header: %r"%m.group(1))
-    elif ( ("bugfix" in m.group(1) or "feature" in m.group(1)) and
-           ("Removed" not in m.group(1)) and
-           '(' not in m.group(2)):
-        warn("Missing subcategory on %s"%m.group(1))
+        warn("Weird header: %r" % m.group(1))
+    elif (("bugfix" in m.group(1) or "feature" in m.group(1)) and
+          ("Removed" not in m.group(1)) and
+          '(' not in m.group(2)):
+        warn("Missing subcategory on %s" % m.group(1))
 
     if m:
         isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower())
@@ -76,6 +75,8 @@ def lintfile(fname):
         elif not re.search('[fF]ixes ([a-z ]*)bug (\d+); bugfix on ',
                            contents):
             warn("bugfix incant is not semicoloned")
+        elif re.search('tor-([0-9]+)', contents):
+            warn("do not prefix versions with 'tor-'")
 
 
 if __name__ == '__main__':
diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py
index 43f5b6eb16..12aed6463a 100755
--- a/scripts/maint/redox.py
+++ b/scripts/maint/redox.py
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 #
-#  Copyright (c) 2008-2015, The Tor Project, Inc.
+#  Copyright (c) 2008-2017, The Tor Project, Inc.
 #  See LICENSE for licensing information.
 #
 # Hi!
diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py
index d6ec0e269d..22e40fd369 100755
--- a/scripts/maint/sortChanges.py
+++ b/scripts/maint/sortChanges.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2014-2015, The Tor Project, Inc.
+# Copyright (c) 2014-2017, The Tor Project, Inc.
 # See LICENSE for licensing information
 
 """This script sorts a bunch of changes files listed on its command
diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl
index 8bd6a18210..beb0b8f26e 100755
--- a/scripts/maint/updateCopyright.pl
+++ b/scripts/maint/updateCopyright.pl
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -i -w -p
 
-$NEWYEAR=2016;
+$NEWYEAR=2017;
 
-s/Copyright(.*) (201[^6]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
+s/Copyright(.*) (201[^7]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
 
 s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/;
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
index 110ecda64c..82a60420b4 100755
--- a/scripts/maint/updateFallbackDirs.py
+++ b/scripts/maint/updateFallbackDirs.py
@@ -1,6 +1,13 @@
 #!/usr/bin/python
 
-# Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc
+# Usage:
+#
+# Regenerate the list:
+# scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc
+#
+# Check the existing list:
+# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok
+# mv fallback_dirs.inc.ok src/or/fallback_dirs.inc
 #
 # This script should be run from a stable, reliable network connection,
 # with no other network activity (and not over tor).
@@ -37,16 +44,13 @@ import dateutil.parser
 # bson_lazy provides bson
 #from bson import json_util
 import copy
+import re
 
-from stem.descriptor.remote import DescriptorDownloader
+from stem.descriptor import DocumentHandler
+from stem.descriptor.remote import get_consensus
 
 import logging
-# INFO tells you why each relay was included or excluded
-# WARN tells you about potential misconfigurations and relay detail changes
-logging.basicConfig(level=logging.WARNING)
 logging.root.name = ''
-# INFO tells you about each consensus download attempt
-logging.getLogger('stem').setLevel(logging.WARNING)
 
 HAVE_IPADDRESS = False
 try:
@@ -80,7 +84,27 @@ PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
 # Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
 PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
 
-# Output fallback name, flags, and ContactInfo in a C comment?
+# Must relays be running now?
+MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
+                       or PERFORM_IPV6_DIRPORT_CHECKS)
+
+# Clients have been using microdesc consensuses by default for a while now
+DOWNLOAD_MICRODESC_CONSENSUS = True
+
+# If a relay delivers an expired consensus, if it expired less than this many
+# seconds ago, we still allow the relay. This should never be less than -90,
+# as all directory mirrors should have downloaded a consensus 90 minutes
+# before it expires. It should never be more than 24 hours, because clients
+# reject consensuses that are older than REASONABLY_LIVE_TIME.
+# For the consensus expiry check to be accurate, the machine running this
+# script needs an accurate clock.
+# We use 24 hours to compensate for #20909, where relays on 0.2.9.5-alpha and
+# 0.3.0.0-alpha-dev and later deliver stale consensuses, but typically recover
+# after ~12 hours.
+# We should make this lower when #20909 is fixed, see #20942.
+CONSENSUS_EXPIRY_TOLERANCE = 24*60*60
+
+# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
 OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
 
 # Output matching ContactInfo in fallbacks list or the blacklist?
@@ -88,6 +112,12 @@ OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
 CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
 CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False
 
+# How the list should be sorted:
+# fingerprint: is useful for stable diffs of fallback lists
+# measured_bandwidth: is useful when pruning the list based on bandwidth
+# contact: is useful for contacting operators once the list has been pruned
+OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
+
 ## OnionOO Settings
 
 ONIONOO = 'https://onionoo.torproject.org/'
@@ -121,22 +151,32 @@ BLACKLIST_EXCLUDES_WHITELIST_ENTRIES = True
 
 WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
 BLACKLIST_FILE_NAME = 'scripts/maint/fallback.blacklist'
+FALLBACK_FILE_NAME  = 'src/or/fallback_dirs.inc'
 
 # The number of bytes we'll read from a filter file before giving up
 MAX_LIST_FILE_SIZE = 1024 * 1024
 
 ## Eligibility Settings
 
-# Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted
-# This causes OnionOO to (correctly) reset its stability timer
-# This issue will be fixed in 0.2.7.7 and 0.2.8.2
-# Until then, the CUTOFFs below ensure a decent level of stability.
-ADDRESS_AND_PORT_STABLE_DAYS = 7
+# Require fallbacks to have the same address and port for a set amount of time
+# We used to have this at 1 week, but that caused many fallback failures, which
+# meant that we had to rebuild the list more often.
+#
+# There was a bug in Tor 0.2.8.1-alpha and earlier where a relay temporarily
+# submits a 0 DirPort when restarted.
+# This causes OnionOO to (correctly) reset its stability timer.
+# Affected relays should upgrade to Tor 0.2.8.7 or later, which has a fix
+# for this issue.
+ADDRESS_AND_PORT_STABLE_DAYS = 30
+# We ignore relays that have been down for more than this period
+MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
 # What time-weighted-fraction of these flags must FallbackDirs
 # Equal or Exceed?
-CUTOFF_RUNNING = .95
-CUTOFF_V2DIR = .95
-CUTOFF_GUARD = .95
+CUTOFF_RUNNING = .90
+CUTOFF_V2DIR = .90
+# Tolerate lower guard flag averages, as guard flags are removed for some time
+# after a relay restarts
+CUTOFF_GUARD = .80
 # What time-weighted-fraction of these flags must FallbackDirs
 # Equal or Fall Under?
 # .00 means no bad exits
@@ -155,12 +195,19 @@ ONIONOO_SCALE_ONE = 999.
 _FB_POG = 0.2
 FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
 
-# We want exactly 100 fallbacks for the initial release
-# This gives us scope to add extra fallbacks to the list as needed
 # Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100
-# Emit a C #error if the number of fallbacks is below
-MIN_FALLBACK_COUNT = 100
+MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
+# Emit a C #error if the number of fallbacks is less than expected
+MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
+
+# The maximum number of fallbacks on the same address, contact, or family
+# With 200 fallbacks, this means each operator can see 1% of client bootstraps
+# (The directory authorities used to see ~12% of client bootstraps each.)
+MAX_FALLBACKS_PER_IP = 1
+MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
+MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
+MAX_FALLBACKS_PER_CONTACT = 3
+MAX_FALLBACKS_PER_FAMILY = 3
 
 ## Fallback Bandwidth Requirements
 
@@ -171,12 +218,12 @@ MIN_FALLBACK_COUNT = 100
 EXIT_BANDWIDTH_FRACTION = 1.0
 
 # If a single fallback's bandwidth is too low, it's pointless adding it
-# We expect fallbacks to handle an extra 30 kilobytes per second of traffic
+# We expect fallbacks to handle an extra 10 kilobytes per second of traffic
 # Make sure they can support a hundred times the expected extra load
-# (Use 102.4 to make it come out nicely in MB/s)
+# (Use 102.4 to make it come out nicely in MByte/s)
 # We convert this to a consensus weight before applying the filter,
 # because all the bandwidth amounts are specified by the relay
-MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0
+MIN_BANDWIDTH = 102.4 * 10.0 * 1024.0
 
 # Clients will time out after 30 seconds trying to download a consensus
 # So allow fallback directories half that to deliver a consensus
@@ -329,6 +376,15 @@ def read_from_file(file_name, max_len):
                  )
   return None
 
+def parse_fallback_file(file_name):
+  file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
+  file_data = cleanse_unprintable(file_data)
+  file_data = remove_bad_chars(file_data, '\n"\0')
+  file_data = re.sub('/\*.*?\*/', '', file_data)
+  file_data = file_data.replace(',', '\n')
+  file_data = file_data.replace(' weight=10', '')
+  return file_data
+
 def load_possibly_compressed_response_json(response):
     if response.info().get('Content-Encoding') == 'gzip':
       buf = StringIO.StringIO( response.read() )
@@ -367,8 +423,8 @@ def onionoo_fetch(what, **kwargs):
   params = kwargs
   params['type'] = 'relay'
   #params['limit'] = 10
-  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS,)
-  params['last_seen_days'] = '-7'
+  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
+  params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
   params['flag'] = 'V2Dir'
   url = ONIONOO + what + '?' + urllib.urlencode(params)
 
@@ -497,6 +553,8 @@ class Candidate(object):
     if (not 'effective_family' in details
         or details['effective_family'] is None):
       details['effective_family'] = []
+    if not 'platform' in details:
+      details['platform'] = None
     details['last_changed_address_or_port'] = parse_ts(
                                       details['last_changed_address_or_port'])
     self._data = details
@@ -511,6 +569,7 @@ class Candidate(object):
     self._compute_ipv6addr()
     if not self.has_ipv6():
       logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
+    self._compute_version()
 
   def _stable_sort_or_addresses(self):
     # replace self._data['or_addresses'] with a stable ordering,
@@ -623,6 +682,59 @@ class Candidate(object):
         self.ipv6orport = int(port)
         return
 
+  def _compute_version(self):
+    # parse the version out of the platform string
+    # The platform looks like: "Tor 0.2.7.6 on Linux"
+    self._data['version'] = None
+    if self._data['platform'] is None:
+      return
+    # be tolerant of weird whitespacing, use a whitespace split
+    tokens = self._data['platform'].split()
+    for token in tokens:
+      vnums = token.split('.')
+      # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
+      if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
+          vnums[2].isdigit()):
+        self._data['version'] = token
+        return
+
+  # From #20509
+  # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
+  # and version 0.3.0.0-alpha-dev
+  # Exhaustive lists are hard to get wrong
+  STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
+                              '0.2.9.2-alpha',
+                              '0.2.9.2-alpha-dev',
+                              '0.2.9.3-alpha',
+                              '0.2.9.3-alpha-dev',
+                              '0.2.9.4-alpha',
+                              '0.2.9.4-alpha-dev',
+                              '0.3.0.0-alpha-dev'
+                              ]
+
+  def is_valid_version(self):
+    # call _compute_version before calling this
+    # is the version of the relay a version we want as a fallback?
+    # checks both recommended versions and bug #20499 / #20509
+    #
+    # if the relay doesn't have a recommended version field, exclude the relay
+    if not self._data.has_key('recommended_version'):
+      log_excluded('%s not a candidate: no recommended_version field',
+                   self._fpr)
+      return False
+    if not self._data['recommended_version']:
+      log_excluded('%s not a candidate: version not recommended', self._fpr)
+      return False
+    # if the relay doesn't have version field, exclude the relay
+    if not self._data.has_key('version'):
+      log_excluded('%s not a candidate: no version field', self._fpr)
+      return False
+    if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
+      logging.warning('%s not a candidate: version delivers stale consensuses',
+                      self._fpr)
+      return False
+    return True
+
   @staticmethod
   def _extract_generic_history(history, which='unknown'):
     # given a tree like this:
@@ -767,41 +879,42 @@ class Candidate(object):
       self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
 
   def is_candidate(self):
-    must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS
-                           or PERFORM_IPV6_DIRPORT_CHECKS)
-    if (must_be_running_now and not self.is_running()):
-      logging.info('%s not a candidate: not running now, unable to check ' +
-                   'DirPort consensus download', self._fpr)
-      return False
-    if (self._data['last_changed_address_or_port'] >
-        self.CUTOFF_ADDRESS_AND_PORT_STABLE):
-      logging.info('%s not a candidate: changed address/port recently (%s)',
-                   self._fpr, self._data['last_changed_address_or_port'])
-      return False
-    if self._running < CUTOFF_RUNNING:
-      logging.info('%s not a candidate: running avg too low (%lf)',
-                   self._fpr, self._running)
-      return False
-    if self._v2dir < CUTOFF_V2DIR:
-      logging.info('%s not a candidate: v2dir avg too low (%lf)',
-                   self._fpr, self._v2dir)
-      return False
-    if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
-      logging.info('%s not a candidate: badexit avg too high (%lf)',
-                   self._fpr, self._badexit)
-      return False
-    # if the relay doesn't report a version, also exclude the relay
-    if (not self._data.has_key('recommended_version')
-        or not self._data['recommended_version']):
-      logging.info('%s not a candidate: version not recommended', self._fpr)
-      return False
-    if self._guard < CUTOFF_GUARD:
-      logging.info('%s not a candidate: guard avg too low (%lf)',
-                   self._fpr, self._guard)
-      return False
-    if (not self._data.has_key('consensus_weight')
-        or self._data['consensus_weight'] < 1):
-      logging.info('%s not a candidate: consensus weight invalid', self._fpr)
+    try:
+      if (MUST_BE_RUNNING_NOW and not self.is_running()):
+        log_excluded('%s not a candidate: not running now, unable to check ' +
+                     'DirPort consensus download', self._fpr)
+        return False
+      if (self._data['last_changed_address_or_port'] >
+          self.CUTOFF_ADDRESS_AND_PORT_STABLE):
+        log_excluded('%s not a candidate: changed address/port recently (%s)',
+                     self._fpr, self._data['last_changed_address_or_port'])
+        return False
+      if self._running < CUTOFF_RUNNING:
+        log_excluded('%s not a candidate: running avg too low (%lf)',
+                     self._fpr, self._running)
+        return False
+      if self._v2dir < CUTOFF_V2DIR:
+        log_excluded('%s not a candidate: v2dir avg too low (%lf)',
+                     self._fpr, self._v2dir)
+        return False
+      if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
+        log_excluded('%s not a candidate: badexit avg too high (%lf)',
+                     self._fpr, self._badexit)
+        return False
+      # this function logs a message depending on which check fails
+      if not self.is_valid_version():
+        return False
+      if self._guard < CUTOFF_GUARD:
+        log_excluded('%s not a candidate: guard avg too low (%lf)',
+                     self._fpr, self._guard)
+        return False
+      if (not self._data.has_key('consensus_weight')
+          or self._data['consensus_weight'] < 1):
+        log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
+        return False
+    except BaseException as e:
+      logging.warning("Exception %s when checking if fallback is a candidate",
+                      str(e))
       return False
     return True
 
@@ -878,26 +991,26 @@ class Candidate(object):
       for key in entry:
         value = entry[key]
         if key == 'id' and value == self._fpr:
-          logging.info('%s is in the blacklist: fingerprint matches',
+          log_excluded('%s is in the blacklist: fingerprint matches',
                        self._fpr)
           return True
         if key == 'ipv4' and value == self.dirip:
           # if the dirport is present, check it too
           if entry.has_key('dirport'):
             if int(entry['dirport']) == self.dirport:
-              logging.info('%s is in the blacklist: IPv4 (%s) and ' +
+              log_excluded('%s is in the blacklist: IPv4 (%s) and ' +
                            'DirPort (%d) match', self._fpr, self.dirip,
                            self.dirport)
               return True
           # if the orport is present, check it too
           elif entry.has_key('orport'):
             if int(entry['orport']) == self.orport:
-              logging.info('%s is in the blacklist: IPv4 (%s) and ' +
+              log_excluded('%s is in the blacklist: IPv4 (%s) and ' +
                            'ORPort (%d) match', self._fpr, self.dirip,
                            self.orport)
               return True
           else:
-            logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' +
+            log_excluded('%s is in the blacklist: IPv4 (%s) matches, and ' +
                          'entry has no DirPort or ORPort', self._fpr,
                          self.dirip)
             return True
@@ -911,19 +1024,19 @@ class Candidate(object):
             # if the dirport is present, check it too
             if entry.has_key('dirport'):
               if int(entry['dirport']) == self.dirport:
-                logging.info('%s is in the blacklist: IPv6 (%s) and ' +
+                log_excluded('%s is in the blacklist: IPv6 (%s) and ' +
                              'DirPort (%d) match', self._fpr, ipv6,
                              self.dirport)
                 return True
             # we've already checked the ORPort, it's part of entry['ipv6']
             else:
-              logging.info('%s is in the blacklist: IPv6 (%s) matches, and' +
+              log_excluded('%s is in the blacklist: IPv6 (%s) matches, and' +
                            'entry has no DirPort', self._fpr, ipv6)
               return True
         elif (key == 'ipv6' or self.has_ipv6()):
           # only log if the fingerprint matches but the IPv6 doesn't
           if entry.has_key('id') and entry['id'] == self._fpr:
-            logging.info('%s skipping IPv6 blacklist comparison: relay ' +
+            log_excluded('%s skipping IPv6 blacklist comparison: relay ' +
                          'has%s IPv6%s, but entry has%s IPv6%s', self._fpr,
                          '' if self.has_ipv6() else ' no',
                          (' (' + ipv6 + ')') if self.has_ipv6() else  '',
@@ -1062,42 +1175,63 @@ class Candidate(object):
         return True
     return False
 
-  # report how long it takes to download a consensus from dirip:dirport
+  # log how long it takes to download a consensus from dirip:dirport
+  # returns True if the download failed, False if it succeeded within max_time
   @staticmethod
-  def fallback_consensus_download_speed(dirip, dirport, nickname, max_time):
+  def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
+                                        max_time):
     download_failed = False
-    downloader = DescriptorDownloader()
-    start = datetime.datetime.utcnow()
     # some directory mirrors respond to requests in ways that hang python
     # sockets, which is why we log this line here
-    logging.info('Initiating consensus download from %s (%s:%d).', nickname,
-                 dirip, dirport)
+    logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
+                 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
+                 nickname, dirip, dirport, fingerprint)
     # there appears to be about 1 second of overhead when comparing stem's
     # internal trace time and the elapsed time calculated here
     TIMEOUT_SLOP = 1.0
+    start = datetime.datetime.utcnow()
     try:
-      downloader.get_consensus(endpoints = [(dirip, dirport)],
-                               timeout = (max_time + TIMEOUT_SLOP),
-                               validate = True,
-                               retries = 0,
-                               fall_back_to_authority = False).run()
+      consensus = get_consensus(
+                              endpoints = [(dirip, dirport)],
+                              timeout = (max_time + TIMEOUT_SLOP),
+                              validate = True,
+                              retries = 0,
+                              fall_back_to_authority = False,
+                              document_handler = DocumentHandler.BARE_DOCUMENT,
+                              microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
+                                ).run()[0]
+      end = datetime.datetime.utcnow()
+      time_since_expiry = (end - consensus.valid_until).total_seconds()
     except Exception, stem_error:
-      logging.info('Unable to retrieve a consensus from %s: %s', nickname,
+      end = datetime.datetime.utcnow()
+      log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
                     stem_error)
       status = 'error: "%s"' % (stem_error)
       level = logging.WARNING
       download_failed = True
-    elapsed = (datetime.datetime.utcnow() - start).total_seconds()
-    if elapsed > max_time:
+    elapsed = (end - start).total_seconds()
+    if download_failed:
+      # keep the error failure status, and avoid using the variables
+      pass
+    elif elapsed > max_time:
       status = 'too slow'
       level = logging.WARNING
       download_failed = True
+    elif (time_since_expiry > 0):
+      status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
+      if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE:
+        status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE)
+        level = logging.INFO
+      else:
+        status += ', invalid'
+        level = logging.WARNING
+        download_failed = True
     else:
       status = 'ok'
       level = logging.DEBUG
-    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' +
+    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
                  'max download time %0.1fs.', elapsed, status, nickname,
-                 dirip, dirport, max_time)
+                 dirip, dirport, fingerprint, max_time)
     return download_failed
 
   # does this fallback download the consensus fast enough?
@@ -1109,12 +1243,14 @@ class Candidate(object):
       ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
                                                 self.dirport,
                                                 self._data['nickname'],
+                                                self._fpr,
                                                 CONSENSUS_DOWNLOAD_SPEED_MAX)
     if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
       # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
       ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
                                                 self.dirport,
                                                 self._data['nickname'],
+                                                self._fpr,
                                                 CONSENSUS_DOWNLOAD_SPEED_MAX)
     return ((not ipv4_failed) and (not ipv6_failed))
 
@@ -1151,6 +1287,7 @@ class Candidate(object):
     # /*
     # nickname
     # flags
+    # adjusted bandwidth, consensus weight
     # [contact]
     # [identical contact counts]
     # */
@@ -1162,6 +1299,13 @@ class Candidate(object):
     s += 'Flags: '
     s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
     s += '\n'
+    # this is an adjusted bandwidth, see calculate_measured_bandwidth()
+    bandwidth = self._data['measured_bandwidth']
+    weight = self._data['consensus_weight']
+    s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
+        bandwidth/(1024.0*1024.0),
+        weight)
+    s += '\n'
     if self._data['contact'] is not None:
       s += cleanse_c_multiline_comment(self._data['contact'])
       if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT:
@@ -1183,6 +1327,7 @@ class Candidate(object):
       s += '\n'
     s += '*/'
     s += '\n'
+    return s
 
   # output the fallback info C string for this fallback
   # this is the text that would go after FallbackDir in a torrc
@@ -1251,7 +1396,8 @@ class CandidateList(dict):
     d = fetch('details',
         fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
                 'consensus_weight,advertised_bandwidth,or_addresses,' +
-                'dir_address,recommended_version,flags,effective_family'))
+                'dir_address,recommended_version,flags,effective_family,' +
+                'platform'))
     logging.debug('Loading details document done.')
 
     if not 'relays' in d: raise Exception("No relays found in document.")
@@ -1297,13 +1443,12 @@ class CandidateList(dict):
     self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
                         reverse=True)
 
-  # sort fallbacks by their fingerprint, lowest to highest
-  # this is useful for stable diffs of fallback lists
-  def sort_fallbacks_by_fingerprint(self):
-    self.fallbacks.sort(key=lambda f: f._fpr)
+  # sort fallbacks by the data field data_field, lowest to highest
+  def sort_fallbacks_by(self, data_field):
+    self.fallbacks.sort(key=lambda f: f._data[data_field])
 
   @staticmethod
-  def load_relaylist(file_name):
+  def load_relaylist(file_obj):
     """ Read each line in the file, and parse it like a FallbackDir line:
         an IPv4 address and optional port:
           <IPv4 address>:<port>
@@ -1318,8 +1463,9 @@ class CandidateList(dict):
         (of string -> string key/value pairs),
         and these dictionaries are placed in an array.
         comments start with # and are ignored """
+    file_data = file_obj['data']
+    file_name = file_obj['name']
     relaylist = []
-    file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
     if file_data is None:
       return relaylist
     for line in file_data.split('\n'):
@@ -1360,12 +1506,12 @@ class CandidateList(dict):
     return relaylist
 
   # apply the fallback whitelist and blacklist
-  def apply_filter_lists(self):
+  def apply_filter_lists(self, whitelist_obj, blacklist_obj):
     excluded_count = 0
     logging.debug('Applying whitelist and blacklist.')
     # parse the whitelist and blacklist
-    whitelist = self.load_relaylist(WHITELIST_FILE_NAME)
-    blacklist = self.load_relaylist(BLACKLIST_FILE_NAME)
+    whitelist = self.load_relaylist(whitelist_obj)
+    blacklist = self.load_relaylist(blacklist_obj)
     filtered_fallbacks = []
     for f in self.fallbacks:
       in_whitelist = f.is_in_whitelist(whitelist)
@@ -1385,7 +1531,7 @@ class CandidateList(dict):
       elif in_blacklist:
         # exclude
         excluded_count += 1
-        logging.info('Excluding %s: in blacklist.', f._fpr)
+        log_excluded('Excluding %s: in blacklist.', f._fpr)
       else:
         if INCLUDE_UNLISTED_ENTRIES:
           # include
@@ -1393,7 +1539,7 @@ class CandidateList(dict):
         else:
           # exclude
           excluded_count += 1
-          logging.info('Excluding %s: in neither blacklist nor whitelist.',
+          log_excluded('Excluding %s: in neither blacklist nor whitelist.',
                        f._fpr)
     self.fallbacks = filtered_fallbacks
     return excluded_count
@@ -1429,8 +1575,8 @@ class CandidateList(dict):
         # the bandwidth we log here is limited by the relay's consensus weight
         # as well as its adverttised bandwidth. See set_measured_bandwidth
         # for details
-        logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' +
-                     'be at least %.1fMB/s', f._fpr,
+        log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
+                     'must be at least %.1fMByte/s', f._fpr,
                      f._data['measured_bandwidth']/(1024.0*1024.0),
                      MIN_BANDWIDTH/(1024.0*1024.0))
     self.fallbacks = above_min_bw_fallbacks
@@ -1470,49 +1616,85 @@ class CandidateList(dict):
     else:
       return None
 
-  # does exclusion_list contain attribute?
+  # return a new bag suitable for storing attributes
+  @staticmethod
+  def attribute_new():
+    return dict()
+
+  # get the count of attribute in attribute_bag
+  # if attribute is None or the empty string, return 0
+  @staticmethod
+  def attribute_count(attribute, attribute_bag):
+    if attribute is None or attribute == '':
+      return 0
+    if attribute not in attribute_bag:
+      return 0
+    return attribute_bag[attribute]
+
+  # does attribute_bag contain more than max_count instances of attribute?
   # if so, return False
   # if not, return True
-  # if attribute is None or the empty string, always return True
+  # if attribute is None or the empty string, or max_count is invalid,
+  # always return True
   @staticmethod
-  def allow(attribute, exclusion_list):
-    if attribute is None or attribute == '':
+  def attribute_allow(attribute, attribute_bag, max_count=1):
+    if attribute is None or attribute == '' or max_count <= 0:
       return True
-    elif attribute in exclusion_list:
+    elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
       return False
     else:
       return True
 
-  # make sure there is only one fallback per IPv4 address, and per IPv6 address
+  # add attribute to attribute_bag, incrementing the count if it is already
+  # present
+  # if attribute is None or the empty string, or count is invalid,
+  # do nothing
+  @staticmethod
+  def attribute_add(attribute, attribute_bag, count=1):
+    if attribute is None or attribute == '' or count <= 0:
+      pass
+    attribute_bag.setdefault(attribute, 0)
+    attribute_bag[attribute] += count
+
+  # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
+  # and per IPv6 address
   # there is only one IPv4 address on each fallback: the IPv4 DirPort address
   # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
   # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
   # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
-  # (clients assume the IPv6 DirPort is the same as the IPv4 DirPort, but
-  # typically only use the IPv6 ORPort)
+  # (clients only use the IPv6 ORPort)
   # if there is no IPv6 address, only the IPv4 address is checked
   # return the number of candidates we excluded
   def limit_fallbacks_same_ip(self):
     ip_limit_fallbacks = []
-    ip_list = []
+    ip_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if (CandidateList.allow(f.dirip, ip_list)
-          and CandidateList.allow(f.ipv6addr, ip_list)):
+      if (CandidateList.attribute_allow(f.dirip, ip_list,
+                                        MAX_FALLBACKS_PER_IPV4)
+          and CandidateList.attribute_allow(f.ipv6addr, ip_list,
+                                            MAX_FALLBACKS_PER_IPV6)):
         ip_limit_fallbacks.append(f)
-        ip_list.append(f.dirip)
+        CandidateList.attribute_add(f.dirip, ip_list)
         if f.has_ipv6():
-          ip_list.append(f.ipv6addr)
-      elif not CandidateList.allow(f.dirip, ip_list):
-        logging.info('Eliminated %s: already have fallback on IPv4 %s'%(
-                                                          f._fpr, f.dirip))
-      elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list):
-        logging.info('Eliminated %s: already have fallback on IPv6 %s'%(
-                                                          f._fpr, f.ipv6addr))
+          CandidateList.attribute_add(f.ipv6addr, ip_list)
+      elif not CandidateList.attribute_allow(f.dirip, ip_list,
+                                             MAX_FALLBACKS_PER_IPV4):
+        log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
+                     %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
+                       f.dirip))
+      elif (f.has_ipv6() and
+            not CandidateList.attribute_allow(f.ipv6addr, ip_list,
+                                              MAX_FALLBACKS_PER_IPV6)):
+        log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
+                     %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
+                                                             ip_list),
+                       f.ipv6addr))
     original_count = len(self.fallbacks)
     self.fallbacks = ip_limit_fallbacks
     return original_count - len(self.fallbacks)
 
-  # make sure there is only one fallback per ContactInfo
+  # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
+  # ContactInfo
   # if there is no ContactInfo, allow the fallback
   # this check can be gamed by providing no ContactInfo, or by setting the
   # ContactInfo to match another fallback
@@ -1520,37 +1702,45 @@ class CandidateList(dict):
   # go down at similar times, its usefulness outweighs the risk
   def limit_fallbacks_same_contact(self):
     contact_limit_fallbacks = []
-    contact_list = []
+    contact_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if CandidateList.allow(f._data['contact'], contact_list):
+      if CandidateList.attribute_allow(f._data['contact'], contact_list,
+                                       MAX_FALLBACKS_PER_CONTACT):
         contact_limit_fallbacks.append(f)
-        contact_list.append(f._data['contact'])
+        CandidateList.attribute_add(f._data['contact'], contact_list)
       else:
-        logging.info(('Eliminated %s: already have fallback on ' +
-                       'ContactInfo %s')%(f._fpr, f._data['contact']))
+        log_excluded(
+          'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
+          %(f._fpr, CandidateList.attribute_count(f._data['contact'],
+                                                  contact_list),
+            f._data['contact']))
     original_count = len(self.fallbacks)
     self.fallbacks = contact_limit_fallbacks
     return original_count - len(self.fallbacks)
 
-  # make sure there is only one fallback per effective family
+  # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
+  # family
   # if there is no family, allow the fallback
-  # this check can't be gamed, because we use effective family, which ensures
-  # mutual family declarations
+  # we use effective family, which ensures mutual family declarations
+  # but the check can be gamed by not declaring a family at all
   # if any indirect families exist, the result depends on the order in which
   # fallbacks are sorted in the list
   def limit_fallbacks_same_family(self):
     family_limit_fallbacks = []
-    fingerprint_list = []
+    fingerprint_list = CandidateList.attribute_new()
     for f in self.fallbacks:
-      if CandidateList.allow(f._fpr, fingerprint_list):
+      if CandidateList.attribute_allow(f._fpr, fingerprint_list,
+                                       MAX_FALLBACKS_PER_FAMILY):
         family_limit_fallbacks.append(f)
-        fingerprint_list.append(f._fpr)
-        fingerprint_list.extend(f._data['effective_family'])
+        CandidateList.attribute_add(f._fpr, fingerprint_list)
+        for family_fingerprint in f._data['effective_family']:
+          CandidateList.attribute_add(family_fingerprint, fingerprint_list)
       else:
-        # technically, we already have a fallback with this fallback in its
-        # effective family
-        logging.info('Eliminated %s: already have fallback in effective ' +
-                      'family'%(f._fpr))
+        # we already have a fallback with this fallback in its effective
+        # family
+        log_excluded(
+          'Eliminated %s: already have %d fallback(s) in effective family'
+          %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
     original_count = len(self.fallbacks)
     self.fallbacks = family_limit_fallbacks
     return original_count - len(self.fallbacks)
@@ -1878,8 +2068,8 @@ class CandidateList(dict):
     min_bw = min_fb._data['measured_bandwidth']
     max_fb = self.fallback_max()
     max_bw = max_fb._data['measured_bandwidth']
-    s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0),
-                                              max_bw/(1024.0*1024.0))
+    s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
+                                                 max_bw/(1024.0*1024.0))
     s += '\n'
     s += '*/'
     if fallback_count < MIN_FALLBACK_COUNT:
@@ -1892,9 +2082,44 @@ class CandidateList(dict):
       s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
     return s
 
+def process_existing():
+  logging.basicConfig(level=logging.INFO)
+  logging.getLogger('stem').setLevel(logging.INFO)
+  whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
+               'name': FALLBACK_FILE_NAME}
+  blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': BLACKLIST_FILE_NAME}
+  list_fallbacks(whitelist, blacklist)
+
+def process_default():
+  logging.basicConfig(level=logging.WARNING)
+  logging.getLogger('stem').setLevel(logging.WARNING)
+  whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': WHITELIST_FILE_NAME}
+  blacklist = {'data': read_from_file(BLACKLIST_FILE_NAME, MAX_LIST_FILE_SIZE),
+               'name': BLACKLIST_FILE_NAME}
+  list_fallbacks(whitelist, blacklist)
+
 ## Main Function
+def main():
+  if get_command() == 'check_existing':
+    process_existing()
+  else:
+    process_default()
+
+def get_command():
+  if len(sys.argv) == 2:
+    return sys.argv[1]
+  else:
+    return None
 
-def list_fallbacks():
+def log_excluded(msg, *args):
+  if get_command() == 'check_existing':
+    logging.warning(msg, *args)
+  else:
+    logging.info(msg, *args)
+
+def list_fallbacks(whitelist, blacklist):
   """ Fetches required onionoo documents and evaluates the
       fallback directory criteria for each of the relays """
 
@@ -1927,7 +2152,7 @@ def list_fallbacks():
   # warning that the details have changed from those in the whitelist.
   # instead, there will be an info-level log during the eligibility check.
   initial_count = len(candidates.fallbacks)
-  excluded_count = candidates.apply_filter_lists()
+  excluded_count = candidates.apply_filter_lists(whitelist, blacklist)
   print candidates.summarise_filters(initial_count, excluded_count)
   eligible_count = len(candidates.fallbacks)
 
@@ -1985,15 +2210,17 @@ def list_fallbacks():
   for s in fetch_source_list():
     print describe_fetch_source(s)
 
+  # sort the list differently depending on why we've created it:
   # if we're outputting the final fallback list, sort by fingerprint
   # this makes diffs much more stable
-  # otherwise, leave sorted by bandwidth, which allows operators to be
-  # contacted in priority order
-  if not OUTPUT_CANDIDATES:
-    candidates.sort_fallbacks_by_fingerprint()
+  # otherwise, if we're trying to find a bandwidth cutoff, or we want to
+  # contact operators in priority order, sort by bandwidth (not yet
+  # implemented)
+  # otherwise, if we're contacting operators, sort by contact
+  candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
 
   for x in candidates.fallbacks:
     print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
 
 if __name__ == "__main__":
-  list_fallbacks()
+  main()
diff --git a/scripts/test/cov-exclude b/scripts/test/cov-exclude
index 5117f11ec4..5cb9b1282d 100755
--- a/scripts/test/cov-exclude
+++ b/scripts/test/cov-exclude
@@ -26,3 +26,9 @@ if ($excluding or $exclude_this) {
   s{^\s*\#\#+:}{        x:};
   s{^   (\s*)(\d+):}{$1!!!$2:};
 }
+
+if (eof and $excluding) {
+    warn "Runaway LCOV_EXCL_START in $ARGV";
+    $excluding = 0;
+}
+
diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake
index b8c5dd4fea..a1c819fffa 100644
--- a/src/common/Makefile.nmake
+++ b/src/common/Makefile.nmake
@@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \
 	log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \
 	util_codedigest.obj
 
-LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \
-	crypto_curve25519.obj curve25519-donna.obj
+LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \
+	tortls.obj crypto_curve25519.obj curve25519-donna.obj
 
 LIBOR_EVENT_OBJECTS = compat_libevent.obj
 
diff --git a/src/common/address.c b/src/common/address.c
index 96b99fa082..555f63da06 100644
--- a/src/common/address.c
+++ b/src/common/address.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -159,6 +159,8 @@ tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa,
   tor_assert(a);
   tor_assert(sa);
 
+  /* This memset is redundant; leaving it in to avoid any future accidents,
+     however. */
   memset(a, 0, sizeof(*a));
 
   if (sa->sa_family == AF_INET) {
@@ -235,8 +237,8 @@ tor_addr_make_null(tor_addr_t *a, sa_family_t family)
  *
  * Return 0 on success, -1 on failure; 1 on transient failure.
  */
-int
-tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr)
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
 {
   /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
    * something.
@@ -562,8 +564,8 @@ tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
 
 /** Convert <b>addr</b> to an in-addr.arpa name or a .ip6.arpa name,
  * and store the result in the <b>outlen</b>-byte buffer at
- * <b>out</b>.  Return the number of chars written to <b>out</b>, not
- * including the trailing \0, on success. Returns -1 on failure. */
+ * <b>out</b>.  Returns a non-negative integer on success.
+ * Returns -1 on failure. */
 int
 tor_addr_to_PTR_name(char *out, size_t outlen,
                      const tor_addr_t *addr)
@@ -1804,9 +1806,10 @@ free_interface_address6_list(smartlist_t *addrs)
  * Returns NULL on failure.
  * Use free_interface_address6_list to free the returned list.
  */
-MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity,
-                                                     sa_family_t family,
-                                                     int include_internal))
+MOCK_IMPL(smartlist_t *,
+get_interface_address6_list,(int severity,
+                             sa_family_t family,
+                             int include_internal))
 {
   smartlist_t *addrs;
   tor_addr_t addr;
@@ -2074,7 +2077,8 @@ parse_port_range(const char *port, uint16_t *port_min_out,
 
 /** Given an IPv4 in_addr struct *<b>in</b> (in network order, as usual),
  *  write it as a string into the <b>buf_len</b>-byte buffer in
- *  <b>buf</b>.
+ *  <b>buf</b>. Returns a non-negative integer on success.
+ *  Returns -1 on failure.
  */
 int
 tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len)
@@ -2125,7 +2129,8 @@ get_interface_address,(int severity, uint32_t *addr))
 }
 
 /** Return true if we can tell that <b>name</b> is a canonical name for the
- * loopback address. */
+ * loopback address.  Return true also for *.local hostnames, which are
+ * multicast DNS names for hosts on the local network. */
 int
 tor_addr_hostname_is_local(const char *name)
 {
@@ -2146,3 +2151,11 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port)
   return ap;
 }
 
+/** Return true iff <a>a</b> and <b>b</b> are the same address and port */
+int
+tor_addr_port_eq(const tor_addr_port_t *a,
+                 const tor_addr_port_t *b)
+{
+  return tor_addr_eq(&a->addr, &b->addr) && a->port == b->port;
+}
+
diff --git a/src/common/address.h b/src/common/address.h
index d57abd0d9e..8091f8cd7b 100644
--- a/src/common/address.h
+++ b/src/common/address.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -190,7 +190,8 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u)
  */
 #define TOR_ADDR_BUF_LEN 48
 
-int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out);
+MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
+                                tor_addr_t *addr_out));
 char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC;
 
 /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6
@@ -344,6 +345,8 @@ get_interface_address_list(int severity, int include_internal)
 }
 
 tor_addr_port_t *tor_addr_port_new(const tor_addr_t *addr, uint16_t port);
+int tor_addr_port_eq(const tor_addr_port_t *a,
+                     const tor_addr_port_t *b);
 
 #ifdef ADDRESS_PRIVATE
 MOCK_DECL(smartlist_t *,get_interface_addresses_raw,(int severity,
diff --git a/src/common/aes.c b/src/common/aes.c
index 35c2d1e3a5..73abef143e 100644
--- a/src/common/aes.c
+++ b/src/common/aes.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/aes.h b/src/common/aes.h
index 1cda53f2fa..6fd9c3ea16 100644
--- a/src/common/aes.h
+++ b/src/common/aes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Implements a minimal interface to counter-mode AES. */
diff --git a/src/common/backtrace.c b/src/common/backtrace.c
index 81e04e94eb..0f0fa857bf 100644
--- a/src/common/backtrace.c
+++ b/src/common/backtrace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -76,7 +76,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
 #ifdef PC_FROM_UCONTEXT
 #if defined(__linux__)
   const size_t n = 1;
-#elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \
+#elif defined(__darwin__) || defined(__APPLE__) || defined(OpenBSD) \
   || defined(__FreeBSD__)
   const size_t n = 2;
 #else
diff --git a/src/common/backtrace.h b/src/common/backtrace.h
index b53fd2c668..8cd1dcf06c 100644
--- a/src/common/backtrace.h
+++ b/src/common/backtrace.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BACKTRACE_H
diff --git a/src/common/ciphers.inc b/src/common/ciphers.inc
index 23f5fd2da4..0084b3e325 100644
--- a/src/common/ciphers.inc
+++ b/src/common/ciphers.inc
@@ -33,6 +33,26 @@
 #else
    XCIPHER(0xc02f, TLS1_TXT_ECDHE_RSA_WITH_AES_128_GCM_SHA256)
 #endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+    CIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
+#else
+   XCIPHER(0xcca9, TLS1_TXT_ECDHE_ECDSA_WITH_CHACHA20_POLY1305)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
+    CIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
+#else
+   XCIPHER(0xcca8, TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305)
+#endif
+#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+    CIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+#else
+   XCIPHER(0xc02c, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384)
+#endif
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+    CIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+#else
+   XCIPHER(0xc030, TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384)
+#endif
 #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
     CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA)
 #else
@@ -53,88 +73,28 @@
 #else
    XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA)
 #endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA
-    CIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
-#else
-   XCIPHER(0xc012, TLS1_TXT_ECDHE_RSA_WITH_DES_192_CBC3_SHA)
-#endif
-#ifdef TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA
-    CIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
-#else
-   XCIPHER(0xc007, TLS1_TXT_ECDHE_ECDSA_WITH_RC4_128_SHA)
-#endif
-#ifdef TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA
-    CIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
-#else
-   XCIPHER(0xc011, TLS1_TXT_ECDHE_RSA_WITH_RC4_128_SHA)
-#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
     CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #endif
-#ifdef TLS1_TXT_DHE_DSS_WITH_AES_128_SHA
-    CIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
-#else
-   XCIPHER(0x0032, TLS1_TXT_DHE_DSS_WITH_AES_128_SHA)
-#endif
-#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA
-    CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#else
-   XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA
     CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #endif
-#ifdef TLS1_TXT_DHE_DSS_WITH_AES_256_SHA
-    CIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
-#else
-   XCIPHER(0x0038, TLS1_TXT_DHE_DSS_WITH_AES_256_SHA)
-#endif
-#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA
-    CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#else
-   XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#endif
-#ifdef SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA
-    CIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
-#else
-   XCIPHER(0x0016, SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA)
-#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_128_SHA
     CIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #else
    XCIPHER(0x002f, TLS1_TXT_RSA_WITH_AES_128_SHA)
 #endif
-#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA
-    CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#else
-   XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
-#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA
     CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #else
    XCIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #endif
-#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA
-    CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#else
-   XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
-#endif
 #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA
     CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #else
    XCIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)
 #endif
-#ifdef SSL3_TXT_RSA_RC4_128_SHA
-    CIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
-#else
-   XCIPHER(0x0005, SSL3_TXT_RSA_RC4_128_SHA)
-#endif
-#ifdef SSL3_TXT_RSA_RC4_128_MD5
-    CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
-#else
-   XCIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
-#endif
diff --git a/src/common/compat.c b/src/common/compat.c
index 4ac443c134..4a1f0013c6 100644
--- a/src/common/compat.c
+++ b/src/common/compat.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -204,7 +204,15 @@ tor_rename(const char *path_old, const char *path_new)
                 sandbox_intern_string(path_new));
 }
 
-#if defined(HAVE_SYS_MMAN_H) || defined(RUNNING_DOXYGEN)
+/* Some MinGW builds have sys/mman.h, but not the corresponding symbols.
+ * Other configs rename the symbols using macros (including getpagesize).
+ * So check for sys/mman.h and unistd.h, and a getpagesize declaration. */
+#if (defined(HAVE_SYS_MMAN_H) && defined(HAVE_UNISTD_H) && \
+     defined(HAVE_DECL_GETPAGESIZE))
+#define COMPAT_HAS_MMAN_AND_PAGESIZE
+#endif
+
+#if defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || defined(RUNNING_DOXYGEN)
 /** Try to create a memory mapping for <b>filename</b> and return it.  On
  * failure, return NULL.  Sets errno properly, using ERANGE to mean
  * "empty file". */
@@ -250,6 +258,12 @@ tor_mmap_file(const char *filename)
   page_size = getpagesize();
   size += (size%page_size) ? page_size-(size%page_size) : 0;
 
+  if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) {
+    log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename);
+    errno = EFBIG;
+    close(fd);
+    return NULL;
+  }
   if (!size) {
     /* Zero-length file. If we call mmap on it, it will succeed but
      * return NULL, and bad things will happen. So just fail. */
@@ -2673,7 +2687,8 @@ static int uname_result_is_set = 0;
 
 /** Return a pointer to a description of our platform.
  */
-MOCK_IMPL(const char *, get_uname, (void))
+MOCK_IMPL(const char *,
+get_uname,(void))
 {
 #ifdef HAVE_UNAME
   struct utsname u;
@@ -3246,7 +3261,7 @@ format_win32_error(DWORD err)
                  FORMAT_MESSAGE_FROM_SYSTEM |
                  FORMAT_MESSAGE_IGNORE_INSERTS,
                  NULL, err,
-                 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                 MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT),
                  (LPVOID)&str,
                  0, NULL);
 
diff --git a/src/common/compat.h b/src/common/compat.h
index ee1c9454de..473ad2b957 100644
--- a/src/common/compat.h
+++ b/src/common/compat.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_H
@@ -414,10 +414,10 @@ struct tm *tor_gmtime_r(const time_t *timep, struct tm *result);
 #endif
 
 #ifndef timercmp
-/** Replacement for timersub on platforms that do not have it: returns true
+/** Replacement for timercmp on platforms that do not have it: returns true
  * iff the relational operator "op" makes the expression tv1 op tv2 true.
  *
- * Note that while this definition should work for all boolean opeators, some
+ * Note that while this definition should work for all boolean operators, some
  * platforms' native timercmp definitions do not support >=, <=, or ==.  So
  * don't use those.
  */
diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c
index 4a3b1af922..31eb4ac496 100644
--- a/src/common/compat_libevent.c
+++ b/src/common/compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -280,6 +280,15 @@ tor_gettimeofday_cache_set(const struct timeval *tv)
   tor_assert(tv);
   memcpy(&cached_time_hires, tv, sizeof(*tv));
 }
+
+/** For testing: called post-fork to make libevent reinitialize
+ * kernel structures. */
+void
+tor_libevent_postfork(void)
+{
+  int r = event_reinit(tor_libevent_get_base());
+  tor_assert(r == 0);
+}
 #endif
 #endif
 
diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h
index c2e34764e4..904938415c 100644
--- a/src/common/compat_libevent.h
+++ b/src/common/compat_libevent.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_LIBEVENT_H
@@ -54,6 +54,7 @@ void tor_gettimeofday_cached(struct timeval *tv);
 void tor_gettimeofday_cache_clear(void);
 #ifdef TOR_UNIT_TESTS
 void tor_gettimeofday_cache_set(const struct timeval *tv);
+void tor_libevent_postfork(void);
 #endif
 
 #ifdef COMPAT_LIBEVENT_PRIVATE
diff --git a/src/common/compat_openssl.h b/src/common/compat_openssl.h
index 1bfe188075..2b94fe5b4e 100644
--- a/src/common/compat_openssl.h
+++ b/src/common/compat_openssl.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_OPENSSL_H
diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c
index c1ae66c1d2..8e4b833573 100644
--- a/src/common/compat_pthreads.c
+++ b/src/common/compat_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/compat_rust.c b/src/common/compat_rust.c
new file mode 100644
index 0000000000..366fd4037b
--- /dev/null
+++ b/src/common/compat_rust.c
@@ -0,0 +1,39 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rust_compat.c
+ * \brief Rust FFI compatibility functions and helpers. This file is only built
+ * if Rust is not used.
+ **/
+
+#include "compat_rust.h"
+#include "util.h"
+
+/**
+ * Free storage pointed to by <b>str</b>, and itself.
+ */
+void
+rust_str_free(rust_str_t str)
+{
+    char *s = (char *)str;
+    tor_free(s);
+}
+
+/**
+ * Return zero-terminated contained string.
+ */
+const char *
+rust_str_get(const rust_str_t str)
+{
+    return (const char *)str;
+}
+
+/* If we were using Rust, we'd say so on startup. */
+rust_str_t
+rust_welcome_string(void)
+{
+    char *s = tor_malloc_zero(1);
+    return (rust_str_t)s;
+}
+
diff --git a/src/common/compat_rust.h b/src/common/compat_rust.h
new file mode 100644
index 0000000000..752a29b56c
--- /dev/null
+++ b/src/common/compat_rust.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rust_compat.h
+ * \brief Headers for rust_compat.c
+ **/
+
+#ifndef TOR_RUST_COMPAT_H
+#define TOR_RUST_COMPAT_H
+
+#include "torint.h"
+
+/**
+ * Strings allocated in Rust must be freed from Rust code again. Let's make
+ * it less likely to accidentally mess up and call tor_free() on it, because
+ * currently it'll just work but might break at any time.
+ */
+typedef uintptr_t rust_str_t;
+
+void rust_str_free(rust_str_t);
+
+const char *rust_str_get(const rust_str_t);
+
+rust_str_t rust_welcome_string(void);
+
+#endif
+
diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c
index f4809060d6..c593e9af8d 100644
--- a/src/common/compat_threads.c
+++ b/src/common/compat_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -94,51 +94,73 @@ in_main_thread(void)
 }
 
 #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE)
-/* As write(), but retry on EINTR */
+/* As write(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 write_ni(int fd, const void *buf, size_t n)
 {
   int r;
  again:
   r = (int) write(fd, buf, n);
-  if (r < 0 && errno == EINTR)
-    goto again;
+  if (r < 0) {
+    if (errno == EINTR)
+      goto again;
+    else
+      return -errno;
+  }
   return r;
 }
-/* As read(), but retry on EINTR */
+/* As read(), but retry on EINTR, and return the negative error code on error.
+ */
 static int
 read_ni(int fd, void *buf, size_t n)
 {
   int r;
  again:
   r = (int) read(fd, buf, n);
-  if (r < 0 && errno == EINTR)
-    goto again;
+  if (r < 0) {
+    if (errno == EINTR)
+      goto again;
+    else
+      return -errno;
+  }
   return r;
 }
 #endif
 
-/** As send(), but retry on EINTR. */
+/** As send(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 send_ni(int fd, const void *buf, size_t n, int flags)
 {
   int r;
  again:
   r = (int) send(fd, buf, n, flags);
-  if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd)))
-    goto again;
+  if (r < 0) {
+    int error = tor_socket_errno(fd);
+    if (ERRNO_IS_EINTR(error))
+      goto again;
+    else
+      return -error;
+  }
   return r;
 }
 
-/** As recv(), but retry on EINTR. */
+/** As recv(), but retry on EINTR, and return the negative error code on
+ * error. */
 static int
 recv_ni(int fd, void *buf, size_t n, int flags)
 {
   int r;
  again:
   r = (int) recv(fd, buf, n, flags);
-  if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd)))
-    goto again;
+  if (r < 0) {
+    int error = tor_socket_errno(fd);
+    if (ERRNO_IS_EINTR(error))
+      goto again;
+    else
+      return -error;
+  }
   return r;
 }
 
@@ -149,7 +171,7 @@ eventfd_alert(int fd)
 {
   uint64_t u = 1;
   int r = write_ni(fd, (void*)&u, sizeof(u));
-  if (r < 0 && errno != EAGAIN)
+  if (r < 0 && -r != EAGAIN)
     return -1;
   return 0;
 }
@@ -160,8 +182,8 @@ eventfd_drain(int fd)
 {
   uint64_t u = 0;
   int r = read_ni(fd, (void*)&u, sizeof(u));
-  if (r < 0 && errno != EAGAIN)
-    return -1;
+  if (r < 0 && -r != EAGAIN)
+    return r;
   return 0;
 }
 #endif
@@ -172,8 +194,8 @@ static int
 pipe_alert(int fd)
 {
   ssize_t r = write_ni(fd, "x", 1);
-  if (r < 0 && errno != EAGAIN)
-    return -1;
+  if (r < 0 && -r != EAGAIN)
+    return (int)r;
   return 0;
 }
 
@@ -188,7 +210,7 @@ pipe_drain(int fd)
     r = read_ni(fd, buf, sizeof(buf));
   } while (r > 0);
   if (r < 0 && errno != EAGAIN)
-    return -1;
+    return -errno;
   /* A value of r = 0 means EOF on the fd so successfully drained. */
   return 0;
 }
@@ -200,13 +222,13 @@ static int
 sock_alert(tor_socket_t fd)
 {
   ssize_t r = send_ni(fd, "x", 1, 0);
-  if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd)))
-    return -1;
+  if (r < 0 && !ERRNO_IS_EAGAIN(-r))
+    return (int)r;
   return 0;
 }
 
 /** Drain all the input from a socket <b>fd</b>, and ignore it.  Return 0 on
- * success, -1 on error. */
+ * success, -errno on error. */
 static int
 sock_drain(tor_socket_t fd)
 {
@@ -215,8 +237,8 @@ sock_drain(tor_socket_t fd)
   do {
     r = recv_ni(fd, buf, sizeof(buf), 0);
   } while (r > 0);
-  if (r < 0 && !ERRNO_IS_EAGAIN(tor_socket_errno(fd)))
-    return -1;
+  if (r < 0 && !ERRNO_IS_EAGAIN(-r))
+    return (int)r;
   /* A value of r = 0 means EOF on the fd so successfully drained. */
   return 0;
 }
@@ -330,3 +352,49 @@ alert_sockets_close(alert_sockets_t *socks)
   socks->read_fd = socks->write_fd = -1;
 }
 
+/*
+ * XXXX We might be smart to move to compiler intrinsics or real atomic
+ * XXXX operations at some point.  But not yet.
+ *
+ */
+
+/** Initialize a new atomic counter with the value 0 */
+void
+atomic_counter_init(atomic_counter_t *counter)
+{
+  memset(counter, 0, sizeof(*counter));
+  tor_mutex_init_nonrecursive(&counter->mutex);
+}
+/** Clean up all resources held by an atomic counter. */
+void
+atomic_counter_destroy(atomic_counter_t *counter)
+{
+  tor_mutex_uninit(&counter->mutex);
+  memset(counter, 0, sizeof(*counter));
+}
+/** Add a value to an atomic counter. */
+void
+atomic_counter_add(atomic_counter_t *counter, size_t add)
+{
+  tor_mutex_acquire(&counter->mutex);
+  counter->val += add;
+  tor_mutex_release(&counter->mutex);
+}
+/** Subtract a value from an atomic counter. */
+void
+atomic_counter_sub(atomic_counter_t *counter, size_t sub)
+{
+  // this relies on unsigned overflow, but that's fine.
+  atomic_counter_add(counter, -sub);
+}
+/** Return the current value of an atomic counter */
+size_t
+atomic_counter_get(atomic_counter_t *counter)
+{
+  size_t val;
+  tor_mutex_acquire(&counter->mutex);
+  val = counter->val;
+  tor_mutex_release(&counter->mutex);
+  return val;
+}
+
diff --git a/src/common/compat_threads.h b/src/common/compat_threads.h
index 171a9f93ff..9fa3d0d0b7 100644
--- a/src/common/compat_threads.h
+++ b/src/common/compat_threads.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_THREADS_H
@@ -147,5 +147,19 @@ void *tor_threadlocal_get(tor_threadlocal_t *threadlocal);
  */
 void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
 
+/**
+ * Atomic counter type; holds a size_t value.
+ */
+typedef struct atomic_counter_t {
+  tor_mutex_t mutex;
+  size_t val;
+} atomic_counter_t;
+
+void atomic_counter_init(atomic_counter_t *counter);
+void atomic_counter_destroy(atomic_counter_t *counter);
+void atomic_counter_add(atomic_counter_t *counter, size_t add);
+void atomic_counter_sub(atomic_counter_t *counter, size_t sub);
+size_t atomic_counter_get(atomic_counter_t *counter);
+
 #endif
 
diff --git a/src/common/compat_time.c b/src/common/compat_time.c
index d044bbe1d7..2ccaa36e49 100644
--- a/src/common/compat_time.c
+++ b/src/common/compat_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/compat_time.h b/src/common/compat_time.h
index 2262446e57..90194c5ebc 100644
--- a/src/common/compat_time.h
+++ b/src/common/compat_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c
index 735be4ad17..915368b94f 100644
--- a/src/common/compat_winthreads.c
+++ b/src/common/compat_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/compress.c b/src/common/compress.c
new file mode 100644
index 0000000000..beeff5fcb8
--- /dev/null
+++ b/src/common/compress.c
@@ -0,0 +1,658 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress.c
+ * \brief Common compression API.
+ **/
+
+#include "orconfig.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include "torint.h"
+
+#ifdef HAVE_NETINET_IN_H
+#include <netinet/in.h>
+#endif
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_lzma.h"
+#include "compress_none.h"
+#include "compress_zlib.h"
+#include "compress_zstd.h"
+
+/** Total number of bytes allocated for compression state overhead. */
+static atomic_counter_t total_compress_allocation;
+
+/** @{ */
+/* These macros define the maximum allowable compression factor.  Anything of
+ * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to
+ * have an uncompression factor (uncompressed size:compressed size ratio) of
+ * any greater than MAX_UNCOMPRESSION_FACTOR.
+ *
+ * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to
+ * be small to limit the attack multiplier, but we also want it to be large
+ * enough so that no legitimate document --even ones we might invent in the
+ * future -- ever compresses by a factor of greater than
+ * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably
+ * large range of possible values. IMO, anything over 8 is probably safe; IMO
+ * anything under 50 is probably sufficient.
+ */
+#define MAX_UNCOMPRESSION_FACTOR 25
+#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64)
+/** @} */
+
+/** Return true if uncompressing an input of size <b>in_size</b> to an input of
+ * size at least <b>size_out</b> looks like a compression bomb. */
+int
+tor_compress_is_compression_bomb(size_t size_in, size_t size_out)
+{
+  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
+    return 0;
+
+  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
+}
+
+/** Guess the size that <b>in_len</b> will be after compression or
+ * decompression. */
+static size_t
+guess_compress_size(int compress, compress_method_t method,
+                    compression_level_t compression_level,
+                    size_t in_len)
+{
+  // ignore these for now.
+  (void)compression_level;
+  if (method == NO_METHOD) {
+    /* Guess that we'll need an extra byte, to avoid a needless realloc
+     * for nul-termination */
+    return (in_len < SIZE_MAX) ? in_len + 1 : in_len;
+  }
+
+  /* Always guess a factor of 2. */
+  if (compress) {
+    in_len /= 2;
+  } else {
+    if (in_len < SIZE_T_CEILING/2)
+      in_len *= 2;
+  }
+  return MAX(in_len, 1024);
+}
+
+/** Internal function to implement tor_compress/tor_uncompress, depending on
+ * whether <b>compress</b> is set.  All arguments are as for tor_compress or
+ * tor_uncompress. */
+static int
+tor_compress_impl(int compress,
+                  char **out, size_t *out_len,
+                  const char *in, size_t in_len,
+                  compress_method_t method,
+                  compression_level_t compression_level,
+                  int complete_only,
+                  int protocol_warn_level)
+{
+  tor_compress_state_t *stream;
+  int rv;
+
+  stream = tor_compress_new(compress, method, compression_level);
+
+  if (stream == NULL) {
+    log_warn(LD_GENERAL, "NULL stream while %scompressing",
+             compress?"":"de");
+    log_debug(LD_GENERAL, "method: %d level: %d at len: %lu",
+              method, compression_level, (unsigned long)in_len);
+    return -1;
+  }
+
+  size_t in_len_orig = in_len;
+  size_t out_remaining, out_alloc;
+  char *outptr;
+
+  out_remaining = out_alloc =
+    guess_compress_size(compress, method, compression_level, in_len);
+  *out = outptr = tor_malloc(out_remaining);
+
+  const int finish = complete_only || compress;
+
+  while (1) {
+    switch (tor_compress_process(stream,
+                                 &outptr, &out_remaining,
+                                 &in, &in_len, finish)) {
+      case TOR_COMPRESS_DONE:
+        if (in_len == 0 || compress) {
+          goto done;
+        } else {
+          // More data is present, and we're decompressing.  So we may need to
+          // reinitialize the stream if we are handling multiple concatenated
+          // inputs.
+          tor_compress_free(stream);
+          stream = tor_compress_new(compress, method, compression_level);
+          if (stream == NULL) {
+            log_warn(LD_GENERAL, "NULL stream while %scompressing",
+                     compress?"":"de");
+            goto err;
+          }
+        }
+        break;
+      case TOR_COMPRESS_OK:
+        if (compress || complete_only) {
+          log_fn(protocol_warn_level, LD_PROTOCOL,
+                 "Unexpected %s while %scompressing",
+                 complete_only?"end of input":"result",
+                 compress?"":"de");
+          log_debug(LD_GENERAL, "method: %d level: %d at len: %lu",
+                    method, compression_level, (unsigned long)in_len);
+          goto err;
+        } else {
+          if (in_len == 0) {
+            goto done;
+          }
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL: {
+        if (!compress && outptr < *out+out_alloc) {
+          // A buffer error in this case means that we have a problem
+          // with our input.
+          log_fn(protocol_warn_level, LD_PROTOCOL,
+                 "Possible truncated or corrupt compressed data");
+          goto err;
+        }
+        if (out_alloc >= SIZE_T_CEILING / 2) {
+          log_warn(LD_GENERAL, "While %scompresing data: ran out of space.",
+                   compress?"":"un");
+          goto err;
+        }
+        if (!compress &&
+            tor_compress_is_compression_bomb(in_len_orig, out_alloc)) {
+          // This should already have been caught down in the backend logic.
+          // LCOV_EXCL_START
+          tor_assert_nonfatal_unreached();
+          goto err;
+          // LCOV_EXCL_STOP
+        }
+        const size_t offset = outptr - *out;
+        out_alloc *= 2;
+        *out = tor_realloc(*out, out_alloc);
+        outptr = *out + offset;
+        out_remaining = out_alloc - offset;
+        break;
+      }
+      case TOR_COMPRESS_ERROR:
+        log_fn(protocol_warn_level, LD_GENERAL,
+               "Error while %scompresing data: bad input?",
+               compress?"":"un");
+        goto err; // bad data.
+      default:
+        // LCOV_EXCL_START
+        tor_assert_nonfatal_unreached();
+        goto err;
+        // LCOV_EXCL_STOP
+    }
+  }
+ done:
+  *out_len = outptr - *out;
+  if (compress && tor_compress_is_compression_bomb(*out_len, in_len_orig)) {
+    log_warn(LD_BUG, "We compressed something and got an insanely high "
+             "compression factor; other Tors would think this was a "
+             "compression bomb.");
+    goto err;
+  }
+  if (!compress) {
+    // NUL-terminate our output.
+    if (out_alloc == *out_len)
+      *out = tor_realloc(*out, out_alloc + 1);
+    (*out)[*out_len] = '\0';
+  }
+  rv = 0;
+  goto out;
+
+ err:
+  tor_free(*out);
+  *out_len = 0;
+  rv = -1;
+  goto out;
+
+ out:
+  tor_compress_free(stream);
+  return rv;
+}
+
+/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
+ * allocated buffer, using the method described in <b>method</b>.  Store the
+ * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
+ * Return 0 on success, -1 on failure.
+ */
+int
+tor_compress(char **out, size_t *out_len,
+             const char *in, size_t in_len,
+             compress_method_t method)
+{
+  return tor_compress_impl(1, out, out_len, in, in_len, method,
+                           BEST_COMPRESSION,
+                           1, LOG_WARN);
+}
+
+/** Given zero or more compressed strings of total length <b>in_len</b> bytes
+ * at <b>in</b>, uncompress them into a newly allocated buffer, using the
+ * method described in <b>method</b>.  Store the uncompressed string in
+ * *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on success, -1 on
+ * failure.
+ *
+ * If any bytes are written to <b>out</b>, an extra byte NUL is always
+ * written at the end, but not counted in <b>out_len</b>.  This is a
+ * safety feature to ensure that the output can be treated as a
+ * NUL-terminated string -- though of course, callers should check
+ * out_len anyway.
+ *
+ * If <b>complete_only</b> is true, we consider a truncated input as a
+ * failure; otherwise we decompress as much as we can.  Warn about truncated
+ * or corrupt inputs at <b>protocol_warn_level</b>.
+ */
+int
+tor_uncompress(char **out, size_t *out_len,
+               const char *in, size_t in_len,
+               compress_method_t method,
+               int complete_only,
+               int protocol_warn_level)
+{
+  return tor_compress_impl(0, out, out_len, in, in_len, method,
+                           BEST_COMPRESSION,
+                           complete_only, protocol_warn_level);
+}
+
+/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
+ * to be compressed or not.  If it is, return the likeliest compression method.
+ * Otherwise, return UNKNOWN_METHOD.
+ */
+compress_method_t
+detect_compression_method(const char *in, size_t in_len)
+{
+  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
+    return GZIP_METHOD;
+  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
+             (ntohs(get_uint16(in)) % 31) == 0) {
+    return ZLIB_METHOD;
+  } else if (in_len > 2 &&
+             fast_memeq(in, "\x5d\x00\x00", 3)) {
+    return LZMA_METHOD;
+  } else if (in_len > 3 &&
+             fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) {
+    return ZSTD_METHOD;
+  } else {
+    return UNKNOWN_METHOD;
+  }
+}
+
+/** Return 1 if a given <b>method</b> is supported; otherwise 0. */
+int
+tor_compress_supports_method(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_method_supported();
+    case LZMA_METHOD:
+      return tor_lzma_method_supported();
+    case ZSTD_METHOD:
+      return tor_zstd_method_supported();
+    case NO_METHOD:
+      return 1;
+    case UNKNOWN_METHOD:
+    default:
+      return 0;
+  }
+}
+
+/**
+ * Return a bitmask of the supported compression types, where 1&lt;&lt;m is
+ * set in the bitmask if and only if compression with method <b>m</b> is
+ * supported.
+ */
+unsigned
+tor_compress_get_supported_method_bitmask(void)
+{
+  static unsigned supported = 0;
+  if (supported == 0) {
+    compress_method_t m;
+    for (m = NO_METHOD; m <= UNKNOWN_METHOD; ++m) {
+      if (tor_compress_supports_method(m)) {
+        supported |= (1u << m);
+      }
+    }
+  }
+  return supported;
+}
+
+/** Table of compression method names.  These should have an "x-" prefix,
+ * if they are not listed in the IANA content coding registry. */
+static const struct {
+  const char *name;
+  compress_method_t method;
+} compression_method_names[] = {
+  { "gzip", GZIP_METHOD },
+  { "deflate", ZLIB_METHOD },
+  // We call this "x-tor-lzma" rather than "x-lzma", because we impose a
+  // lower maximum memory usage on the decoding side.
+  { "x-tor-lzma", LZMA_METHOD },
+  { "x-zstd" , ZSTD_METHOD },
+  { "identity", NO_METHOD },
+
+  /* Later entries in this table are not canonical; these are recognized but
+   * not emitted. */
+  { "x-gzip", GZIP_METHOD },
+};
+
+/** Return the canonical string representation of the compression method
+ * <b>method</b>, or NULL if the method isn't recognized. */
+const char *
+compression_method_get_name(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) {
+    if (method == compression_method_names[i].method)
+      return compression_method_names[i].name;
+  }
+  return NULL;
+}
+
+/** Table of compression human readable method names. */
+static const struct {
+  compress_method_t method;
+  const char *name;
+} compression_method_human_names[] = {
+  { NO_METHOD, "uncompressed" },
+  { GZIP_METHOD, "gzipped" },
+  { ZLIB_METHOD, "deflated" },
+  { LZMA_METHOD, "LZMA compressed" },
+  { ZSTD_METHOD, "Zstandard compressed" },
+  { UNKNOWN_METHOD, "unknown encoding" },
+};
+
+/** Return a human readable string representation of the compression method
+ * <b>method</b>, or NULL if the method isn't recognized. */
+const char *
+compression_method_get_human_name(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_human_names); ++i) {
+    if (method == compression_method_human_names[i].method)
+      return compression_method_human_names[i].name;
+  }
+  return NULL;
+}
+
+/** Return the compression method represented by the string <b>name</b>, or
+ * UNKNOWN_METHOD if the string isn't recognized. */
+compress_method_t
+compression_method_get_by_name(const char *name)
+{
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(compression_method_names); ++i) {
+    if (!strcmp(compression_method_names[i].name, name))
+      return compression_method_names[i].method;
+  }
+  return UNKNOWN_METHOD;
+}
+
+/** Return a string representation of the version of the library providing the
+ * compression method given in <b>method</b>. Returns NULL if <b>method</b> is
+ * unknown or unsupported. */
+const char *
+tor_compress_version_str(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_get_version_str();
+    case LZMA_METHOD:
+      return tor_lzma_get_version_str();
+    case ZSTD_METHOD:
+      return tor_zstd_get_version_str();
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+    default:
+      return NULL;
+  }
+}
+
+/** Return a string representation of the version of the library, found at
+ * compile time, providing the compression method given in <b>method</b>.
+ * Returns NULL if <b>method</b> is unknown or unsupported. */
+const char *
+tor_compress_header_version_str(compress_method_t method)
+{
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      return tor_zlib_get_header_version_str();
+    case LZMA_METHOD:
+      return tor_lzma_get_header_version_str();
+    case ZSTD_METHOD:
+      return tor_zstd_get_header_version_str();
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+    default:
+      return NULL;
+  }
+}
+
+/** Return the approximate number of bytes allocated for all
+ * supported compression schemas. */
+size_t
+tor_compress_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_compress_allocation) +
+         tor_zlib_get_total_allocation() +
+         tor_lzma_get_total_allocation() +
+         tor_zstd_get_total_allocation();
+}
+
+/** Internal state for an incremental compression/decompression.  The body of
+ * this struct is not exposed. */
+struct tor_compress_state_t {
+  compress_method_t method; /**< The compression method. */
+
+  union {
+    tor_zlib_compress_state_t *zlib_state;
+    tor_lzma_compress_state_t *lzma_state;
+    tor_zstd_compress_state_t *zstd_state;
+  } u; /**< Compression backend state. */
+};
+
+/** Construct and return a tor_compress_state_t object using <b>method</b>.  If
+ * <b>compress</b>, it's for compression; otherwise it's for decompression. */
+tor_compress_state_t *
+tor_compress_new(int compress, compress_method_t method,
+                 compression_level_t compression_level)
+{
+  tor_compress_state_t *state;
+
+  state = tor_malloc_zero(sizeof(tor_compress_state_t));
+  state->method = method;
+
+  switch (method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD: {
+      tor_zlib_compress_state_t *zlib_state =
+        tor_zlib_compress_new(compress, method, compression_level);
+
+      if (zlib_state == NULL)
+        goto err;
+
+      state->u.zlib_state = zlib_state;
+      break;
+    }
+    case LZMA_METHOD: {
+      tor_lzma_compress_state_t *lzma_state =
+        tor_lzma_compress_new(compress, method, compression_level);
+
+      if (lzma_state == NULL)
+        goto err;
+
+      state->u.lzma_state = lzma_state;
+      break;
+    }
+    case ZSTD_METHOD: {
+      tor_zstd_compress_state_t *zstd_state =
+        tor_zstd_compress_new(compress, method, compression_level);
+
+      if (zstd_state == NULL)
+        goto err;
+
+      state->u.zstd_state = zstd_state;
+      break;
+    }
+    case NO_METHOD: {
+      break;
+    }
+    case UNKNOWN_METHOD:
+      goto err;
+  }
+
+  atomic_counter_add(&total_compress_allocation,
+                     sizeof(tor_compress_state_t));
+  return state;
+
+ err:
+  tor_free(state);
+  return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_compress_process(tor_compress_state_t *state,
+                     char **out, size_t *out_len,
+                     const char **in, size_t *in_len,
+                     int finish)
+{
+  tor_assert(state != NULL);
+  const size_t in_len_orig = *in_len;
+  const size_t out_len_orig = *out_len;
+  tor_compress_output_t rv;
+
+  if (*out_len == 0 && (*in_len > 0 || finish)) {
+    // If we still have input data, but no space for output data, we might as
+    // well return early and let the caller do the reallocation of the out
+    // variable.
+    return TOR_COMPRESS_BUFFER_FULL;
+  }
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      rv = tor_zlib_compress_process(state->u.zlib_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case LZMA_METHOD:
+      rv =tor_lzma_compress_process(state->u.lzma_state,
+                                    out, out_len, in, in_len,
+                                    finish);
+      break;
+    case ZSTD_METHOD:
+      rv = tor_zstd_compress_process(state->u.zstd_state,
+                                     out, out_len, in, in_len,
+                                     finish);
+      break;
+    case NO_METHOD:
+      rv = tor_cnone_compress_process(out, out_len, in, in_len,
+                                      finish);
+      break;
+    default:
+    case UNKNOWN_METHOD:
+      goto err;
+  }
+  if (BUG((rv == TOR_COMPRESS_OK) &&
+          *in_len == in_len_orig &&
+          *out_len == out_len_orig)) {
+    return TOR_COMPRESS_ERROR;
+  }
+
+  return rv;
+ err:
+  return TOR_COMPRESS_ERROR;
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_compress_free(tor_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      tor_zlib_compress_free(state->u.zlib_state);
+      break;
+    case LZMA_METHOD:
+      tor_lzma_compress_free(state->u.lzma_state);
+      break;
+    case ZSTD_METHOD:
+      tor_zstd_compress_free(state->u.zstd_state);
+      break;
+    case NO_METHOD:
+      break;
+    case UNKNOWN_METHOD:
+      break;
+  }
+
+  atomic_counter_sub(&total_compress_allocation,
+                     sizeof(tor_compress_state_t));
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_compress_state_size(const tor_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+
+  size_t size = sizeof(tor_compress_state_t);
+
+  switch (state->method) {
+    case GZIP_METHOD:
+    case ZLIB_METHOD:
+      size += tor_zlib_compress_state_size(state->u.zlib_state);
+      break;
+    case LZMA_METHOD:
+      size += tor_lzma_compress_state_size(state->u.lzma_state);
+      break;
+    case ZSTD_METHOD:
+      size += tor_zstd_compress_state_size(state->u.zstd_state);
+      break;
+    case NO_METHOD:
+    case UNKNOWN_METHOD:
+      break;
+  }
+
+  return size;
+}
+
+/** Initialize all compression modules. */
+void
+tor_compress_init(void)
+{
+  atomic_counter_init(&total_compress_allocation);
+
+  tor_zlib_init();
+  tor_lzma_init();
+  tor_zstd_init();
+}
+
diff --git a/src/common/compress.h b/src/common/compress.h
new file mode 100644
index 0000000000..59c8b7b9b5
--- /dev/null
+++ b/src/common/compress.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress.h
+ * \brief Headers for compress.c
+ **/
+
+#ifndef TOR_COMPRESS_H
+#define TOR_COMPRESS_H
+
+/** Enumeration of what kind of compression to use.  Only ZLIB_METHOD and
+ * GZIP_METHOD is guaranteed to be supported by the compress/uncompress
+ * functions here. Call tor_compress_supports_method() to check if a given
+ * compression schema is supported by Tor. */
+typedef enum {
+  NO_METHOD=0, // This method must be first.
+  GZIP_METHOD=1,
+  ZLIB_METHOD=2,
+  LZMA_METHOD=3,
+  ZSTD_METHOD=4,
+  UNKNOWN_METHOD=5, // This method must be last. Add new ones in the middle.
+} compress_method_t;
+
+/**
+ * Enumeration to define tradeoffs between memory usage and compression level.
+ * BEST_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
+ * memory.
+ **/
+typedef enum {
+  BEST_COMPRESSION, HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
+} compression_level_t;
+
+int tor_compress(char **out, size_t *out_len,
+                 const char *in, size_t in_len,
+                 compress_method_t method);
+
+int tor_uncompress(char **out, size_t *out_len,
+                   const char *in, size_t in_len,
+                   compress_method_t method,
+                   int complete_only,
+                   int protocol_warn_level);
+
+compress_method_t detect_compression_method(const char *in, size_t in_len);
+
+int tor_compress_is_compression_bomb(size_t size_in, size_t size_out);
+
+int tor_compress_supports_method(compress_method_t method);
+unsigned tor_compress_get_supported_method_bitmask(void);
+const char *compression_method_get_name(compress_method_t method);
+const char *compression_method_get_human_name(compress_method_t method);
+compress_method_t compression_method_get_by_name(const char *name);
+
+const char *tor_compress_version_str(compress_method_t method);
+
+const char *tor_compress_header_version_str(compress_method_t method);
+
+size_t tor_compress_get_total_allocation(void);
+
+/** Return values from tor_compress_process; see that function's documentation
+ * for details. */
+typedef enum {
+  TOR_COMPRESS_OK,
+  TOR_COMPRESS_DONE,
+  TOR_COMPRESS_BUFFER_FULL,
+  TOR_COMPRESS_ERROR
+} tor_compress_output_t;
+
+/** Internal state for an incremental compression/decompression. */
+typedef struct tor_compress_state_t tor_compress_state_t;
+
+tor_compress_state_t *tor_compress_new(int compress,
+                                       compress_method_t method,
+                                       compression_level_t level);
+
+tor_compress_output_t tor_compress_process(tor_compress_state_t *state,
+                                           char **out, size_t *out_len,
+                                           const char **in, size_t *in_len,
+                                           int finish);
+void tor_compress_free(tor_compress_state_t *state);
+
+size_t tor_compress_state_size(const tor_compress_state_t *state);
+
+void tor_compress_init(void);
+
+#endif // TOR_COMPRESS_H.
+
diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c
new file mode 100644
index 0000000000..d453d9f718
--- /dev/null
+++ b/src/common/compress_lzma.c
@@ -0,0 +1,357 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_lzma.c
+ * \brief Compression backend for LZMA.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_lzma.h"
+
+#ifdef HAVE_LZMA
+#include <lzma.h>
+#endif
+
+/** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */
+#define MEMORY_LIMIT (16 * 1024 * 1024)
+
+/** Total number of bytes allocated for LZMA state. */
+static atomic_counter_t total_lzma_allocation;
+
+#ifdef HAVE_LZMA
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return 6;
+    case MEDIUM_COMPRESSION: return 4;
+    case LOW_COMPRESSION: return 2;
+  }
+}
+
+/** Convert a given <b>error</b> to a human readable error string. */
+static const char *
+lzma_error_str(lzma_ret error)
+{
+  switch (error) {
+    case LZMA_OK:
+      return "Operation completed successfully";
+    case LZMA_STREAM_END:
+      return "End of stream";
+    case LZMA_NO_CHECK:
+      return "Input stream lacks integrity check";
+    case LZMA_UNSUPPORTED_CHECK:
+      return "Unable to calculate integrity check";
+    case LZMA_GET_CHECK:
+      return "Integrity check available";
+    case LZMA_MEM_ERROR:
+      return "Unable to allocate memory";
+    case LZMA_MEMLIMIT_ERROR:
+      return "Memory limit reached";
+    case LZMA_FORMAT_ERROR:
+      return "Unknown file format";
+    case LZMA_OPTIONS_ERROR:
+      return "Unsupported options";
+    case LZMA_DATA_ERROR:
+      return "Corrupt input data";
+    case LZMA_BUF_ERROR:
+      return "Unable to progress";
+    case LZMA_PROG_ERROR:
+      return "Programming error";
+    default:
+      return "Unknown LZMA error";
+  }
+}
+#endif // HAVE_LZMA.
+
+/** Return 1 if LZMA compression is supported; otherwise 0. */
+int
+tor_lzma_method_supported(void)
+{
+#ifdef HAVE_LZMA
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/** Return a string representation of the version of the currently running
+ * version of liblzma. Returns NULL if LZMA is unsupported. */
+const char *
+tor_lzma_get_version_str(void)
+{
+#ifdef HAVE_LZMA
+  return lzma_version_string();
+#else
+  return NULL;
+#endif
+}
+
+/** Return a string representation of the version of liblzma used at
+ * compilation time. Returns NULL if LZMA is unsupported. */
+const char *
+tor_lzma_get_header_version_str(void)
+{
+#ifdef HAVE_LZMA
+  return LZMA_VERSION_STRING;
+#else
+  return NULL;
+#endif
+}
+
+/** Internal LZMA state for incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_lzma_compress_state_t {
+#ifdef HAVE_LZMA
+  lzma_stream stream; /**< The LZMA stream. */
+#endif
+
+  int compress; /**< True if we are compressing; false if we are inflating */
+
+  /** Number of bytes read so far.  Used to detect compression bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect compression bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+#ifdef HAVE_LZMA
+/** Return an approximate number of bytes stored in memory to hold the LZMA
+ * encoder/decoder state. */
+static size_t
+tor_lzma_state_size_precalc(int compress, compression_level_t level)
+{
+  uint64_t memory_usage;
+
+  if (compress)
+    memory_usage = lzma_easy_encoder_memusage(memory_level(level));
+  else
+    memory_usage = lzma_easy_decoder_memusage(memory_level(level));
+
+  if (memory_usage == UINT64_MAX) {
+    // LCOV_EXCL_START
+    log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s",
+                         compress ? "encoder" : "decoder");
+    goto err;
+    // LCOV_EXCL_STOP
+  }
+
+  if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX)
+    memory_usage = SIZE_MAX;
+  else
+    memory_usage += sizeof(tor_lzma_compress_state_t);
+
+  return (size_t)memory_usage;
+
+ err:
+  return 0; // LCOV_EXCL_LINE
+}
+#endif // HAVE_LZMA.
+
+/** Construct and return a tor_lzma_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_lzma_compress_state_t *
+tor_lzma_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t level)
+{
+  tor_assert(method == LZMA_METHOD);
+
+#ifdef HAVE_LZMA
+  tor_lzma_compress_state_t *result;
+  lzma_ret retval;
+  lzma_options_lzma stream_options;
+
+  // Note that we do not explicitly initialize the lzma_stream object here,
+  // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is
+  // also what `tor_malloc_zero()` does.
+  result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t));
+  result->compress = compress;
+  result->allocation = tor_lzma_state_size_precalc(compress, level);
+
+  if (compress) {
+    lzma_lzma_preset(&stream_options, memory_level(level));
+
+    retval = lzma_alone_encoder(&result->stream, &stream_options);
+
+    if (retval != LZMA_OK) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).",
+               lzma_error_str(retval), retval);
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  } else {
+    retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT);
+
+    if (retval != LZMA_OK) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).",
+               lzma_error_str(retval), retval);
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  atomic_counter_add(&total_lzma_allocation, result->allocation);
+  return result;
+
+ err:
+  tor_free(result); // LCOV_EXCL_LINE
+  return NULL;
+#else // HAVE_LZMA.
+  (void)compress;
+  (void)method;
+  (void)level;
+
+  return NULL;
+#endif // HAVE_LZMA.
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_lzma_compress_process(tor_lzma_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+#ifdef HAVE_LZMA
+  lzma_ret retval;
+  lzma_action action;
+
+  tor_assert(state != NULL);
+  tor_assert(*in_len <= UINT_MAX);
+  tor_assert(*out_len <= UINT_MAX);
+
+  state->stream.next_in = (unsigned char *)*in;
+  state->stream.avail_in = *in_len;
+  state->stream.next_out = (unsigned char *)*out;
+  state->stream.avail_out = *out_len;
+
+  action = finish ? LZMA_FINISH : LZMA_RUN;
+
+  retval = lzma_code(&state->stream, action);
+
+  state->input_so_far += state->stream.next_in - ((unsigned char *)*in);
+  state->output_so_far += state->stream.next_out - ((unsigned char *)*out);
+
+  *out = (char *)state->stream.next_out;
+  *out_len = state->stream.avail_out;
+  *in = (const char *)state->stream.next_in;
+  *in_len = state->stream.avail_in;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  switch (retval) {
+    case LZMA_OK:
+      if (state->stream.avail_out == 0 || finish)
+        return TOR_COMPRESS_BUFFER_FULL;
+
+      return TOR_COMPRESS_OK;
+
+    case LZMA_BUF_ERROR:
+      if (state->stream.avail_in == 0 && !finish)
+        return TOR_COMPRESS_OK;
+
+      return TOR_COMPRESS_BUFFER_FULL;
+
+    case LZMA_STREAM_END:
+      return TOR_COMPRESS_DONE;
+
+    // We list all the possible values of `lzma_ret` here to silence the
+    // `switch-enum` warning and to detect if a new member was added.
+    case LZMA_NO_CHECK:
+    case LZMA_UNSUPPORTED_CHECK:
+    case LZMA_GET_CHECK:
+    case LZMA_MEM_ERROR:
+    case LZMA_MEMLIMIT_ERROR:
+    case LZMA_FORMAT_ERROR:
+    case LZMA_OPTIONS_ERROR:
+    case LZMA_DATA_ERROR:
+    case LZMA_PROG_ERROR:
+    default:
+      log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.",
+               state->compress ? "compression" : "decompression",
+               lzma_error_str(retval));
+      return TOR_COMPRESS_ERROR;
+  }
+#else // HAVE_LZMA.
+  (void)state;
+  (void)out;
+  (void)out_len;
+  (void)in;
+  (void)in_len;
+  (void)finish;
+  return TOR_COMPRESS_ERROR;
+#endif // HAVE_LZMA.
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_lzma_compress_free(tor_lzma_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_lzma_allocation, state->allocation);
+
+#ifdef HAVE_LZMA
+  lzma_end(&state->stream);
+#endif
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all LZMA states. */
+size_t
+tor_lzma_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_lzma_allocation);
+}
+
+/** Initialize the lzma module */
+void
+tor_lzma_init(void)
+{
+  atomic_counter_init(&total_lzma_allocation);
+}
+
diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h
new file mode 100644
index 0000000000..1433c89f88
--- /dev/null
+++ b/src/common/compress_lzma.h
@@ -0,0 +1,43 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_lzma.h
+ * \brief Header for compress_lzma.c
+ **/
+
+#ifndef TOR_COMPRESS_LZMA_H
+#define TOR_COMPRESS_LZMA_H
+
+int tor_lzma_method_supported(void);
+
+const char *tor_lzma_get_version_str(void);
+
+const char *tor_lzma_get_header_version_str(void);
+
+/** Internal state for an incremental LZMA compression/decompression. */
+typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t;
+
+tor_lzma_compress_state_t *
+tor_lzma_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_lzma_compress_process(tor_lzma_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_lzma_compress_free(tor_lzma_compress_state_t *state);
+
+size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state);
+
+size_t tor_lzma_get_total_allocation(void);
+
+void tor_lzma_init(void);
+
+#endif // TOR_COMPRESS_LZMA_H.
+
diff --git a/src/common/compress_none.c b/src/common/compress_none.c
new file mode 100644
index 0000000000..34314e4af7
--- /dev/null
+++ b/src/common/compress_none.c
@@ -0,0 +1,53 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_none.c
+ * \brief Compression backend for identity compression.
+ *
+ * We actually define this backend so that we can treat the identity transform
+ * as another case of compression.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_none.h"
+
+/** Transfer some bytes using the identity transformation.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_cnone_compress_process(char **out, size_t *out_len,
+                           const char **in, size_t *in_len,
+                           int finish)
+{
+  size_t n_to_copy = MIN(*in_len, *out_len);
+
+  memcpy(*out, *in, n_to_copy);
+  *out += n_to_copy;
+  *in += n_to_copy;
+  *out_len -= n_to_copy;
+  *in_len -= n_to_copy;
+  if (*in_len == 0) {
+    return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK;
+  } else {
+    return TOR_COMPRESS_BUFFER_FULL;
+  }
+}
+
diff --git a/src/common/compress_none.h b/src/common/compress_none.h
new file mode 100644
index 0000000000..d1ebb4b625
--- /dev/null
+++ b/src/common/compress_none.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_none.h
+ * \brief Header for compress_none.c
+ **/
+
+#ifndef TOR_COMPRESS_NONE_H
+#define TOR_COMPRESS_NONE_H
+
+tor_compress_output_t
+tor_cnone_compress_process(char **out, size_t *out_len,
+                           const char **in, size_t *in_len,
+                           int finish);
+
+#endif // TOR_COMPRESS_NONE_H.
+
diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c
new file mode 100644
index 0000000000..284542e885
--- /dev/null
+++ b/src/common/compress_zlib.c
@@ -0,0 +1,304 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.c
+ * \brief Compression backend for gzip and zlib.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_zlib.h"
+
+/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
+   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
+   that nobody will care if the compile outputs a no-such-identifier warning.
+
+   Sorry, but we like -Werror over here, so I guess we need to define these.
+   I hope that zlib 1.2.6 doesn't break these too.
+*/
+#ifndef _LARGEFILE64_SOURCE
+#define _LARGEFILE64_SOURCE 0
+#endif
+#ifndef _LFS64_LARGEFILE
+#define _LFS64_LARGEFILE 0
+#endif
+#ifndef _FILE_OFFSET_BITS
+#define _FILE_OFFSET_BITS 0
+#endif
+#ifndef off64_t
+#define off64_t int64_t
+#endif
+
+#include <zlib.h>
+
+#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
+#error "We require zlib version 1.2 or later."
+#endif
+
+static size_t tor_zlib_state_size_precalc(int inflate,
+                                          int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static atomic_counter_t total_zlib_allocation;
+
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION: return 9;
+    case HIGH_COMPRESSION: return 8;
+    case MEDIUM_COMPRESSION: return 7;
+    case LOW_COMPRESSION: return 6;
+  }
+}
+
+/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
+static inline int
+method_bits(compress_method_t method, compression_level_t level)
+{
+  /* Bits+16 means "use gzip" in zlib >= 1.2 */
+  const int flag = method == GZIP_METHOD ? 16 : 0;
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return flag + 15;
+    case MEDIUM_COMPRESSION: return flag + 13;
+    case LOW_COMPRESSION: return flag + 11;
+  }
+}
+
+/** Return 1 if zlib/gzip compression is supported; otherwise 0. */
+int
+tor_zlib_method_supported(void)
+{
+  /* We currently always support zlib/gzip, but we keep this function around in
+   * case we some day decide to deprecate zlib/gzip support.
+   */
+  return 1;
+}
+
+/** Return a string representation of the version of the currently running
+ * version of zlib. */
+const char *
+tor_zlib_get_version_str(void)
+{
+  return zlibVersion();
+}
+
+/** Return a string representation of the version of the version of zlib
+* used at compilation. */
+const char *
+tor_zlib_get_header_version_str(void)
+{
+  return ZLIB_VERSION;
+}
+
+/** Internal zlib state for an incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zlib_compress_state_t {
+  struct z_stream_s stream; /**< The zlib stream */
+  int compress; /**< True if we are compressing; false if we are inflating */
+
+  /** Number of bytes read so far.  Used to detect zlib bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect zlib bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
+{
+  windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+  if (inflate_) {
+    /* From zconf.h:
+
+       "The memory requirements for inflate are (in bytes) 1 << windowBits
+       that is, 32K for windowBits=15 (default value) plus a few kilobytes
+       for small objects."
+    */
+    return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+      (1 << 15) + A_FEW_KILOBYTES;
+  } else {
+    /* Also from zconf.h:
+
+       "The memory requirements for deflate are (in bytes):
+            (1 << (windowBits+2)) +  (1 << (memLevel+9))
+        ... plus a few kilobytes for small objects."
+    */
+    return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) +
+      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+  }
+#undef A_FEW_KILOBYTES
+}
+
+/** Construct and return a tor_zlib_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress_,
+                      compress_method_t method,
+                      compression_level_t compression_level)
+{
+  tor_zlib_compress_state_t *out;
+  int bits, memlevel;
+
+  if (! compress_) {
+    /* use this setting for decompression, since we might have the
+     * max number of window bits */
+    compression_level = BEST_COMPRESSION;
+  }
+
+  out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t));
+  out->stream.zalloc = Z_NULL;
+  out->stream.zfree = Z_NULL;
+  out->stream.opaque = NULL;
+  out->compress = compress_;
+  bits = method_bits(method, compression_level);
+  memlevel = memory_level(compression_level);
+  if (compress_) {
+    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
+                     bits, memlevel,
+                     Z_DEFAULT_STRATEGY) != Z_OK)
+    goto err; // LCOV_EXCL_LINE
+  } else {
+    if (inflateInit2(&out->stream, bits) != Z_OK)
+      goto err; // LCOV_EXCL_LINE
+  }
+  out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
+
+  atomic_counter_add(&total_zlib_allocation, out->allocation);
+
+  return out;
+
+ err:
+  tor_free(out);
+  return NULL;
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+  int err;
+  tor_assert(state != NULL);
+  if (*in_len > UINT_MAX ||
+      *out_len > UINT_MAX) {
+    return TOR_COMPRESS_ERROR;
+  }
+
+  state->stream.next_in = (unsigned char*) *in;
+  state->stream.avail_in = (unsigned int)*in_len;
+  state->stream.next_out = (unsigned char*) *out;
+  state->stream.avail_out = (unsigned int)*out_len;
+
+  if (state->compress) {
+    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
+  } else {
+    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
+  }
+
+  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
+  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
+
+  *out = (char*) state->stream.next_out;
+  *out_len = state->stream.avail_out;
+  *in = (const char *) state->stream.next_in;
+  *in_len = state->stream.avail_in;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  switch (err)
+    {
+    case Z_STREAM_END:
+      return TOR_COMPRESS_DONE;
+    case Z_BUF_ERROR:
+      if (state->stream.avail_in == 0 && !finish)
+        return TOR_COMPRESS_OK;
+      return TOR_COMPRESS_BUFFER_FULL;
+    case Z_OK:
+      if (state->stream.avail_out == 0 || finish)
+        return TOR_COMPRESS_BUFFER_FULL;
+      return TOR_COMPRESS_OK;
+    default:
+      log_warn(LD_GENERAL, "Gzip returned an error: %s",
+               state->stream.msg ? state->stream.msg : "<no message>");
+      return TOR_COMPRESS_ERROR;
+    }
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zlib_compress_free(tor_zlib_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_zlib_allocation, state->allocation);
+
+  if (state->compress)
+    deflateEnd(&state->stream);
+  else
+    inflateEnd(&state->stream);
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_zlib_allocation);
+}
+
+/** Set up global state for the zlib module */
+void
+tor_zlib_init(void)
+{
+  atomic_counter_init(&total_zlib_allocation);
+}
+
diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h
new file mode 100644
index 0000000000..df5c196ac7
--- /dev/null
+++ b/src/common/compress_zlib.h
@@ -0,0 +1,43 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zlib.h
+ * \brief Header for compress_zlib.c
+ **/
+
+#ifndef TOR_COMPRESS_ZLIB_H
+#define TOR_COMPRESS_ZLIB_H
+
+int tor_zlib_method_supported(void);
+
+const char *tor_zlib_get_version_str(void);
+
+const char *tor_zlib_get_header_version_str(void);
+
+/** Internal state for an incremental zlib/gzip compression/decompression. */
+typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t;
+
+tor_zlib_compress_state_t *
+tor_zlib_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zlib_compress_process(tor_zlib_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_zlib_compress_free(tor_zlib_compress_state_t *state);
+
+size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state);
+
+size_t tor_zlib_get_total_allocation(void);
+
+void tor_zlib_init(void);
+
+#endif // TOR_COMPRESS_ZLIB_H.
+
diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c
new file mode 100644
index 0000000000..11edfffa0e
--- /dev/null
+++ b/src/common/compress_zstd.c
@@ -0,0 +1,444 @@
+/* Copyright (c) 2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zstd.c
+ * \brief Compression backend for Zstandard.
+ *
+ * This module should never be invoked directly. Use the compress module
+ * instead.
+ **/
+
+#include "orconfig.h"
+
+#include "util.h"
+#include "torlog.h"
+#include "compress.h"
+#include "compress_zstd.h"
+
+#ifdef HAVE_ZSTD
+DISABLE_GCC_WARNING(unused-const-variable)
+#include <zstd.h>
+ENABLE_GCC_WARNING(unused-const-variable)
+#endif
+
+/** Total number of bytes allocated for Zstandard state. */
+static atomic_counter_t total_zstd_allocation;
+
+#ifdef HAVE_ZSTD
+/** Given <b>level</b> return the memory level. */
+static int
+memory_level(compression_level_t level)
+{
+  switch (level) {
+    default:
+    case BEST_COMPRESSION:
+    case HIGH_COMPRESSION: return 9;
+    case MEDIUM_COMPRESSION: return 8;
+    case LOW_COMPRESSION: return 7;
+  }
+}
+#endif // HAVE_ZSTD.
+
+/** Return 1 if Zstandard compression is supported; otherwise 0. */
+int
+tor_zstd_method_supported(void)
+{
+#ifdef HAVE_ZSTD
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/** Return a string representation of the version of the currently running
+ * version of libzstd. Returns NULL if Zstandard is unsupported. */
+const char *
+tor_zstd_get_version_str(void)
+{
+#ifdef HAVE_ZSTD
+  static char version_str[16];
+  size_t version_number;
+
+  version_number = ZSTD_versionNumber();
+  tor_snprintf(version_str, sizeof(version_str),
+               "%d.%d.%d",
+               (int) version_number / 10000 % 100,
+               (int) version_number / 100 % 100,
+               (int) version_number % 100);
+
+  return version_str;
+#else
+  return NULL;
+#endif
+}
+
+/** Return a string representation of the version of the version of libzstd
+ * used at compilation time. Returns NULL if Zstandard is unsupported. */
+const char *
+tor_zstd_get_header_version_str(void)
+{
+#ifdef HAVE_ZSTD
+  return ZSTD_VERSION_STRING;
+#else
+  return NULL;
+#endif
+}
+
+/** Internal Zstandard state for incremental compression/decompression.
+ * The body of this struct is not exposed. */
+struct tor_zstd_compress_state_t {
+#ifdef HAVE_ZSTD
+  union {
+    /** Compression stream. Used when <b>compress</b> is true. */
+    ZSTD_CStream *compress_stream;
+    /** Decompression stream. Used when <b>compress</b> is false. */
+    ZSTD_DStream *decompress_stream;
+  } u; /**< Zstandard stream objects. */
+#endif // HAVE_ZSTD.
+
+  int compress; /**< True if we are compressing; false if we are inflating */
+  int have_called_end; /**< True if we are compressing and we've called
+                        * ZSTD_endStream */
+
+  /** Number of bytes read so far.  Used to detect compression bombs. */
+  size_t input_so_far;
+  /** Number of bytes written so far.  Used to detect compression bombs. */
+  size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
+};
+
+#ifdef HAVE_ZSTD
+/** Return an approximate number of bytes stored in memory to hold the
+ * Zstandard compression/decompression state. */
+static size_t
+tor_zstd_state_size_precalc(int compress, int preset)
+{
+  tor_assert(preset > 0);
+
+  size_t memory_usage = sizeof(tor_zstd_compress_state_t);
+
+  // The Zstandard library provides a number of functions that would be useful
+  // here, but they are, unfortunately, still considered experimental and are
+  // thus only available in libzstd if we link against the library statically.
+  //
+  // The code in this function tries to approximate the calculations without
+  // being able to use the following:
+  //
+  // - We do not have access to neither the internal members of ZSTD_CStream
+  //   and ZSTD_DStream and their internal context objects.
+  //
+  // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they
+  //   are unexposed.
+  //
+  // In the future it might be useful to check if libzstd have started
+  // providing these functions in a stable manner and simplify this function.
+  if (compress) {
+    // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine:
+    memory_usage += 192;
+
+    // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to
+    // variables that are not exposed via the public API. We use a _very_
+    // simplified function to calculate the estimated amount of bytes used in
+    // this struct.
+    // memory_usage += (preset - 0.5) * 1024 * 1024;
+    memory_usage += (preset * 1024 * 1024) - (512 * 1024);
+    // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes.
+    // - stream->outBuffSize: 128 KB:
+    memory_usage += 128 * 1024;
+    // - stream->inBuffSize: 2048 KB:
+    memory_usage += 2048 * 1024;
+  } else {
+    // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream)
+    // function here. This function uses the following fields to make its
+    // estimate:
+
+    // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine:
+    memory_usage += 208;
+    // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB.
+    memory_usage += 150 * 1024;
+
+    // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes.
+    // - stream->inBuffSize: 0 KB.
+    // - stream->outBuffSize: 0 KB.
+  }
+
+  return memory_usage;
+}
+#endif // HAVE_ZSTD.
+
+/** Construct and return a tor_zstd_compress_state_t object using
+ * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for
+ * decompression. */
+tor_zstd_compress_state_t *
+tor_zstd_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t level)
+{
+  tor_assert(method == ZSTD_METHOD);
+
+#ifdef HAVE_ZSTD
+  const int preset = memory_level(level);
+  tor_zstd_compress_state_t *result;
+  size_t retval;
+
+  result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t));
+  result->compress = compress;
+  result->allocation = tor_zstd_state_size_precalc(compress, preset);
+
+  if (compress) {
+    result->u.compress_stream = ZSTD_createCStream();
+
+    if (result->u.compress_stream == NULL) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error while creating Zstandard compression "
+               "stream");
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+
+    retval = ZSTD_initCStream(result->u.compress_stream, preset);
+
+    if (ZSTD_isError(retval)) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
+               ZSTD_getErrorName(retval));
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  } else {
+    result->u.decompress_stream = ZSTD_createDStream();
+
+    if (result->u.decompress_stream == NULL) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Error while creating Zstandard decompression "
+               "stream");
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+
+    retval = ZSTD_initDStream(result->u.decompress_stream);
+
+    if (ZSTD_isError(retval)) {
+      // LCOV_EXCL_START
+      log_warn(LD_GENERAL, "Zstandard stream initialization error: %s",
+               ZSTD_getErrorName(retval));
+      goto err;
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  atomic_counter_add(&total_zstd_allocation, result->allocation);
+  return result;
+
+ err:
+  // LCOV_EXCL_START
+  if (compress) {
+    ZSTD_freeCStream(result->u.compress_stream);
+  } else {
+    ZSTD_freeDStream(result->u.decompress_stream);
+  }
+
+  tor_free(result);
+  return NULL;
+  // LCOV_EXCL_STOP
+#else // HAVE_ZSTD.
+  (void)compress;
+  (void)method;
+  (void)level;
+
+  return NULL;
+#endif // HAVE_ZSTD.
+}
+
+/** Compress/decompress some bytes using <b>state</b>.  Read up to
+ * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
+ * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
+ * we've reached the end of the input.
+ *
+ * Return TOR_COMPRESS_DONE if we've finished the entire
+ * compression/decompression.
+ * Return TOR_COMPRESS_OK if we're processed everything from the input.
+ * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>.
+ * Return TOR_COMPRESS_ERROR if the stream is corrupt.
+ */
+tor_compress_output_t
+tor_zstd_compress_process(tor_zstd_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish)
+{
+#ifdef HAVE_ZSTD
+  size_t retval;
+
+  tor_assert(state != NULL);
+  tor_assert(*in_len <= UINT_MAX);
+  tor_assert(*out_len <= UINT_MAX);
+
+  ZSTD_inBuffer input = { *in, *in_len, 0 };
+  ZSTD_outBuffer output = { *out, *out_len, 0 };
+
+  if (BUG(finish == 0 && state->have_called_end)) {
+    finish = 1;
+  }
+
+  if (state->compress) {
+    if (! state->have_called_end)
+      retval = ZSTD_compressStream(state->u.compress_stream,
+                                   &output, &input);
+    else
+      retval = 0;
+  } else {
+    retval = ZSTD_decompressStream(state->u.decompress_stream,
+                                   &output, &input);
+  }
+
+  state->input_so_far += input.pos;
+  state->output_so_far += output.pos;
+
+  *out = (char *)output.dst + output.pos;
+  *out_len = output.size - output.pos;
+  *in = (char *)input.src + input.pos;
+  *in_len = input.size - input.pos;
+
+  if (! state->compress &&
+      tor_compress_is_compression_bomb(state->input_so_far,
+                                       state->output_so_far)) {
+    log_warn(LD_DIR, "Possible compression bomb; abandoning stream.");
+    return TOR_COMPRESS_ERROR;
+  }
+
+  if (ZSTD_isError(retval)) {
+    log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.",
+             state->compress ? "compression" : "decompression",
+             ZSTD_getErrorName(retval));
+    return TOR_COMPRESS_ERROR;
+  }
+
+  if (state->compress && !state->have_called_end) {
+    retval = ZSTD_flushStream(state->u.compress_stream, &output);
+
+    *out = (char *)output.dst + output.pos;
+    *out_len = output.size - output.pos;
+
+    if (ZSTD_isError(retval)) {
+      log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.",
+               ZSTD_getErrorName(retval));
+      return TOR_COMPRESS_ERROR;
+    }
+
+    // ZSTD_flushStream returns 0 if the frame is done, or >0 if it
+    // is incomplete.
+    if (retval > 0) {
+      return TOR_COMPRESS_BUFFER_FULL;
+    }
+  }
+
+  if (!finish) {
+    // The caller says we're not done with the input, so no need to write an
+    // epilogue.
+    return TOR_COMPRESS_OK;
+  } else if (state->compress) {
+    if (*in_len) {
+      // We say that we're not done with the input, so we can't write an
+      // epilogue.
+      return TOR_COMPRESS_OK;
+    }
+
+    retval = ZSTD_endStream(state->u.compress_stream, &output);
+    state->have_called_end = 1;
+    *out = (char *)output.dst + output.pos;
+    *out_len = output.size - output.pos;
+
+    if (ZSTD_isError(retval)) {
+      log_warn(LD_GENERAL, "Zstandard compression unable to write "
+               "epilogue: %s.",
+               ZSTD_getErrorName(retval));
+      return TOR_COMPRESS_ERROR;
+    }
+
+    // endStream returns the number of bytes that is needed to write the
+    // epilogue.
+    if (retval > 0)
+      return TOR_COMPRESS_BUFFER_FULL;
+
+    return TOR_COMPRESS_DONE;
+  } else /* if (!state->compress) */ {
+    // ZSTD_decompressStream returns 0 if the frame is done, or >0 if it
+    // is incomplete.
+    // We check this above.
+    tor_assert_nonfatal(!ZSTD_isError(retval));
+    // Start a new frame if this frame is done
+    if (retval == 0)
+      return TOR_COMPRESS_DONE;
+    // Don't check out_len, it might have some space left if the next output
+    // chunk is larger than the remaining space
+    else if (*in_len > 0)
+      return  TOR_COMPRESS_BUFFER_FULL;
+    else
+      return TOR_COMPRESS_OK;
+  }
+
+#else // HAVE_ZSTD.
+  (void)state;
+  (void)out;
+  (void)out_len;
+  (void)in;
+  (void)in_len;
+  (void)finish;
+
+  return TOR_COMPRESS_ERROR;
+#endif // HAVE_ZSTD.
+}
+
+/** Deallocate <b>state</b>. */
+void
+tor_zstd_compress_free(tor_zstd_compress_state_t *state)
+{
+  if (state == NULL)
+    return;
+
+  atomic_counter_sub(&total_zstd_allocation, state->allocation);
+
+#ifdef HAVE_ZSTD
+  if (state->compress) {
+    ZSTD_freeCStream(state->u.compress_stream);
+  } else {
+    ZSTD_freeDStream(state->u.decompress_stream);
+  }
+#endif // HAVE_ZSTD.
+
+  tor_free(state);
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state)
+{
+  tor_assert(state != NULL);
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all Zstandard
+ * states. */
+size_t
+tor_zstd_get_total_allocation(void)
+{
+  return atomic_counter_get(&total_zstd_allocation);
+}
+
+/** Initialize the zstd module */
+void
+tor_zstd_init(void)
+{
+  atomic_counter_init(&total_zstd_allocation);
+}
+
diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h
new file mode 100644
index 0000000000..d3e65c2f16
--- /dev/null
+++ b/src/common/compress_zstd.h
@@ -0,0 +1,43 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_zstd.h
+ * \brief Header for compress_zstd.c
+ **/
+
+#ifndef TOR_COMPRESS_ZSTD_H
+#define TOR_COMPRESS_ZSTD_H
+
+int tor_zstd_method_supported(void);
+
+const char *tor_zstd_get_version_str(void);
+
+const char *tor_zstd_get_header_version_str(void);
+
+/** Internal state for an incremental Zstandard compression/decompression. */
+typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t;
+
+tor_zstd_compress_state_t *
+tor_zstd_compress_new(int compress,
+                      compress_method_t method,
+                      compression_level_t compression_level);
+
+tor_compress_output_t
+tor_zstd_compress_process(tor_zstd_compress_state_t *state,
+                          char **out, size_t *out_len,
+                          const char **in, size_t *in_len,
+                          int finish);
+
+void tor_zstd_compress_free(tor_zstd_compress_state_t *state);
+
+size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state);
+
+size_t tor_zstd_get_total_allocation(void);
+
+void tor_zstd_init(void);
+
+#endif // TOR_COMPRESS_ZSTD_H.
+
diff --git a/src/common/confline.c b/src/common/confline.c
new file mode 100644
index 0000000000..15fd96bf38
--- /dev/null
+++ b/src/common/confline.c
@@ -0,0 +1,527 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "compat.h"
+#include "confline.h"
+#include "torlog.h"
+#include "util.h"
+#include "container.h"
+
+static int config_get_lines_aux(const char *string, config_line_t **result,
+                                int extended, int allow_include,
+                                int *has_include, int recursion_level,
+                                config_line_t **last);
+static smartlist_t *config_get_file_list(const char *path);
+static int config_get_included_list(const char *path, int recursion_level,
+                                    int extended, config_line_t **list,
+                                    config_line_t **list_last);
+static int config_process_include(const char *path, int recursion_level,
+                                  int extended, config_line_t **list,
+                                  config_line_t **list_last);
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * append it to *<b>lst</b>. */
+void
+config_line_append(config_line_t **lst,
+                   const char *key,
+                   const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = NULL;
+  while (*lst)
+    lst = &((*lst)->next);
+
+  (*lst) = newline;
+}
+
+/** Helper: allocate a new configuration option mapping 'key' to 'val',
+ * and prepend it to *<b>lst</b> */
+void
+config_line_prepend(config_line_t **lst,
+                    const char *key,
+                    const char *val)
+{
+  tor_assert(lst);
+
+  config_line_t *newline;
+
+  newline = tor_malloc_zero(sizeof(config_line_t));
+  newline->key = tor_strdup(key);
+  newline->value = tor_strdup(val);
+  newline->next = *lst;
+  *lst = newline;
+}
+
+/** Return the first line in <b>lines</b> whose key is exactly <b>key</b>, or
+ * NULL if no such key exists.
+ *
+ * (In options parsing, this is for handling commandline-only options only;
+ * other options should be looked up in the appropriate data structure.) */
+const config_line_t *
+config_line_find(const config_line_t *lines,
+                 const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
+/** Auxiliary function that does all the work of config_get_lines.
+ * <b>recursion_level</b> is the count of how many nested %includes we have.
+ * Returns the a pointer to the last element of the <b>result</b> in
+ * <b>last</b>. */
+static int
+config_get_lines_aux(const char *string, config_line_t **result, int extended,
+                     int allow_include, int *has_include, int recursion_level,
+                     config_line_t **last)
+{
+  config_line_t *list = NULL, **next, *list_last = NULL;
+  char *k, *v;
+  const char *parse_err;
+  int include_used = 0;
+
+  if (recursion_level > MAX_INCLUDE_RECURSION_LEVEL) {
+    log_warn(LD_CONFIG, "Error while parsing configuration: more than %d "
+             "nested %%includes.", MAX_INCLUDE_RECURSION_LEVEL);
+    return -1;
+  }
+
+  next = &list;
+  do {
+    k = v = NULL;
+    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
+    if (!string) {
+      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
+               parse_err?parse_err:"<unknown>");
+      config_free_lines(list);
+      tor_free(k);
+      tor_free(v);
+      return -1;
+    }
+    if (k && v) {
+      unsigned command = CONFIG_LINE_NORMAL;
+      if (extended) {
+        if (k[0] == '+') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          command = CONFIG_LINE_APPEND;
+        } else if (k[0] == '/') {
+          char *k_new = tor_strdup(k+1);
+          tor_free(k);
+          k = k_new;
+          tor_free(v);
+          v = tor_strdup("");
+          command = CONFIG_LINE_CLEAR;
+        }
+      }
+
+      if (allow_include && !strcmp(k, "%include")) {
+        tor_free(k);
+        include_used = 1;
+
+        config_line_t *include_list;
+        if (config_process_include(v, recursion_level, extended, &include_list,
+                                   &list_last) < 0) {
+          log_warn(LD_CONFIG, "Error reading included configuration "
+                   "file or directory: \"%s\".", v);
+          config_free_lines(list);
+          tor_free(v);
+          return -1;
+        }
+        *next = include_list;
+        if (list_last)
+          next = &list_last->next;
+        tor_free(v);
+      } else {
+        /* This list can get long, so we keep a pointer to the end of it
+         * rather than using config_line_append over and over and getting
+         * n^2 performance. */
+        *next = tor_malloc_zero(sizeof(**next));
+        (*next)->key = k;
+        (*next)->value = v;
+        (*next)->next = NULL;
+        (*next)->command = command;
+        list_last = *next;
+        next = &((*next)->next);
+      }
+    } else {
+      tor_free(k);
+      tor_free(v);
+    }
+  } while (*string);
+
+  if (last) {
+    *last = list_last;
+  }
+  if (has_include) {
+    *has_include = include_used;
+  }
+  *result = list;
+  return 0;
+}
+
+/** Helper: parse the config string and strdup into key/value
+ * strings. Set *result to the list, or NULL if parsing the string
+ * failed. Set *has_include to 1 if <b>result</b> has values from
+ * %included files.  Return 0 on success, -1 on failure. Warn and ignore any
+ * misformatted lines.
+ *
+ * If <b>extended</b> is set, then treat keys beginning with / and with + as
+ * indicating "clear" and "append" respectively. */
+int
+config_get_lines_include(const char *string, config_line_t **result,
+                         int extended, int *has_include)
+{
+  return config_get_lines_aux(string, result, extended, 1, has_include, 1,
+                              NULL);
+}
+
+/** Same as config_get_lines_include but does not allow %include */
+int
+config_get_lines(const char *string, config_line_t **result, int extended)
+{
+  return config_get_lines_aux(string, result, extended, 0, NULL, 1, NULL);
+}
+
+/** Adds a list of configuration files present on <b>path</b> to
+ * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
+ * only that file will be added to <b>file_list</b>. If it is a directory,
+ * all paths for files on that directory root (no recursion) except for files
+ * whose name starts with a dot will be added to <b>file_list</b>.
+ * Return 0 on success, -1 on failure. Ignores empty files.
+ */
+static smartlist_t *
+config_get_file_list(const char *path)
+{
+  smartlist_t *file_list = smartlist_new();
+  file_status_t file_type = file_status(path);
+  if (file_type == FN_FILE) {
+    smartlist_add_strdup(file_list, path);
+    return file_list;
+  } else if (file_type == FN_DIR) {
+    smartlist_t *all_files = tor_listdir(path);
+    if (!all_files) {
+      smartlist_free(file_list);
+      return NULL;
+    }
+    smartlist_sort_strings(all_files);
+    SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
+      if (f[0] == '.') {
+        tor_free(f);
+        continue;
+      }
+
+      char *fullname;
+      tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
+      tor_free(f);
+
+      if (file_status(fullname) != FN_FILE) {
+        tor_free(fullname);
+        continue;
+      }
+      smartlist_add(file_list, fullname);
+    } SMARTLIST_FOREACH_END(f);
+    smartlist_free(all_files);
+    return file_list;
+  } else if (file_type == FN_EMPTY) {
+      return file_list;
+  } else {
+    smartlist_free(file_list);
+    return NULL;
+  }
+}
+
+/** Creates a list of config lines present on included <b>path</b>.
+ * Set <b>list</b> to the list and <b>list_last</b> to the last element of
+ * <b>list</b>. Return 0 on success, -1 on failure. */
+static int
+config_get_included_list(const char *path, int recursion_level, int extended,
+                         config_line_t **list, config_line_t **list_last)
+{
+  char *included_conf = read_file_to_str(path, 0, NULL);
+  if (!included_conf) {
+    return -1;
+  }
+
+  if (config_get_lines_aux(included_conf, list, extended, 1, NULL,
+                           recursion_level+1, list_last) < 0) {
+    tor_free(included_conf);
+    return -1;
+  }
+
+  tor_free(included_conf);
+  return 0;
+}
+
+/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
+ * list of configuration settings obtained and <b>list_last</b> to the last
+ * element of the same list. Return 0 on success, -1 on failure. */
+static int
+config_process_include(const char *path, int recursion_level, int extended,
+                       config_line_t **list, config_line_t **list_last)
+{
+  config_line_t *ret_list = NULL;
+  config_line_t **next = &ret_list;
+#if 0
+  // Disabled -- we already unescape_string() on the result. */
+  char *unquoted_path = get_unquoted_path(path);
+  if (!unquoted_path) {
+    return -1;
+  }
+
+  smartlist_t *config_files = config_get_file_list(unquoted_path);
+  if (!config_files) {
+    tor_free(unquoted_path);
+    return -1;
+  }
+  tor_free(unquoted_path);
+#endif
+  smartlist_t *config_files = config_get_file_list(path);
+  if (!config_files) {
+    return -1;
+  }
+
+  int rv = -1;
+  SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
+    config_line_t *included_list = NULL;
+    if (config_get_included_list(config_file, recursion_level, extended,
+                                  &included_list, list_last) < 0) {
+      goto done;
+    }
+
+    *next = included_list;
+    if (*list_last)
+      next = &(*list_last)->next;
+
+  } SMARTLIST_FOREACH_END(config_file);
+  *list = ret_list;
+  rv = 0;
+
+ done:
+  SMARTLIST_FOREACH(config_files, char *, f, tor_free(f));
+  smartlist_free(config_files);
+  return rv;
+}
+
+/**
+ * Free all the configuration lines on the linked list <b>front</b>.
+ */
+void
+config_free_lines(config_line_t *front)
+{
+  config_line_t *tmp;
+
+  while (front) {
+    tmp = front;
+    front = tmp->next;
+
+    tor_free(tmp->key);
+    tor_free(tmp->value);
+    tor_free(tmp);
+  }
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
+config_line_t *
+config_lines_dup(const config_line_t *inp)
+{
+  return config_lines_dup_and_filter(inp, NULL);
+}
+
+/** Return a newly allocated deep copy of the lines in <b>inp</b>,
+ * but only the ones that match <b>key</b>. */
+config_line_t *
+config_lines_dup_and_filter(const config_line_t *inp,
+                            const char *key)
+{
+  config_line_t *result = NULL;
+  config_line_t **next_out = &result;
+  while (inp) {
+    if (key && strcasecmpstart(inp->key, key)) {
+      inp = inp->next;
+      continue;
+    }
+    *next_out = tor_malloc_zero(sizeof(config_line_t));
+    (*next_out)->key = tor_strdup(inp->key);
+    (*next_out)->value = tor_strdup(inp->value);
+    inp = inp->next;
+    next_out = &((*next_out)->next);
+  }
+  (*next_out) = NULL;
+  return result;
+}
+
+/** Return true iff a and b contain identical keys and values in identical
+ * order. */
+int
+config_lines_eq(config_line_t *a, config_line_t *b)
+{
+  while (a && b) {
+    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
+      return 0;
+    a = a->next;
+    b = b->next;
+  }
+  if (a || b)
+    return 0;
+  return 1;
+}
+
+/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
+int
+config_count_key(const config_line_t *a, const char *key)
+{
+  int n = 0;
+  while (a) {
+    if (!strcasecmp(a->key, key)) {
+      ++n;
+    }
+    a = a->next;
+  }
+  return n;
+}
+
+/** Given a string containing part of a configuration file or similar format,
+ * advance past comments and whitespace and try to parse a single line.  If we
+ * parse a line successfully, set *<b>key_out</b> to a new string holding the
+ * key portion and *<b>value_out</b> to a new string holding the value portion
+ * of the line, and return a pointer to the start of the next line.  If we run
+ * out of data, return a pointer to the end of the string.  If we encounter an
+ * error, return NULL and set *<b>err_out</b> (if provided) to an error
+ * message.
+ */
+const char *
+parse_config_line_from_str_verbose(const char *line, char **key_out,
+                                   char **value_out,
+                                   const char **err_out)
+{
+  /*
+    See torrc_format.txt for a description of the (silly) format this parses.
+   */
+  const char *key, *val, *cp;
+  int continuation = 0;
+
+  tor_assert(key_out);
+  tor_assert(value_out);
+
+  *key_out = *value_out = NULL;
+  key = val = NULL;
+  /* Skip until the first keyword. */
+  while (1) {
+    while (TOR_ISSPACE(*line))
+      ++line;
+    if (*line == '#') {
+      while (*line && *line != '\n')
+        ++line;
+    } else {
+      break;
+    }
+  }
+
+  if (!*line) { /* End of string? */
+    *key_out = *value_out = NULL;
+    return line;
+  }
+
+  /* Skip until the next space or \ followed by newline. */
+  key = line;
+  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
+         ! (line[0] == '\\' && line[1] == '\n'))
+    ++line;
+  *key_out = tor_strndup(key, line-key);
+
+  /* Skip until the value. */
+  while (*line == ' ' || *line == '\t')
+    ++line;
+
+  val = line;
+
+  /* Find the end of the line. */
+  if (*line == '\"') { // XXX No continuation handling is done here
+    if (!(line = unescape_string(line, value_out, NULL))) {
+      if (err_out)
+        *err_out = "Invalid escape sequence in quoted string";
+      return NULL;
+    }
+    while (*line == ' ' || *line == '\t')
+      ++line;
+    if (*line == '\r' && *(++line) == '\n')
+      ++line;
+    if (*line && *line != '#' && *line != '\n') {
+      if (err_out)
+        *err_out = "Excess data after quoted string";
+      return NULL;
+    }
+  } else {
+    /* Look for the end of the line. */
+    while (*line && *line != '\n' && (*line != '#' || continuation)) {
+      if (*line == '\\' && line[1] == '\n') {
+        continuation = 1;
+        line += 2;
+      } else if (*line == '#') {
+        do {
+          ++line;
+        } while (*line && *line != '\n');
+        if (*line == '\n')
+          ++line;
+      } else {
+        ++line;
+      }
+    }
+
+    if (*line == '\n') {
+      cp = line++;
+    } else {
+      cp = line;
+    }
+    /* Now back cp up to be the last nonspace character */
+    while (cp>val && TOR_ISSPACE(*(cp-1)))
+      --cp;
+
+    tor_assert(cp >= val);
+
+    /* Now copy out and decode the value. */
+    *value_out = tor_strndup(val, cp-val);
+    if (continuation) {
+      char *v_out, *v_in;
+      v_out = v_in = *value_out;
+      while (*v_in) {
+        if (*v_in == '#') {
+          do {
+            ++v_in;
+          } while (*v_in && *v_in != '\n');
+          if (*v_in == '\n')
+            ++v_in;
+        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
+          v_in += 2;
+        } else {
+          *v_out++ = *v_in++;
+        }
+      }
+      *v_out = '\0';
+    }
+  }
+
+  if (*line == '#') {
+    do {
+      ++line;
+    } while (*line && *line != '\n');
+  }
+  while (TOR_ISSPACE(*line)) ++line;
+
+  return line;
+}
+
diff --git a/src/common/confline.h b/src/common/confline.h
new file mode 100644
index 0000000000..eb863e8fd8
--- /dev/null
+++ b/src/common/confline.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONFLINE_H
+#define TOR_CONFLINE_H
+
+/** Ordinary configuration line. */
+#define CONFIG_LINE_NORMAL 0
+/** Appends to previous configuration for the same option, even if we
+ * would ordinary replace it. */
+#define CONFIG_LINE_APPEND 1
+/* Removes all previous configuration for an option. */
+#define CONFIG_LINE_CLEAR 2
+
+#define MAX_INCLUDE_RECURSION_LEVEL 31
+
+/** A linked list of lines in a config file, or elsewhere */
+typedef struct config_line_t {
+  char *key;
+  char *value;
+  struct config_line_t *next;
+
+  /** What special treatment (if any) does this line require? */
+  unsigned int command:2;
+  /** If true, subsequent assignments to this linelist should replace
+   * it, not extend it.  Set only on the first item in a linelist in an
+   * or_options_t. */
+  unsigned int fragile:1;
+} config_line_t;
+
+void config_line_append(config_line_t **lst,
+                        const char *key, const char *val);
+void config_line_prepend(config_line_t **lst,
+                         const char *key, const char *val);
+config_line_t *config_lines_dup(const config_line_t *inp);
+config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
+                                           const char *key);
+const config_line_t *config_line_find(const config_line_t *lines,
+                                      const char *key);
+int config_lines_eq(config_line_t *a, config_line_t *b);
+int config_count_key(const config_line_t *a, const char *key);
+int config_get_lines(const char *string, config_line_t **result, int extended);
+int config_get_lines_include(const char *string, config_line_t **result,
+                             int extended, int *has_include);
+void config_free_lines(config_line_t *front);
+const char *parse_config_line_from_str_verbose(const char *line,
+                                       char **key_out, char **value_out,
+                                       const char **err_out);
+#endif
+
diff --git a/src/common/container.c b/src/common/container.c
index ec59dccf62..689e7e55e9 100644
--- a/src/common/container.c
+++ b/src/common/container.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -132,6 +132,24 @@ smartlist_remove(smartlist_t *sl, const void *element)
     }
 }
 
+/** As <b>smartlist_remove</b>, but do not change the order of
+ * any elements not removed */
+void
+smartlist_remove_keeporder(smartlist_t *sl, const void *element)
+{
+  int i, j, num_used_orig = sl->num_used;
+  if (element == NULL)
+    return;
+
+  for (i=j=0; j < num_used_orig; ++j) {
+    if (sl->list[j] == element) {
+      --sl->num_used;
+    } else {
+      sl->list[i++] = sl->list[j];
+    }
+  }
+}
+
 /** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
  * return NULL. */
 void *
diff --git a/src/common/container.h b/src/common/container.h
index 71495b660a..db68d892f5 100644
--- a/src/common/container.h
+++ b/src/common/container.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CONTAINER_H
@@ -33,6 +33,7 @@ void smartlist_clear(smartlist_t *sl);
 void smartlist_add(smartlist_t *sl, void *element);
 void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
 void smartlist_remove(smartlist_t *sl, const void *element);
+void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
 void *smartlist_pop_last(smartlist_t *sl);
 void smartlist_reverse(smartlist_t *sl);
 void smartlist_string_remove(smartlist_t *sl, const char *element);
@@ -223,6 +224,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
 
 #define SMARTLIST_FOREACH_END(var)              \
     var = NULL;                                 \
+    (void) var ## _sl_idx;                      \
   } STMT_END
 
 /**
diff --git a/src/common/crypto.c b/src/common/crypto.c
index f8495bb107..d2801e0d45 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -467,7 +467,7 @@ crypto_new_pk_from_rsa_(RSA *rsa)
   return env;
 }
 
-/** Helper, used by tor-checkkey.c and tor-gencert.c.  Return the RSA from a
+/** Helper, used by tor-gencert.c.  Return the RSA from a
  * crypto_pk_t. */
 RSA *
 crypto_pk_get_rsa_(crypto_pk_t *env)
@@ -479,7 +479,7 @@ crypto_pk_get_rsa_(crypto_pk_t *env)
  * private is set, include the private-key portion of the key. Return a valid
  * pointer on success, and NULL on failure. */
 MOCK_IMPL(EVP_PKEY *,
-          crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private))
+crypto_pk_get_evp_pkey_,(crypto_pk_t *env, int private))
 {
   RSA *key = NULL;
   EVP_PKEY *pkey = NULL;
@@ -516,7 +516,7 @@ crypto_dh_get_dh_(crypto_dh_t *dh)
  * be set.
  */
 MOCK_IMPL(crypto_pk_t *,
-          crypto_pk_new,(void))
+crypto_pk_new,(void))
 {
   RSA *rsa;
 
@@ -606,7 +606,7 @@ crypto_cipher_free(crypto_cipher_t *env)
  * Return 0 on success, -1 on failure.
  */
 MOCK_IMPL(int,
-          crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits))
+crypto_pk_generate_key_with_bits,(crypto_pk_t *env, int bits))
 {
   tor_assert(env);
 
@@ -1122,10 +1122,10 @@ crypto_pk_private_decrypt(crypto_pk_t *env, char *to,
  * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be
  * at least the length of the modulus of <b>env</b>.
  */
-int
-crypto_pk_public_checksig(const crypto_pk_t *env, char *to,
-                          size_t tolen,
-                          const char *from, size_t fromlen)
+MOCK_IMPL(int,
+crypto_pk_public_checksig,(const crypto_pk_t *env, char *to,
+                           size_t tolen,
+                           const char *from, size_t fromlen))
 {
   int r;
   tor_assert(env);
@@ -1149,9 +1149,10 @@ crypto_pk_public_checksig(const crypto_pk_t *env, char *to,
  * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for
  * SHA1(data).  Else return -1.
  */
-int
-crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data,
-                               size_t datalen, const char *sig, size_t siglen)
+MOCK_IMPL(int,
+crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data,
+                                  size_t datalen, const char *sig,
+                                  size_t siglen))
 {
   char digest[DIGEST_LEN];
   char *buf;
@@ -1521,7 +1522,7 @@ crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out)
   if (crypto_pk_get_digest(pk, digest)) {
     return -1;
   }
-  if (crypto_digest(hashed_digest, digest, DIGEST_LEN)) {
+  if (crypto_digest(hashed_digest, digest, DIGEST_LEN) < 0) {
     return -1;
   }
   base16_encode(fp_out, FINGERPRINT_LEN + 1, hashed_digest, DIGEST_LEN);
@@ -1715,19 +1716,21 @@ crypto_cipher_decrypt_with_iv(const char *key,
 
 /** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
  * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, 1 on failure.
+ * Return 0 on success, -1 on failure.
  */
 int
 crypto_digest(char *digest, const char *m, size_t len)
 {
   tor_assert(m);
   tor_assert(digest);
-  return (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL);
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL)
+    return -1;
+  return 0;
 }
 
 /** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
  * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, 1 on failure. */
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
 int
 crypto_digest256(char *digest, const char *m, size_t len,
                  digest_algorithm_t algorithm)
@@ -1735,16 +1738,22 @@ crypto_digest256(char *digest, const char *m, size_t len,
   tor_assert(m);
   tor_assert(digest);
   tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
   if (algorithm == DIGEST_SHA256)
-    return (SHA256((const uint8_t*)m,len,(uint8_t*)digest) == NULL);
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
   else
-    return (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-            == -1);
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+
+  if (!ret)
+    return -1;
+  return 0;
 }
 
 /** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
  * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, 1 on failure. */
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
 int
 crypto_digest512(char *digest, const char *m, size_t len,
                  digest_algorithm_t algorithm)
@@ -1752,12 +1761,18 @@ crypto_digest512(char *digest, const char *m, size_t len,
   tor_assert(m);
   tor_assert(digest);
   tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
   if (algorithm == DIGEST_SHA512)
-    return (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-            == NULL);
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
   else
-    return (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-            == -1);
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+
+  if (!ret)
+    return -1;
+  return 0;
 }
 
 /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
@@ -2124,6 +2139,35 @@ crypto_hmac_sha256(char *hmac_out,
   tor_assert(rv);
 }
 
+/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
+ * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
+ * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
+ * <b>mac_out</b>. This function can't fail. */
+void
+crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                    const uint8_t *key, size_t key_len,
+                    const uint8_t *msg, size_t msg_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t key_len_netorder = tor_htonll(key_len);
+
+  tor_assert(mac_out);
+  tor_assert(key);
+  tor_assert(msg);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+
+  /* Order matters here that is any subsystem using this function should
+   * expect this very precise ordering in the MAC construction. */
+  crypto_digest_add_bytes(digest, (const char *) &key_len_netorder,
+                          sizeof(key_len_netorder));
+  crypto_digest_add_bytes(digest, (const char *) key, key_len);
+  crypto_digest_add_bytes(digest, (const char *) msg, msg_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, len_out);
+  crypto_digest_free(digest);
+}
+
 /** Internal state for a eXtendable-Output Function (XOF). */
 struct crypto_xof_t {
   keccak_state s;
@@ -2643,7 +2687,7 @@ crypto_expand_key_material_TAP(const uint8_t *key_in, size_t key_in_len,
   for (cp = key_out, i=0; cp < key_out+key_out_len;
        ++i, cp += DIGEST_LEN) {
     tmp[key_in_len] = i;
-    if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1))
+    if (crypto_digest((char*)digest, (const char *)tmp, key_in_len+1) < 0)
       goto exit;
     memcpy(cp, digest, MIN(DIGEST_LEN, key_out_len-(cp-key_out)));
   }
@@ -2867,7 +2911,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
   size_t n;
 
   for (i = 0; filenames[i]; ++i) {
-    log_debug(LD_FS, "Opening %s for entropy", filenames[i]);
+    log_debug(LD_FS, "Considering %s for entropy", filenames[i]);
     fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0);
     if (fd<0) continue;
     log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]);
@@ -3430,3 +3474,15 @@ crypto_global_cleanup(void)
 
 /** @} */
 
+#ifdef USE_DMALLOC
+/** Tell the crypto library to use Tor's allocation functions rather than
+ * calling libc's allocation functions directly. Return 0 on success, -1
+ * on failure. */
+int
+crypto_use_tor_alloc_functions(void)
+{
+  int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
+  return r ? 0 : -1;
+}
+#endif
+
diff --git a/src/common/crypto.h b/src/common/crypto.h
index 116e0a62fd..c70d91c262 100644
--- a/src/common/crypto.h
+++ b/src/common/crypto.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -59,10 +59,12 @@
 #define DIGEST256_LEN 32
 /** Length of the output of our 64-bit optimized message digests (SHA512). */
 #define DIGEST512_LEN 64
-/** Length of our symmetric cipher's keys. */
+/** Length of our symmetric cipher's keys of 128-bit. */
 #define CIPHER_KEY_LEN 16
-/** Length of our symmetric cipher's IV. */
+/** Length of our symmetric cipher's IV of 128-bit. */
 #define CIPHER_IV_LEN 16
+/** Length of our symmetric cipher's keys of 256-bit. */
+#define CIPHER256_KEY_LEN 32
 /** Length of our public keys. */
 #define PK_BYTES (1024/8)
 /** Length of our DH keys. */
@@ -129,6 +131,10 @@ int crypto_early_init(void) ATTR_WUR;
 int crypto_global_init(int hardwareAccel,
                        const char *accelName,
                        const char *accelPath) ATTR_WUR;
+#ifdef USE_DMALLOC
+int crypto_use_tor_alloc_functions(void);
+#endif
+
 void crypto_thread_cleanup(void);
 int crypto_global_cleanup(void);
 
@@ -178,10 +184,12 @@ int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen,
 int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen,
                               const char *from, size_t fromlen,
                               int padding, int warnOnFailure);
-int crypto_pk_public_checksig(const crypto_pk_t *env, char *to, size_t tolen,
-                              const char *from, size_t fromlen);
-int crypto_pk_public_checksig_digest(crypto_pk_t *env, const char *data,
-                               size_t datalen, const char *sig, size_t siglen);
+MOCK_DECL(int, crypto_pk_public_checksig,(const crypto_pk_t *env,
+                                          char *to, size_t tolen,
+                                          const char *from, size_t fromlen));
+MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env,
+                                         const char *data, size_t datalen,
+                                         const char *sig, size_t siglen));
 int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
                            const char *from, size_t fromlen);
 int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
@@ -255,6 +263,10 @@ void crypto_digest_assign(crypto_digest_t *into,
 void crypto_hmac_sha256(char *hmac_out,
                         const char *key, size_t key_len,
                         const char *msg, size_t msg_len);
+void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
+                         const uint8_t *key, size_t key_len,
+                         const uint8_t *msg, size_t msg_len);
+
 crypto_xof_t *crypto_xof_new(void);
 void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
 void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c
index fcbee3aba2..b99f13a93b 100644
--- a/src/common/crypto_curve25519.c
+++ b/src/common/crypto_curve25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -80,7 +80,7 @@ curve25519_impl(uint8_t *output, const uint8_t *secret,
 /**
  * Helper function: Multiply the scalar "secret" by the Curve25519
  * basepoint (X=9), and store the result in "output".  Return 0 on
- * success, -1 on false.
+ * success, -1 on failure.
  */
 STATIC int
 curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret)
diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h
index 4011820949..e7790edac0 100644
--- a/src/common/crypto_curve25519.h
+++ b/src/common/crypto_curve25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_CURVE25519_H
diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c
index 30ed772274..188e18c710 100644
--- a/src/common/crypto_ed25519.c
+++ b/src/common/crypto_ed25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,6 +15,7 @@
  * keys to and from the corresponding Curve25519 keys.
  */
 
+#define CRYPTO_ED25519_PRIVATE
 #include "orconfig.h"
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -31,10 +32,7 @@
 #include "ed25519/ref10/ed25519_ref10.h"
 #include "ed25519/donna/ed25519_donna_tor.h"
 
-#include <openssl/sha.h>
-
 static void pick_ed25519_impl(void);
-static int ed25519_impl_spot_check(void);
 
 /** An Ed25519 implementation, as a set of function pointers. */
 typedef struct {
@@ -211,6 +209,14 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
   return 0;
 }
 
+/** Return true iff 'pubkey' is set to zero (eg to indicate that it is not
+ * set). */
+int
+ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
+{
+  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+}
+
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
  * string <b>prefix_str</b>. Set <b>final_msg_len_out</b> to the size of the
  * final array. If an error occured, return NULL. It's the resonsibility of the
@@ -267,11 +273,11 @@ ed25519_sign(ed25519_signature_t *signature_out,
  * Like ed25519_sign(), but also prefix <b>msg</b> with <b>prefix_str</b>
  * before signing. <b>prefix_str</b> must be a NUL-terminated string.
  */
-int
-ed25519_sign_prefixed(ed25519_signature_t *signature_out,
-                      const uint8_t *msg, size_t msg_len,
-                      const char *prefix_str,
-                      const ed25519_keypair_t *keypair)
+MOCK_IMPL(int,
+ed25519_sign_prefixed,(ed25519_signature_t *signature_out,
+                       const uint8_t *msg, size_t msg_len,
+                       const char *prefix_str,
+                       const ed25519_keypair_t *keypair))
 {
   int retval;
   size_t prefixed_msg_len;
@@ -300,10 +306,10 @@ ed25519_sign_prefixed(ed25519_signature_t *signature_out,
  *
  * Return 0 if the signature is valid; -1 if it isn't.
  */
-int
-ed25519_checksig(const ed25519_signature_t *signature,
-                 const uint8_t *msg, size_t len,
-                 const ed25519_public_key_t *pubkey)
+MOCK_IMPL(int,
+ed25519_checksig,(const ed25519_signature_t *signature,
+                  const uint8_t *msg, size_t len,
+                  const ed25519_public_key_t *pubkey))
 {
   return
     get_ed_impl()->open(signature->sig, msg, len, pubkey->pubkey) < 0 ? -1 : 0;
@@ -346,10 +352,10 @@ ed25519_checksig_prefixed(const ed25519_signature_t *signature,
  * was valid. Otherwise return -N, where N is the number of invalid
  * signatures.
  */
-int
-ed25519_checksig_batch(int *okay_out,
-                       const ed25519_checkable_t *checkable,
-                       int n_checkable)
+MOCK_IMPL(int,
+ed25519_checksig_batch,(int *okay_out,
+                        const ed25519_checkable_t *checkable,
+                        int n_checkable))
 {
   int i, res;
   const ed25519_impl_t *impl = get_ed_impl();
@@ -434,14 +440,16 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
 {
   const char string[] = "Derive high part of ed25519 key from curve25519 key";
   ed25519_public_key_t pubkey_check;
-  SHA512_CTX ctx;
-  uint8_t sha512_output[64];
+  crypto_digest_t *ctx;
+  uint8_t sha512_output[DIGEST512_LEN];
 
   memcpy(out->seckey.seckey, inp->seckey.secret_key, 32);
-  SHA512_Init(&ctx);
-  SHA512_Update(&ctx, out->seckey.seckey, 32);
-  SHA512_Update(&ctx, string, sizeof(string));
-  SHA512_Final(sha512_output, &ctx);
+
+  ctx = crypto_digest512_new(DIGEST_SHA512);
+  crypto_digest_add_bytes(ctx, (const char*)out->seckey.seckey, 32);
+  crypto_digest_add_bytes(ctx, (const char*)string, sizeof(string));
+  crypto_digest_get_digest(ctx, (char *)sha512_output, sizeof(sha512_output));
+  crypto_digest_free(ctx);
   memcpy(out->seckey.seckey + 32, sha512_output, 32);
 
   ed25519_public_key_generate(&out->pubkey, &out->seckey);
@@ -620,10 +628,22 @@ ed25519_pubkey_eq(const ed25519_public_key_t *key1,
   return tor_memeq(key1->pubkey, key2->pubkey, ED25519_PUBKEY_LEN);
 }
 
+/**
+ * Set <b>dest</b> to contain the same key as <b>src</b>.
+ */
+void
+ed25519_pubkey_copy(ed25519_public_key_t *dest,
+                    const ed25519_public_key_t *src)
+{
+  tor_assert(dest);
+  tor_assert(src);
+  memcpy(dest, src, sizeof(ed25519_public_key_t));
+}
+
 /** Check whether the given Ed25519 implementation seems to be working.
  * If so, return 0; otherwise return -1. */
-static int
-ed25519_impl_spot_check(void)
+MOCK_IMPL(STATIC int,
+ed25519_impl_spot_check,(void))
 {
   static const uint8_t alicesk[32] = {
     0xc5,0xaa,0x8d,0xf4,0x3f,0x9f,0x83,0x7b,
diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h
index 31afc49ccc..77a3313adc 100644
--- a/src/common/crypto_ed25519.h
+++ b/src/common/crypto_ed25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_ED25519_H
@@ -51,21 +51,24 @@ int ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong);
 int ed25519_sign(ed25519_signature_t *signature_out,
                  const uint8_t *msg, size_t len,
                  const ed25519_keypair_t *key);
-int ed25519_checksig(const ed25519_signature_t *signature,
-                     const uint8_t *msg, size_t len,
-                     const ed25519_public_key_t *pubkey);
+MOCK_DECL(int,ed25519_checksig,(const ed25519_signature_t *signature,
+                                const uint8_t *msg, size_t len,
+                                const ed25519_public_key_t *pubkey));
+
+MOCK_DECL(int,
+ed25519_sign_prefixed,(ed25519_signature_t *signature_out,
+                       const uint8_t *msg, size_t len,
+                       const char *prefix_str,
+                       const ed25519_keypair_t *keypair));
 
-int
-ed25519_sign_prefixed(ed25519_signature_t *signature_out,
-                      const uint8_t *msg, size_t len,
-                      const char *prefix_str,
-                      const ed25519_keypair_t *keypair);
 int
 ed25519_checksig_prefixed(const ed25519_signature_t *signature,
                           const uint8_t *msg, size_t len,
                           const char *prefix_str,
                           const ed25519_public_key_t *pubkey);
 
+int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey);
+
 /**
  * A collection of information necessary to check an Ed25519 signature. Used
  * for batch verification.
@@ -81,9 +84,9 @@ typedef struct {
   size_t len;
 } ed25519_checkable_t;
 
-int ed25519_checksig_batch(int *okay_out,
-                           const ed25519_checkable_t *checkable,
-                           int n_checkable);
+MOCK_DECL(int, ed25519_checksig_batch,(int *okay_out,
+                                       const ed25519_checkable_t *checkable,
+                                       int n_checkable));
 
 int ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
                                             int *signbit_out,
@@ -118,6 +121,8 @@ void ed25519_keypair_free(ed25519_keypair_t *kp);
 
 int ed25519_pubkey_eq(const ed25519_public_key_t *key1,
                       const ed25519_public_key_t *key2);
+void ed25519_pubkey_copy(ed25519_public_key_t *dest,
+                         const ed25519_public_key_t *src);
 
 void ed25519_set_impl_params(int use_donna);
 void ed25519_init(void);
@@ -127,5 +132,9 @@ void crypto_ed25519_testing_force_impl(const char *name);
 void crypto_ed25519_testing_restore_impl(void);
 #endif
 
+#ifdef CRYPTO_ED25519_PRIVATE
+MOCK_DECL(STATIC int, ed25519_impl_spot_check, (void));
+#endif
+
 #endif
 
diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c
index 2f6d847c83..1d090a8770 100644
--- a/src/common/crypto_format.c
+++ b/src/common/crypto_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -161,6 +161,27 @@ curve25519_public_from_base64(curve25519_public_key_t *pkey,
   }
 }
 
+/** For logging convenience: Convert <b>pkey</b> to a statically allocated
+ * base64 string and return it. Not threadsafe. Format not meant to be
+ * computer-readable; it may change in the future. Subsequent calls invalidate
+ * previous returns. */
+const char *
+ed25519_fmt(const ed25519_public_key_t *pkey)
+{
+  static char formatted[ED25519_BASE64_LEN+1];
+  if (pkey) {
+    if (ed25519_public_key_is_zero(pkey)) {
+      strlcpy(formatted, "<unset>", sizeof(formatted));
+    } else {
+      int r = ed25519_public_to_base64(formatted, pkey);
+      tor_assert(!r);
+    }
+  } else {
+    strlcpy(formatted, "<null>", sizeof(formatted));
+  }
+  return formatted;
+}
+
 /** Try to decode the string <b>input</b> into an ed25519 public key. On
  * success, store the value in <b>pkey</b> and return 0. Otherwise return
  * -1. */
diff --git a/src/common/crypto_format.h b/src/common/crypto_format.h
index 012e228cc4..390916cf04 100644
--- a/src/common/crypto_format.h
+++ b/src/common/crypto_format.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_FORMAT_H
@@ -28,6 +28,7 @@ int ed25519_public_from_base64(ed25519_public_key_t *pkey,
                                const char *input);
 int ed25519_public_to_base64(char *output,
                              const ed25519_public_key_t *pkey);
+const char *ed25519_fmt(const ed25519_public_key_t *pkey);
 
 /* XXXX move these to crypto_format.h */
 #define ED25519_SIG_BASE64_LEN 86
diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c
index 31e37c007d..db8892e376 100644
--- a/src/common/crypto_pwbox.c
+++ b/src/common/crypto_pwbox.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c
index 5dbd2ad91f..076df815a9 100644
--- a/src/common/crypto_s2k.c
+++ b/src/common/crypto_s2k.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/crypto_s2k.h b/src/common/crypto_s2k.h
index 9b186450b1..04212b868a 100644
--- a/src/common/crypto_s2k.h
+++ b/src/common/crypto_s2k.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CRYPTO_S2K_H_INCLUDED
diff --git a/src/common/di_ops.c b/src/common/di_ops.c
index 4ed49e1164..e47998107d 100644
--- a/src/common/di_ops.c
+++ b/src/common/di_ops.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/di_ops.h b/src/common/di_ops.h
index 0a154302bf..e174fcc4e4 100644
--- a/src/common/di_ops.h
+++ b/src/common/di_ops.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/handles.h b/src/common/handles.h
index 1ee2322579..6d7262ab80 100644
--- a/src/common/handles.h
+++ b/src/common/handles.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/include.am b/src/common/include.am
index cb307e9d5f..d12895b107 100644
--- a/src/common/include.am
+++ b/src/common/include.am
@@ -85,6 +85,7 @@ LIBOR_A_SRC = \
   src/common/compat.c					\
   src/common/compat_threads.c				\
   src/common/compat_time.c				\
+  src/common/confline.c					\
   src/common/container.c				\
   src/common/log.c					\
   src/common/memarea.c					\
@@ -94,21 +95,31 @@ LIBOR_A_SRC = \
   src/common/util_format.c				\
   src/common/util_process.c				\
   src/common/sandbox.c					\
+  src/common/storagedir.c				\
   src/common/workqueue.c				\
   $(libor_extra_source)					\
   $(threads_impl_source)				\
   $(readpassphrase_source)
 
+if USE_RUST
+else
+LIBOR_A_SRC += src/common/compat_rust.c
+endif
+
 src/common/src_common_libor_testing_a-log.$(OBJEXT) \
   src/common/log.$(OBJEXT): micro-revision.i
 
 LIBOR_CRYPTO_A_SRC = \
   src/common/aes.c		\
+  src/common/compress.c	\
+  src/common/compress_lzma.c	\
+  src/common/compress_none.c	\
+  src/common/compress_zlib.c	\
+  src/common/compress_zstd.c	\
   src/common/crypto.c		\
   src/common/crypto_pwbox.c     \
   src/common/crypto_s2k.c	\
   src/common/crypto_format.c	\
-  src/common/torgzip.c		\
   src/common/tortls.c		\
   src/common/crypto_curve25519.c \
   src/common/crypto_ed25519.c
@@ -143,8 +154,15 @@ COMMONHEADERS = \
   src/common/compat.h				\
   src/common/compat_libevent.h			\
   src/common/compat_openssl.h			\
+  src/common/compat_rust.h			\
   src/common/compat_threads.h			\
   src/common/compat_time.h			\
+  src/common/compress.h				\
+  src/common/compress_lzma.h			\
+  src/common/compress_none.h			\
+  src/common/compress_zlib.h			\
+  src/common/compress_zstd.h			\
+  src/common/confline.h				\
   src/common/container.h			\
   src/common/crypto.h				\
   src/common/crypto_curve25519.h		\
@@ -159,9 +177,9 @@ COMMONHEADERS = \
   src/common/procmon.h				\
   src/common/pubsub.h				\
   src/common/sandbox.h				\
+  src/common/storagedir.h			\
   src/common/testsupport.h			\
   src/common/timers.h				\
-  src/common/torgzip.h				\
   src/common/torint.h				\
   src/common/torlog.h				\
   src/common/tortls.h				\
diff --git a/src/common/log.c b/src/common/log.c
index 4db1c9f0d0..87c260799d 100644
--- a/src/common/log.c
+++ b/src/common/log.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -682,7 +682,7 @@ tor_log_get_logfile_names(smartlist_t *out)
       continue;
     if (lf->filename == NULL)
       continue;
-    smartlist_add(out, tor_strdup(lf->filename));
+    smartlist_add_strdup(out, lf->filename);
   }
 
   UNLOCK_LOGS();
@@ -1086,7 +1086,7 @@ add_file_log(const log_severity_list_t *severity, const char *filename,
   int open_flags = O_WRONLY|O_CREAT;
   open_flags |= truncate_log ? O_TRUNC : O_APPEND;
 
-  fd = tor_open_cloexec(filename, open_flags, 0644);
+  fd = tor_open_cloexec(filename, open_flags, 0640);
   if (fd<0)
     return -1;
   if (tor_fd_seekend(fd)<0) {
@@ -1177,7 +1177,7 @@ static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
 };
 
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
@@ -1319,10 +1319,8 @@ parse_log_severity_config(const char **cfg_ptr,
     if (got_an_unqualified_range > 1)
       return -1;
 
-    space = strchr(cfg, ' ');
+    space = find_whitespace(cfg);
     dash = strchr(cfg, '-');
-    if (!space)
-      space = strchr(cfg, '\0');
     if (dash && dash < space) {
       sev_lo = tor_strndup(cfg, dash-cfg);
       sev_hi = tor_strndup(dash+1, space-(dash+1));
diff --git a/src/common/memarea.c b/src/common/memarea.c
index 7d16b702e3..659d1edf54 100644
--- a/src/common/memarea.c
+++ b/src/common/memarea.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2016, The Tor Project, Inc. */
+/* Copyright (c) 2008-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /** \file memarea.c
@@ -12,6 +12,9 @@
 #include "util.h"
 #include "compat.h"
 #include "torlog.h"
+#include "container.h"
+
+#ifndef DISABLE_MEMORY_SENTINELS
 
 /** If true, we try to detect any attempts to write beyond the length of a
  * memarea. */
@@ -304,3 +307,91 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
+#else
+
+struct memarea_t {
+  smartlist_t *pieces;
+};
+
+memarea_t *
+memarea_new(void)
+{
+  memarea_t *ma = tor_malloc_zero(sizeof(memarea_t));
+  ma->pieces = smartlist_new();
+  return ma;
+}
+void
+memarea_drop_all(memarea_t *area)
+{
+  memarea_clear(area);
+  smartlist_free(area->pieces);
+  tor_free(area);
+}
+void
+memarea_clear(memarea_t *area)
+{
+  SMARTLIST_FOREACH(area->pieces, void *, p, tor_free_(p));
+  smartlist_clear(area->pieces);
+}
+int
+memarea_owns_ptr(const memarea_t *area, const void *ptr)
+{
+  SMARTLIST_FOREACH(area->pieces, const void *, p, if (ptr == p) return 1;);
+  return 0;
+}
+
+void *
+memarea_alloc(memarea_t *area, size_t sz)
+{
+  void *result = tor_malloc(sz);
+  smartlist_add(area->pieces, result);
+  return result;
+}
+
+void *
+memarea_alloc_zero(memarea_t *area, size_t sz)
+{
+  void *result = tor_malloc_zero(sz);
+  smartlist_add(area->pieces, result);
+  return result;
+}
+void *
+memarea_memdup(memarea_t *area, const void *s, size_t n)
+{
+  void *r = memarea_alloc(area, n);
+  memcpy(r, s, n);
+  return r;
+}
+char *
+memarea_strdup(memarea_t *area, const char *s)
+{
+  size_t n = strlen(s);
+  char *r = memarea_alloc(area, n+1);
+  memcpy(r, s, n);
+  r[n] = 0;
+  return r;
+}
+char *
+memarea_strndup(memarea_t *area, const char *s, size_t n)
+{
+  size_t ln = strnlen(s, n);
+  char *r = memarea_alloc(area, ln+1);
+  memcpy(r, s, ln);
+  r[ln] = 0;
+  return r;
+}
+void
+memarea_get_stats(memarea_t *area,
+                  size_t *allocated_out, size_t *used_out)
+{
+  (void)area;
+  *allocated_out = *used_out = 128;
+}
+void
+memarea_assert_ok(memarea_t *area)
+{
+  (void)area;
+}
+
+#endif
+
diff --git a/src/common/memarea.h b/src/common/memarea.h
index 85bca51ad3..85012c1c34 100644
--- a/src/common/memarea.h
+++ b/src/common/memarea.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2016, The Tor Project, Inc. */
+/* Copyright (c) 2008-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 /* Tor dependencies */
 
diff --git a/src/common/procmon.c b/src/common/procmon.c
index c485c760c7..d49e7f18f5 100644
--- a/src/common/procmon.c
+++ b/src/common/procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/procmon.h b/src/common/procmon.h
index 49ead24092..b07cff2c4a 100644
--- a/src/common/procmon.h
+++ b/src/common/procmon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/pubsub.c b/src/common/pubsub.c
index b3faf40e00..336e8a6e7f 100644
--- a/src/common/pubsub.c
+++ b/src/common/pubsub.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/pubsub.h b/src/common/pubsub.h
index bbb4f02a42..6f4ce08754 100644
--- a/src/common/pubsub.h
+++ b/src/common/pubsub.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/sandbox.c b/src/common/sandbox.c
index 3d27ea66b5..312cf37722 100644
--- a/src/common/sandbox.c
+++ b/src/common/sandbox.c
@@ -1,7 +1,7 @@
  /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,14 @@
 #define _LARGEFILE64_SOURCE
 #endif
 
-/** Malloc mprotect limit in bytes. */
-#define MALLOC_MP_LIM 1048576
+/** Malloc mprotect limit in bytes.
+ *
+ * 28/06/2017: This value was increased from 16 MB to 20 MB after we introduced
+ * LZMA support in Tor (0.3.1.1-alpha). We limit our LZMA coder to 16 MB, but
+ * liblzma have a small overhead that we need to compensate for to avoid being
+ * killed by the sandbox.
+ */
+#define MALLOC_MP_LIM (20*1024*1024)
 
 #include <stdio.h>
 #include <string.h>
@@ -1614,7 +1620,7 @@ sandbox_getaddrinfo(const char *name, const char *servname,
     return err;
   }
 
-  /* Otherwise, the sanbox is on.  If we have an item, yield its cached
+  /* Otherwise, the sandbox is on.  If we have an item, yield its cached
      result. */
   if (item) {
     *res = item->res;
diff --git a/src/common/sandbox.h b/src/common/sandbox.h
index c5963e3119..a6b83153af 100644
--- a/src/common/sandbox.h
+++ b/src/common/sandbox.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/storagedir.c b/src/common/storagedir.c
new file mode 100644
index 0000000000..c471ea911f
--- /dev/null
+++ b/src/common/storagedir.c
@@ -0,0 +1,586 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "container.h"
+#include "compat.h"
+#include "confline.h"
+#include "memarea.h"
+#include "sandbox.h"
+#include "storagedir.h"
+#include "torlog.h"
+#include "util.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#define FNAME_MIN_NUM 1000
+
+/** A storage_dir_t represents a directory full of similar cached
+ * files. Filenames are decimal integers. Files can be cleaned as needed
+ * to limit total disk usage. */
+struct storage_dir_t {
+  /** Directory holding the files for this storagedir. */
+  char *directory;
+  /** Either NULL, or a directory listing of the directory (as a smartlist
+   * of strings */
+  smartlist_t *contents;
+  /** The largest number of non-temporary files we'll place in the
+   * directory. */
+  int max_files;
+  /** If true, then 'usage' has been computed. */
+  int usage_known;
+  /** The total number of bytes used in this directory */
+  uint64_t usage;
+};
+
+/** Create or open a new storage directory at <b>dirname</b>, with
+ * capacity for up to <b>max_files</b> files.
+ */
+storage_dir_t *
+storage_dir_new(const char *dirname, int max_files)
+{
+  if (check_private_dir(dirname, CPD_CREATE, NULL) < 0)
+    return NULL;
+
+  storage_dir_t *d = tor_malloc_zero(sizeof(storage_dir_t));
+  d->directory = tor_strdup(dirname);
+  d->max_files = max_files;
+  return d;
+}
+
+/**
+ * Drop all in-RAM storage for <b>d</b>.  Does not delete any files.
+ */
+void
+storage_dir_free(storage_dir_t *d)
+{
+  if (d == NULL)
+    return;
+  tor_free(d->directory);
+  if (d->contents) {
+    SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+    smartlist_free(d->contents);
+  }
+  tor_free(d);
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>d</b> will need.
+ *
+ * The presence of this function is why we need an upper limit on the
+ * number of files in a storage_dir_t: we need to approve file operations
+ * one by one.
+ */
+int
+storage_dir_register_with_sandbox(storage_dir_t *d, sandbox_cfg_t **cfg)
+{
+  int problems = 0;
+  int idx;
+  for (idx = FNAME_MIN_NUM; idx < FNAME_MIN_NUM + d->max_files; ++idx) {
+    char *path = NULL, *tmppath = NULL;
+    tor_asprintf(&path, "%s/%d", d->directory, idx);
+    tor_asprintf(&tmppath, "%s/%d.tmp", d->directory, idx);
+
+    problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(path));
+    problems += sandbox_cfg_allow_open_filename(cfg, tor_strdup(tmppath));
+    problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(path));
+    problems += sandbox_cfg_allow_stat_filename(cfg, tor_strdup(tmppath));
+    problems += sandbox_cfg_allow_rename(cfg,
+                                      tor_strdup(tmppath), tor_strdup(path));
+
+    tor_free(path);
+    tor_free(tmppath);
+  }
+
+  return problems ? -1 : 0;
+}
+
+/**
+ * Remove all files in <b>d</b> whose names end with ".tmp".
+ *
+ * Requires that the contents field of <b>d</b> is set.
+ */
+static void
+storage_dir_clean_tmpfiles(storage_dir_t *d)
+{
+  if (!d->contents)
+    return;
+  SMARTLIST_FOREACH_BEGIN(d->contents, char *, fname) {
+    if (strcmpend(fname, ".tmp"))
+      continue;
+    char *path = NULL;
+    tor_asprintf(&path, "%s/%s", d->directory, fname);
+    if (unlink(sandbox_intern_string(path))) {
+      log_warn(LD_FS, "Unable to unlink %s while cleaning "
+               "temporary files: %s", escaped(path), strerror(errno));
+      tor_free(path);
+      continue;
+    }
+    tor_free(path);
+    SMARTLIST_DEL_CURRENT(d->contents, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(fname);
+
+  d->usage_known = 0;
+}
+
+/**
+ * Re-scan the directory <b>d</b> to learn its contents.
+ */
+static int
+storage_dir_rescan(storage_dir_t *d)
+{
+  if (d->contents) {
+    SMARTLIST_FOREACH(d->contents, char *, cp, tor_free(cp));
+    smartlist_free(d->contents);
+  }
+  d->usage = 0;
+  d->usage_known = 0;
+  if (NULL == (d->contents = tor_listdir(d->directory))) {
+    return -1;
+  }
+  storage_dir_clean_tmpfiles(d);
+  return 0;
+}
+
+/**
+ * Return a smartlist containing the filenames within <b>d</b>.
+ */
+const smartlist_t *
+storage_dir_list(storage_dir_t *d)
+{
+  if (! d->contents)
+    storage_dir_rescan(d);
+  return d->contents;
+}
+
+/**
+ * Return the total number of bytes used for storage in <b>d</b>.
+ */
+uint64_t
+storage_dir_get_usage(storage_dir_t *d)
+{
+  if (d->usage_known)
+    return d->usage;
+
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(storage_dir_list(d), const char *, cp) {
+    char *path = NULL;
+    struct stat st;
+    tor_asprintf(&path, "%s/%s", d->directory, cp);
+    if (stat(sandbox_intern_string(path), &st) == 0) {
+      total += st.st_size;
+    }
+    tor_free(path);
+  } SMARTLIST_FOREACH_END(cp);
+
+  d->usage = total;
+  d->usage_known = 1;
+  return d->usage;
+}
+
+/** Mmap a specified file within <b>d</b>.
+ *
+ * On failure, return NULL and set errno as for tor_mmap_file(). */
+tor_mmap_t *
+storage_dir_map(storage_dir_t *d, const char *fname)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  tor_mmap_t *result = tor_mmap_file(path);
+  int errval = errno;
+  tor_free(path);
+  if (result == NULL)
+    errno = errval;
+  return result;
+}
+
+/** Read a file within <b>d</b> into a newly allocated buffer.  Set
+ * *<b>sz_out</b> to its size. */
+uint8_t *
+storage_dir_read(storage_dir_t *d, const char *fname, int bin, size_t *sz_out)
+{
+  const int flags = bin ? RFTS_BIN : 0;
+
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  struct stat st;
+  char *contents = read_file_to_str(path, flags, &st);
+  if (contents && sz_out) {
+    // it fits in RAM, so we know its size is less than SIZE_MAX
+#if UINT64_MAX > SIZE_MAX
+    tor_assert((uint64_t)st.st_size <= SIZE_MAX);
+#endif
+    *sz_out = (size_t) st.st_size;
+  }
+
+  tor_free(path);
+  return (uint8_t *) contents;
+}
+
+/** Helper: Find an unused filename within the directory */
+static char *
+find_unused_fname(storage_dir_t *d)
+{
+  if (!d->contents) {
+    if (storage_dir_rescan(d) < 0)
+      return NULL;
+  }
+
+  char buf[16];
+  int i;
+  /* Yuck; this is quadratic.  Fortunately, that shouldn't matter much,
+   * since disk writes are more expensive by a lot. */
+  for (i = FNAME_MIN_NUM; i < FNAME_MIN_NUM + d->max_files; ++i) {
+    tor_snprintf(buf, sizeof(buf), "%d", i);
+    if (!smartlist_contains_string(d->contents, buf)) {
+      return tor_strdup(buf);
+    }
+  }
+  return NULL;
+}
+
+/** Helper: As storage_dir_save_bytes_to_file, but store a smartlist of
+ * sized_chunk_t rather than a single byte array. */
+static int
+storage_dir_save_chunks_to_file(storage_dir_t *d,
+                                const smartlist_t *chunks,
+                                int binary,
+                                char **fname_out)
+{
+  uint64_t total_length = 0;
+  char *fname = find_unused_fname(d);
+  if (!fname)
+    return -1;
+
+  SMARTLIST_FOREACH(chunks, const sized_chunk_t *, ch,
+                    total_length += ch->len);
+
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+
+  int r = write_chunks_to_file(path, chunks, binary, 0);
+  if (r == 0) {
+    if (d->usage_known)
+      d->usage += total_length;
+    if (fname_out) {
+      *fname_out = tor_strdup(fname);
+    }
+    if (d->contents)
+      smartlist_add(d->contents, tor_strdup(fname));
+  }
+  tor_free(fname);
+  tor_free(path);
+  return r;
+}
+
+/** Try to write the <b>length</b> bytes at <b>data</b> into a new file
+ * in <b>d</b>.  On success, return 0 and set *<b>fname_out</b> to a
+ * newly allocated string containing the filename.  On failure, return
+ * -1. */
+int
+storage_dir_save_bytes_to_file(storage_dir_t *d,
+                               const uint8_t *data,
+                               size_t length,
+                               int binary,
+                               char **fname_out)
+{
+  smartlist_t *chunks = smartlist_new();
+  sized_chunk_t chunk = { (const char *)data, length };
+  smartlist_add(chunks, &chunk);
+  int r = storage_dir_save_chunks_to_file(d, chunks, binary, fname_out);
+  smartlist_free(chunks);
+  return r;
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but saves a NUL-terminated string
+ * <b>str</b>.
+ */
+int
+storage_dir_save_string_to_file(storage_dir_t *d,
+                                const char *str,
+                                int binary,
+                                char **fname_out)
+{
+  return storage_dir_save_bytes_to_file(d,
+                (const uint8_t*)str, strlen(str), binary, fname_out);
+}
+
+/**
+ * As storage_dir_save_bytes_to_file, but associates the data with the
+ * key-value pairs in <b>labels</b>. Files stored in this format can be
+ * recovered with storage_dir_map_labeled() or storage_dir_read_labeled().
+ */
+int
+storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                  const config_line_t *labels,
+                                  const uint8_t *data,
+                                  size_t length,
+                                  char **fname_out)
+{
+  /*
+   * The storage format is to prefix the data with the key-value pairs in
+   * <b>labels</b>, and a single NUL separator.  But code outside this module
+   * MUST NOT rely on that format.
+   */
+
+  smartlist_t *chunks = smartlist_new();
+  memarea_t *area = memarea_new();
+  const config_line_t *line;
+  for (line = labels; line; line = line->next) {
+    sized_chunk_t *sz = memarea_alloc(area, sizeof(sized_chunk_t));
+    sz->len = strlen(line->key) + 1 + strlen(line->value) + 1;
+    const size_t allocated = sz->len + 1;
+    char *bytes = memarea_alloc(area, allocated);
+    tor_snprintf(bytes, allocated, "%s %s\n", line->key, line->value);
+    sz->bytes = bytes;
+    smartlist_add(chunks, sz);
+  }
+
+  sized_chunk_t *nul = memarea_alloc(area, sizeof(sized_chunk_t));
+  nul->len = 1;
+  nul->bytes = "\0";
+  smartlist_add(chunks, nul);
+
+  sized_chunk_t *datachunk = memarea_alloc(area, sizeof(sized_chunk_t));
+  datachunk->bytes = (const char *)data;
+  datachunk->len = length;
+  smartlist_add(chunks, datachunk);
+
+  int r = storage_dir_save_chunks_to_file(d, chunks, 1, fname_out);
+  smartlist_free(chunks);
+  memarea_drop_all(area);
+  return r;
+}
+
+/**
+ * Map a file that was created with storage_dir_save_labeled_to_file().  On
+ * failure, return NULL.  On success, write a set of newly allocated labels
+ * into *<b>labels_out</b>, a pointer to the data into *<b>data_out</b>, and
+ * the data's size into *<b>sz_out</b>. On success, also return a tor_mmap_t
+ * object whose contents should not be used -- it needs to be kept around,
+ * though, for as long as <b>data_out</b> is going to be valid.
+ *
+ * On failure, set errno as for tor_mmap_file() if the file was missing or
+ * empty, and set errno to EINVAL if the file was not in the labeled
+ * format expected.
+ */
+tor_mmap_t *
+storage_dir_map_labeled(storage_dir_t *dir,
+                         const char *fname,
+                         config_line_t **labels_out,
+                         const uint8_t **data_out,
+                         size_t *sz_out)
+{
+  tor_mmap_t *m = storage_dir_map(dir, fname);
+  int errval;
+  if (! m) {
+    errval = errno;
+    goto err;
+  }
+  const char *nulp = memchr(m->data, '\0', m->size);
+  if (! nulp) {
+    errval = EINVAL;
+    goto err;
+  }
+  if (labels_out && config_get_lines(m->data, labels_out, 0) < 0) {
+    errval = EINVAL;
+    goto err;
+  }
+  size_t offset = nulp - m->data + 1;
+  tor_assert(offset <= m->size);
+  *data_out = (const uint8_t *)(m->data + offset);
+  *sz_out = m->size - offset;
+
+  return m;
+ err:
+  tor_munmap_file(m);
+  errno = errval;
+  return NULL;
+}
+
+/** As storage_dir_map_labeled, but return a new byte array containing the
+ * data. */
+uint8_t *
+storage_dir_read_labeled(storage_dir_t *dir,
+                          const char *fname,
+                          config_line_t **labels_out,
+                          size_t *sz_out)
+{
+  const uint8_t *data = NULL;
+  tor_mmap_t *m = storage_dir_map_labeled(dir, fname, labels_out,
+                                           &data, sz_out);
+  if (m == NULL)
+    return NULL;
+  uint8_t *result = tor_memdup(data, *sz_out);
+  tor_munmap_file(m);
+  return result;
+}
+
+/* Reduce the cached usage amount in <b>d</b> by <b>removed_file_size</b>.
+ * This function is a no-op if <b>d->usage_known</b> is 0. */
+static void
+storage_dir_reduce_usage(storage_dir_t *d, uint64_t removed_file_size)
+{
+  if (d->usage_known) {
+    if (! BUG(d->usage < removed_file_size)) {
+      /* This bug can also be triggered if an external process resized a file
+       * between the call to storage_dir_get_usage() that last checked
+       * actual usage (rather than relaying on cached usage), and the call to
+       * this function. */
+      d->usage -= removed_file_size;
+    } else {
+      /* If we underflowed the cached directory size, re-check the sizes of all
+       * the files in the directory. This makes storage_dir_shrink() quadratic,
+       * but only if a process is continually changing file sizes in the
+       * storage directory (in which case, we have bigger issues).
+       *
+       * We can't just reset usage_known, because storage_dir_shrink() relies
+       * on knowing the usage. */
+      storage_dir_rescan(d);
+      (void)storage_dir_get_usage(d);
+    }
+  }
+}
+
+/**
+ * Remove the file called <b>fname</b> from <b>d</b>.
+ */
+void
+storage_dir_remove_file(storage_dir_t *d,
+                        const char *fname)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s/%s", d->directory, fname);
+  const char *ipath = sandbox_intern_string(path);
+
+  uint64_t size = 0;
+  if (d->usage_known) {
+    struct stat st;
+    if (stat(ipath, &st) == 0) {
+      size = st.st_size;
+    }
+  }
+  if (unlink(ipath) == 0) {
+    storage_dir_reduce_usage(d, size);
+  } else {
+    log_warn(LD_FS, "Unable to unlink %s while removing file: %s",
+             escaped(path), strerror(errno));
+    tor_free(path);
+    return;
+  }
+  if (d->contents) {
+    smartlist_string_remove(d->contents, fname);
+  }
+
+  tor_free(path);
+}
+
+/** Helper type: used to sort the members of storage directory by mtime. */
+typedef struct shrinking_dir_entry_t {
+  time_t mtime;
+  uint64_t size;
+  char *path;
+} shrinking_dir_entry_t;
+
+/** Helper: use with qsort to sort shrinking_dir_entry_t structs. */
+static int
+shrinking_dir_entry_compare(const void *a_, const void *b_)
+{
+  const shrinking_dir_entry_t *a = a_;
+  const shrinking_dir_entry_t *b = b_;
+
+  if (a->mtime < b->mtime)
+    return -1;
+  else if (a->mtime > b->mtime)
+    return 1;
+  else
+    return 0;
+}
+
+/**
+ * Try to free space by removing the oldest files in <b>d</b>. Delete
+ * until no more than <b>target_size</b> bytes are left, and at least
+ * <b>min_to_remove</b> files have been removed... or until there is
+ * nothing left to remove.
+ *
+ * Return 0 on success; -1 on failure.
+ */
+int
+storage_dir_shrink(storage_dir_t *d,
+                   uint64_t target_size,
+                   int min_to_remove)
+{
+  if (d->usage_known && d->usage <= target_size && !min_to_remove) {
+    /* Already small enough. */
+    return 0;
+  }
+
+  if (storage_dir_rescan(d) < 0)
+    return -1;
+
+  const uint64_t orig_usage = storage_dir_get_usage(d);
+  if (orig_usage <= target_size && !min_to_remove) {
+    /* Okay, small enough after rescan! */
+    return 0;
+  }
+
+  const int n = smartlist_len(d->contents);
+  shrinking_dir_entry_t *ents = tor_calloc(n, sizeof(shrinking_dir_entry_t));
+  SMARTLIST_FOREACH_BEGIN(d->contents, const char *, fname) {
+    shrinking_dir_entry_t *ent = &ents[fname_sl_idx];
+    struct stat st;
+    tor_asprintf(&ent->path, "%s/%s", d->directory, fname);
+    if (stat(sandbox_intern_string(ent->path), &st) == 0) {
+      ent->mtime = st.st_mtime;
+      ent->size = st.st_size;
+    }
+  } SMARTLIST_FOREACH_END(fname);
+
+  qsort(ents, n, sizeof(shrinking_dir_entry_t), shrinking_dir_entry_compare);
+
+  int idx = 0;
+  while ((d->usage > target_size || min_to_remove > 0) && idx < n) {
+    if (unlink(sandbox_intern_string(ents[idx].path)) == 0) {
+      storage_dir_reduce_usage(d, ents[idx].size);
+      --min_to_remove;
+    }
+    ++idx;
+  }
+
+  for (idx = 0; idx < n; ++idx) {
+    tor_free(ents[idx].path);
+  }
+  tor_free(ents);
+
+  storage_dir_rescan(d);
+
+  return 0;
+}
+
+/** Remove all files in <b>d</b>. */
+int
+storage_dir_remove_all(storage_dir_t *d)
+{
+  return storage_dir_shrink(d, 0, d->max_files);
+}
+
+/**
+ * Return the largest number of non-temporary files we're willing to
+ * store in <b>d</b>.
+ */
+int
+storage_dir_get_max_files(storage_dir_t *d)
+{
+  return d->max_files;
+}
+
diff --git a/src/common/storagedir.h b/src/common/storagedir.h
new file mode 100644
index 0000000000..db25057e65
--- /dev/null
+++ b/src/common/storagedir.h
@@ -0,0 +1,51 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_STORAGEDIR_H
+#define TOR_STORAGEDIR_H
+
+typedef struct storage_dir_t storage_dir_t;
+struct config_line_t;
+struct sandbox_cfg_elem;
+
+storage_dir_t * storage_dir_new(const char *dirname, int n_files);
+void storage_dir_free(storage_dir_t *d);
+int storage_dir_register_with_sandbox(storage_dir_t *d,
+                                      struct sandbox_cfg_elem **cfg);
+const smartlist_t *storage_dir_list(storage_dir_t *d);
+uint64_t storage_dir_get_usage(storage_dir_t *d);
+tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname);
+uint8_t *storage_dir_read(storage_dir_t *d, const char *fname, int bin,
+                          size_t *sz_out);
+int storage_dir_save_bytes_to_file(storage_dir_t *d,
+                                   const uint8_t *data,
+                                   size_t length,
+                                   int binary,
+                                   char **fname_out);
+int storage_dir_save_string_to_file(storage_dir_t *d,
+                                    const char *data,
+                                    int binary,
+                                    char **fname_out);
+int storage_dir_save_labeled_to_file(storage_dir_t *d,
+                                      const struct config_line_t *labels,
+                                      const uint8_t *data,
+                                      size_t length,
+                                      char **fname_out);
+tor_mmap_t *storage_dir_map_labeled(storage_dir_t *dir,
+                                     const char *fname,
+                                     struct config_line_t **labels_out,
+                                     const uint8_t **data_out,
+                                     size_t *size_out);
+uint8_t *storage_dir_read_labeled(storage_dir_t *d, const char *fname,
+                                   struct config_line_t **labels_out,
+                                   size_t *sz_out);
+void storage_dir_remove_file(storage_dir_t *d,
+                             const char *fname);
+int storage_dir_shrink(storage_dir_t *d,
+                       uint64_t target_size,
+                       int min_to_remove);
+int storage_dir_remove_all(storage_dir_t *d);
+int storage_dir_get_max_files(storage_dir_t *d);
+
+#endif
+
diff --git a/src/common/testsupport.h b/src/common/testsupport.h
index 9ad2ba77e0..9fc96199b4 100644
--- a/src/common/testsupport.h
+++ b/src/common/testsupport.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TESTSUPPORT_H
diff --git a/src/common/timers.c b/src/common/timers.c
index 41b2008ac4..c43c49c083 100644
--- a/src/common/timers.c
+++ b/src/common/timers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,8 @@
 
 #include "orconfig.h"
 
+#define TOR_TIMERS_PRIVATE
+
 #include "compat.h"
 #include "compat_libevent.h"
 #include "timers.h"
@@ -148,6 +150,21 @@ libevent_timer_reschedule(void)
   event_add(global_timer_event, &d);
 }
 
+/** Run the callback of every timer that has expired, based on the current
+ * output of monotime_get(). */
+STATIC void
+timers_run_pending(void)
+{
+  monotime_t now;
+  monotime_get(&now);
+  timer_advance_to_cur_time(&now);
+
+  tor_timer_t *t;
+  while ((t = timeouts_get(global_timeouts))) {
+    t->callback.cb(t, t->callback.arg, &now);
+  }
+}
+
 /**
  * Invoked when the libevent timer has expired: see which tor_timer_t events
  * have fired, activate their callbacks, and reschedule the libevent timer.
@@ -159,14 +176,7 @@ libevent_timer_callback(evutil_socket_t fd, short what, void *arg)
   (void)what;
   (void)arg;
 
-  monotime_t now;
-  monotime_get(&now);
-  timer_advance_to_cur_time(&now);
-
-  tor_timer_t *t;
-  while ((t = timeouts_get(global_timeouts))) {
-    t->callback.cb(t, t->callback.arg, &now);
-  }
+  timers_run_pending();
 
   libevent_timer_reschedule();
 }
@@ -255,6 +265,20 @@ timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg)
 }
 
 /**
+ * Set *<b>cb_out</b> (if provided) to this timer's callback function,
+ * and *<b>arg_out</b> (if provided) to this timer's callback argument.
+ */
+void
+timer_get_cb(const tor_timer_t *t,
+             timer_cb_fn_t *cb_out, void **arg_out)
+{
+  if (cb_out)
+    *cb_out = t->callback.cb;
+  if (arg_out)
+    *arg_out = t->callback.arg;
+}
+
+/**
  * Schedule the timer t to fire at the current time plus a delay of
  * <b>delay</b> microseconds.  All times are relative to monotime_get().
  */
diff --git a/src/common/timers.h b/src/common/timers.h
index 5f918f8e15..d9602cd2ae 100644
--- a/src/common/timers.h
+++ b/src/common/timers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TIMERS_H
@@ -13,6 +13,8 @@ typedef void (*timer_cb_fn_t)(tor_timer_t *, void *,
                               const struct monotime_t *);
 tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg);
 void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg);
+void timer_get_cb(const tor_timer_t *t,
+                  timer_cb_fn_t *cb_out, void **arg_out);
 void timer_schedule(tor_timer_t *t, const struct timeval *delay);
 void timer_disable(tor_timer_t *t);
 void timer_free(tor_timer_t *t);
@@ -20,5 +22,9 @@ void timer_free(tor_timer_t *t);
 void timers_initialize(void);
 void timers_shutdown(void);
 
+#ifdef TOR_TIMERS_PRIVATE
+STATIC void timers_run_pending(void);
+#endif
+
 #endif
 
diff --git a/src/common/torgzip.c b/src/common/torgzip.c
deleted file mode 100644
index c44399aa74..0000000000
--- a/src/common/torgzip.c
+++ /dev/null
@@ -1,586 +0,0 @@
-/* Copyright (c) 2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torgzip.c
- * \brief A simple in-memory gzip implementation.
- **/
-
-#include "orconfig.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <assert.h>
-#include <string.h>
-#include "torint.h"
-
-#ifdef HAVE_NETINET_IN_H
-#include <netinet/in.h>
-#endif
-
-#include "util.h"
-#include "torlog.h"
-#include "torgzip.h"
-
-/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
-   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
-   that nobody will care if the compile outputs a no-such-identifier warning.
-
-   Sorry, but we like -Werror over here, so I guess we need to define these.
-   I hope that zlib 1.2.6 doesn't break these too.
-*/
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE 0
-#endif
-#ifndef _LFS64_LARGEFILE
-#define _LFS64_LARGEFILE 0
-#endif
-#ifndef _FILE_OFFSET_BITS
-#define _FILE_OFFSET_BITS 0
-#endif
-#ifndef off64_t
-#define off64_t int64_t
-#endif
-
-#include <zlib.h>
-
-#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
-#error "We require zlib version 1.2 or later."
-#endif
-
-static size_t tor_zlib_state_size_precalc(int inflate,
-                                          int windowbits, int memlevel);
-
-/** Total number of bytes allocated for zlib state */
-static size_t total_zlib_allocation = 0;
-
-/** Return a string representation of the version of the currently running
- * version of zlib. */
-const char *
-tor_zlib_get_version_str(void)
-{
-  return zlibVersion();
-}
-
-/** Return a string representation of the version of the version of zlib
-* used at compilation. */
-const char *
-tor_zlib_get_header_version_str(void)
-{
-  return ZLIB_VERSION;
-}
-
-/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
-static inline int
-method_bits(compress_method_t method, zlib_compression_level_t level)
-{
-  /* Bits+16 means "use gzip" in zlib >= 1.2 */
-  const int flag = method == GZIP_METHOD ? 16 : 0;
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return flag + 15;
-    case MEDIUM_COMPRESSION: return flag + 13;
-    case LOW_COMPRESSION: return flag + 11;
-  }
-}
-
-static inline int
-get_memlevel(zlib_compression_level_t level)
-{
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return 8;
-    case MEDIUM_COMPRESSION: return 7;
-    case LOW_COMPRESSION: return 6;
-  }
-}
-
-/** @{ */
-/* These macros define the maximum allowable compression factor.  Anything of
- * size greater than CHECK_FOR_COMPRESSION_BOMB_AFTER is not allowed to
- * have an uncompression factor (uncompressed size:compressed size ratio) of
- * any greater than MAX_UNCOMPRESSION_FACTOR.
- *
- * Picking a value for MAX_UNCOMPRESSION_FACTOR is a trade-off: we want it to
- * be small to limit the attack multiplier, but we also want it to be large
- * enough so that no legitimate document --even ones we might invent in the
- * future -- ever compresses by a factor of greater than
- * MAX_UNCOMPRESSION_FACTOR. Within those parameters, there's a reasonably
- * large range of possible values. IMO, anything over 8 is probably safe; IMO
- * anything under 50 is probably sufficient.
- */
-#define MAX_UNCOMPRESSION_FACTOR 25
-#define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64)
-/** @} */
-
-/** Return true if uncompressing an input of size <b>in_size</b> to an input
- * of size at least <b>size_out</b> looks like a compression bomb. */
-static int
-is_compression_bomb(size_t size_in, size_t size_out)
-{
-  if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER)
-    return 0;
-
-  return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR);
-}
-
-/** Given <b>in_len</b> bytes at <b>in</b>, compress them into a newly
- * allocated buffer, using the method described in <b>method</b>.  Store the
- * compressed string in *<b>out</b>, and its length in *<b>out_len</b>.
- * Return 0 on success, -1 on failure.
- */
-int
-tor_gzip_compress(char **out, size_t *out_len,
-                  const char *in, size_t in_len,
-                  compress_method_t method)
-{
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                   method_bits(method, HIGH_COMPRESSION),
-                   get_memlevel(HIGH_COMPRESSION),
-                   Z_DEFAULT_STRATEGY) != Z_OK) {
-    //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
-    log_warn(LD_GENERAL, "Error from deflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    //LCOV_EXCL_STOP
-  }
-
-  /* Guess 50% compression. */
-  out_size = in_len / 2;
-  if (out_size < 1024) out_size = 1024;
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (deflate(stream, Z_FINISH))
-      {
-      case Z_STREAM_END:
-        goto done;
-      case Z_OK:
-        /* In case zlib doesn't work as I think .... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-        /* Falls through. */
-      case Z_BUF_ERROR:
-        offset = stream->next_out - ((unsigned char*)*out);
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in compression.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->total_out;
-#ifdef OPENBSD
-  /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
-   *    third-party API!"
-   * "Oh, that trick will just make people do unsafe casts to the unsigned
-   *    type in their cross-platform code!"
-   * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
-   *    the newly unsigned field isn't negative." */
-  tor_assert(stream->total_out >= 0);
-#endif
-  if (deflateEnd(stream)!=Z_OK) {
-    // LCOV_EXCL_START -- unreachable if we handled the zlib structure right
-    tor_assert_nonfatal_unreached();
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-  tor_free(stream);
-
-  if (is_compression_bomb(*out_len, in_len)) {
-    log_warn(LD_BUG, "We compressed something and got an insanely high "
-          "compression factor; other Tors would think this was a zlib bomb.");
-    goto err;
-  }
-
-  return 0;
- err:
-  if (stream) {
-    deflateEnd(stream);
-    tor_free(stream);
-  }
-  tor_free(*out);
-  return -1;
-}
-
-/** Given zero or more zlib-compressed or gzip-compressed strings of
- * total length
- * <b>in_len</b> bytes at <b>in</b>, uncompress them into a newly allocated
- * buffer, using the method described in <b>method</b>.  Store the uncompressed
- * string in *<b>out</b>, and its length in *<b>out_len</b>.  Return 0 on
- * success, -1 on failure.
- *
- * If <b>complete_only</b> is true, we consider a truncated input as a
- * failure; otherwise we decompress as much as we can.  Warn about truncated
- * or corrupt inputs at <b>protocol_warn_level</b>.
- */
-int
-tor_gzip_uncompress(char **out, size_t *out_len,
-                    const char *in, size_t in_len,
-                    compress_method_t method,
-                    int complete_only,
-                    int protocol_warn_level)
-{
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-  int r;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (inflateInit2(stream,
-                   method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
-    // LCOV_EXCL_START -- can only hit this if we give bad inputs.
-    log_warn(LD_GENERAL, "Error from inflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-
-  out_size = in_len * 2;  /* guess 50% compression. */
-  if (out_size < 1024) out_size = 1024;
-  if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
-    goto err;
-
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
-      {
-      case Z_STREAM_END:
-        if (stream->avail_in == 0)
-          goto done;
-        /* There may be more compressed data here. */
-        if ((r = inflateEnd(stream)) != Z_OK) {
-          log_warn(LD_BUG, "Error freeing gzip structures");
-          goto err;
-        }
-        if (inflateInit2(stream,
-                         method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
-          log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
-                   stream->msg?stream->msg:"<no message>");
-          goto err;
-        }
-        break;
-      case Z_OK:
-        if (!complete_only && stream->avail_in == 0)
-          goto done;
-        /* In case zlib doesn't work as I think.... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-        /* Falls through. */
-      case Z_BUF_ERROR:
-        if (stream->avail_out > 0) {
-          log_fn(protocol_warn_level, LD_PROTOCOL,
-                 "possible truncated or corrupt zlib data");
-          goto err;
-        }
-        offset = stream->next_out - (unsigned char*)*out;
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in uncompression.");
-          goto err;
-        }
-        if (is_compression_bomb(in_len, out_size)) {
-          log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
-                   "not proceeding.");
-          goto err;
-        }
-        if (out_size >= SIZE_T_CEILING) {
-          log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->next_out - (unsigned char*)*out;
-  r = inflateEnd(stream);
-  tor_free(stream);
-  if (r != Z_OK) {
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-  }
-
-  /* NUL-terminate output. */
-  if (out_size == *out_len)
-    *out = tor_realloc(*out, out_size + 1);
-  (*out)[*out_len] = '\0';
-
-  return 0;
- err:
-  if (stream) {
-    inflateEnd(stream);
-    tor_free(stream);
-  }
-  if (*out) {
-    tor_free(*out);
-  }
-  return -1;
-}
-
-/** Try to tell whether the <b>in_len</b>-byte string in <b>in</b> is likely
- * to be compressed or not.  If it is, return the likeliest compression method.
- * Otherwise, return UNKNOWN_METHOD.
- */
-compress_method_t
-detect_compression_method(const char *in, size_t in_len)
-{
-  if (in_len > 2 && fast_memeq(in, "\x1f\x8b", 2)) {
-    return GZIP_METHOD;
-  } else if (in_len > 2 && (in[0] & 0x0f) == 8 &&
-             (ntohs(get_uint16(in)) % 31) == 0) {
-    return ZLIB_METHOD;
-  } else {
-    return UNKNOWN_METHOD;
-  }
-}
-
-/** Internal state for an incremental zlib compression/decompression.  The
- * body of this struct is not exposed. */
-struct tor_zlib_state_t {
-  struct z_stream_s stream; /**< The zlib stream */
-  int compress; /**< True if we are compressing; false if we are inflating */
-
-  /** Number of bytes read so far.  Used to detect zlib bombs. */
-  size_t input_so_far;
-  /** Number of bytes written so far.  Used to detect zlib bombs. */
-  size_t output_so_far;
-
-  /** Approximate number of bytes allocated for this object. */
-  size_t allocation;
-};
-
-/** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
- * <b>compress</b>, it's for compression; otherwise it's for
- * decompression. */
-tor_zlib_state_t *
-tor_zlib_new(int compress_, compress_method_t method,
-             zlib_compression_level_t compression_level)
-{
-  tor_zlib_state_t *out;
-  int bits, memlevel;
-
- if (! compress_) {
-   /* use this setting for decompression, since we might have the
-    * max number of window bits */
-   compression_level = HIGH_COMPRESSION;
- }
-
- out = tor_malloc_zero(sizeof(tor_zlib_state_t));
- out->stream.zalloc = Z_NULL;
- out->stream.zfree = Z_NULL;
- out->stream.opaque = NULL;
- out->compress = compress_;
- bits = method_bits(method, compression_level);
- memlevel = get_memlevel(compression_level);
- if (compress_) {
-   if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                    bits, memlevel,
-                    Z_DEFAULT_STRATEGY) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- } else {
-   if (inflateInit2(&out->stream, bits) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- }
- out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
-
- total_zlib_allocation += out->allocation;
-
- return out;
-
- err:
- tor_free(out);
- return NULL;
-}
-
-/** Compress/decompress some bytes using <b>state</b>.  Read up to
- * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes
- * to *<b>out</b>, adjusting the values as we go.  If <b>finish</b> is true,
- * we've reached the end of the input.
- *
- * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression.
- * Return TOR_ZLIB_OK if we're processed everything from the input.
- * Return TOR_ZLIB_BUF_FULL if we're out of space on <b>out</b>.
- * Return TOR_ZLIB_ERR if the stream is corrupt.
- */
-tor_zlib_output_t
-tor_zlib_process(tor_zlib_state_t *state,
-                 char **out, size_t *out_len,
-                 const char **in, size_t *in_len,
-                 int finish)
-{
-  int err;
-  tor_assert(*in_len <= UINT_MAX);
-  tor_assert(*out_len <= UINT_MAX);
-  state->stream.next_in = (unsigned char*) *in;
-  state->stream.avail_in = (unsigned int)*in_len;
-  state->stream.next_out = (unsigned char*) *out;
-  state->stream.avail_out = (unsigned int)*out_len;
-
-  if (state->compress) {
-    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
-  } else {
-    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
-  }
-
-  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
-  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
-
-  *out = (char*) state->stream.next_out;
-  *out_len = state->stream.avail_out;
-  *in = (const char *) state->stream.next_in;
-  *in_len = state->stream.avail_in;
-
-  if (! state->compress &&
-      is_compression_bomb(state->input_so_far, state->output_so_far)) {
-    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
-    return TOR_ZLIB_ERR;
-  }
-
-  switch (err)
-    {
-    case Z_STREAM_END:
-      return TOR_ZLIB_DONE;
-    case Z_BUF_ERROR:
-      if (state->stream.avail_in == 0 && !finish)
-        return TOR_ZLIB_OK;
-      return TOR_ZLIB_BUF_FULL;
-    case Z_OK:
-      if (state->stream.avail_out == 0 || finish)
-        return TOR_ZLIB_BUF_FULL;
-      return TOR_ZLIB_OK;
-    default:
-      log_warn(LD_GENERAL, "Gzip returned an error: %s",
-               state->stream.msg ? state->stream.msg : "<no message>");
-      return TOR_ZLIB_ERR;
-    }
-}
-
-/** Deallocate <b>state</b>. */
-void
-tor_zlib_free(tor_zlib_state_t *state)
-{
-  if (!state)
-    return;
-
- total_zlib_allocation -= state->allocation;
-
-  if (state->compress)
-    deflateEnd(&state->stream);
-  else
-    inflateEnd(&state->stream);
-
-  tor_free(state);
-}
-
-/** Return an approximate number of bytes used in RAM to hold a state with
- * window bits <b>windowBits</b> and compression level 'memlevel' */
-static size_t
-tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
-{
-  windowbits &= 15;
-
-#define A_FEW_KILOBYTES 2048
-
-  if (inflate_) {
-    /* From zconf.h:
-
-       "The memory requirements for inflate are (in bytes) 1 << windowBits
-       that is, 32K for windowBits=15 (default value) plus a few kilobytes
-       for small objects."
-    */
-    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
-      (1 << 15) + A_FEW_KILOBYTES;
-  } else {
-    /* Also from zconf.h:
-
-       "The memory requirements for deflate are (in bytes):
-            (1 << (windowBits+2)) +  (1 << (memLevel+9))
-        ... plus a few kilobytes for small objects."
-    */
-    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
-      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
-  }
-#undef A_FEW_KILOBYTES
-}
-
-/** Return the approximate number of bytes allocated for <b>state</b>. */
-size_t
-tor_zlib_state_size(const tor_zlib_state_t *state)
-{
-  return state->allocation;
-}
-
-/** Return the approximate number of bytes allocated for all zlib states. */
-size_t
-tor_zlib_get_total_allocation(void)
-{
-  return total_zlib_allocation;
-}
-
diff --git a/src/common/torgzip.h b/src/common/torgzip.h
deleted file mode 100644
index 00f62dcb45..0000000000
--- a/src/common/torgzip.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file torgzip.h
- * \brief Headers for torgzip.h
- **/
-
-#ifndef TOR_TORGZIP_H
-#define TOR_TORGZIP_H
-
-/** Enumeration of what kind of compression to use.  Only ZLIB_METHOD is
- * guaranteed to be supported by the compress/uncompress functions here;
- * GZIP_METHOD may be supported if we built against zlib version 1.2 or later
- * and is_gzip_supported() returns true. */
-typedef enum {
-  NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3
-} compress_method_t;
-
-/**
- * Enumeration to define tradeoffs between memory usage and compression level.
- * HIGH_COMPRESSION saves the most bandwidth; LOW_COMPRESSION saves the most
- * memory.
- **/
-typedef enum {
-  HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION
-} zlib_compression_level_t;
-
-int
-tor_gzip_compress(char **out, size_t *out_len,
-                  const char *in, size_t in_len,
-                  compress_method_t method);
-int
-tor_gzip_uncompress(char **out, size_t *out_len,
-                    const char *in, size_t in_len,
-                    compress_method_t method,
-                    int complete_only,
-                    int protocol_warn_level);
-
-int is_gzip_supported(void);
-
-const char *
-tor_zlib_get_version_str(void);
-
-const char *
-tor_zlib_get_header_version_str(void);
-
-compress_method_t detect_compression_method(const char *in, size_t in_len);
-
-/** Return values from tor_zlib_process; see that function's documentation for
- * details. */
-typedef enum {
-  TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR
-} tor_zlib_output_t;
-/** Internal state for an incremental zlib compression/decompression. */
-typedef struct tor_zlib_state_t tor_zlib_state_t;
-tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method,
-                               zlib_compression_level_t level);
-
-tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
-                                   char **out, size_t *out_len,
-                                   const char **in, size_t *in_len,
-                                   int finish);
-void tor_zlib_free(tor_zlib_state_t *state);
-
-size_t tor_zlib_state_size(const tor_zlib_state_t *state);
-size_t tor_zlib_get_total_allocation(void);
-
-#endif
-
diff --git a/src/common/torint.h b/src/common/torint.h
index 58c30f41a8..ee31459e94 100644
--- a/src/common/torint.h
+++ b/src/common/torint.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/torlog.h b/src/common/torlog.h
index 20b7d938f0..0149ce9a5b 100644
--- a/src/common/torlog.h
+++ b/src/common/torlog.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -99,10 +99,14 @@
 #define LD_CHANNEL   (1u<<21)
 /** Scheduler */
 #define LD_SCHED     (1u<<22)
+/** Guard nodes */
+#define LD_GUARD     (1u<<23)
+/** Generation and application of consensus diffs. */
+#define LD_CONSDIFF  (1u<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<23)
+#define LD_DOS       (1u<<25)
 /** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 24
+#define N_LOGGING_DOMAINS 26
 
 /** This log message is not safe to send to a callback-based logger
  * immediately.  Used as a flag, not a log domain. */
diff --git a/src/common/tortls.c b/src/common/tortls.c
index a4e188603c..71de59896a 100644
--- a/src/common/tortls.c
+++ b/src/common/tortls.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "orconfig.h"
 
 #define TORTLS_PRIVATE
+#define TORTLS_OPENSSL_PRIVATE
 
 #include <assert.h>
 #ifdef _WIN32 /*wrkard for dtls1.h >= 0.9.8m of "#include <winsock.h>"*/
@@ -136,6 +137,7 @@ static void tor_tls_context_decref(tor_tls_context_t *ctx);
 static void tor_tls_context_incref(tor_tls_context_t *ctx);
 
 static int check_cert_lifetime_internal(int severity, const X509 *cert,
+                                   time_t now,
                                    int past_tolerance, int future_tolerance);
 
 /** Global TLS contexts. We keep them here because nobody else needs
@@ -458,11 +460,11 @@ tor_x509_name_new(const char *cname)
  * Return a certificate on success, NULL on failure.
  */
 MOCK_IMPL(STATIC X509 *,
-          tor_tls_create_certificate,(crypto_pk_t *rsa,
-                                      crypto_pk_t *rsa_sign,
-                                      const char *cname,
-                                      const char *cname_sign,
-                                      unsigned int cert_lifetime))
+tor_tls_create_certificate,(crypto_pk_t *rsa,
+                            crypto_pk_t *rsa_sign,
+                            const char *cname,
+                            const char *cname_sign,
+                            unsigned int cert_lifetime))
 {
   /* OpenSSL generates self-signed certificates with random 64-bit serial
    * numbers, so let's do that too. */
@@ -482,8 +484,22 @@ MOCK_IMPL(STATIC X509 *,
    * then we might pick a time where we're about to expire. Lastly, be
    * sure to start on a day boundary. */
   time_t now = time(NULL);
-  start_time = crypto_rand_time_range(now - cert_lifetime, now) + 2*24*3600;
-  start_time -= start_time % (24*3600);
+  /* Our certificate lifetime will be cert_lifetime no matter what, but if we
+   * start cert_lifetime in the past, we'll have 0 real lifetime.  instead we
+   * start up to (cert_lifetime - min_real_lifetime - start_granularity) in
+   * the past. */
+  const time_t min_real_lifetime = 24*3600;
+  const time_t start_granularity = 24*3600;
+  time_t earliest_start_time = now - cert_lifetime + min_real_lifetime
+    + start_granularity;
+  /* Don't actually start in the future! */
+  if (earliest_start_time >= now)
+    earliest_start_time = now - 1;
+  start_time = crypto_rand_time_range(earliest_start_time, now);
+  /* Round the start time back to the start of a day. */
+  start_time -= start_time % start_granularity;
+
+  end_time = start_time + cert_lifetime;
 
   tor_assert(rsa);
   tor_assert(cname);
@@ -517,12 +533,12 @@ MOCK_IMPL(STATIC X509 *,
 
   if (!X509_time_adj(X509_get_notBefore(x509),0,&start_time))
     goto error;
-  end_time = start_time + cert_lifetime;
   if (!X509_time_adj(X509_get_notAfter(x509),0,&end_time))
     goto error;
   if (!X509_set_pubkey(x509, pkey))
     goto error;
-  if (!X509_sign(x509, sign_pkey, EVP_sha1()))
+
+  if (!X509_sign(x509, sign_pkey, EVP_sha256()))
     goto error;
 
   goto done;
@@ -605,6 +621,12 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256
        TLS1_TXT_DHE_RSA_WITH_AES_128_GCM_SHA256 ":"
 #endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_CCM
+       TLS1_TXT_DHE_RSA_WITH_AES_256_CCM ":"
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_CCM
+       TLS1_TXT_DHE_RSA_WITH_AES_128_CCM ":"
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256
        TLS1_TXT_DHE_RSA_WITH_AES_256_SHA256 ":"
 #endif
@@ -614,8 +636,14 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
        /* Required */
        TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
        /* Required */
-       TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
-       ;
+       TLS1_TXT_DHE_RSA_WITH_AES_128_SHA ":"
+#ifdef TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305
+       TLS1_TXT_ECDHE_RSA_WITH_CHACHA20_POLY1305 ":"
+#endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305
+       TLS1_TXT_DHE_RSA_WITH_CHACHA20_POLY1305
+#endif
+  ;
 
 /* Note: to set up your own private testing network with link crypto
  * disabled, set your Tors' cipher list to
@@ -656,7 +684,7 @@ tor_x509_cert_free(tor_x509_cert_t *cert)
  * Steals a reference to x509_cert.
  */
 MOCK_IMPL(STATIC tor_x509_cert_t *,
-          tor_x509_cert_new,(X509 *x509_cert))
+tor_x509_cert_new,(X509 *x509_cert))
 {
   tor_x509_cert_t *cert;
   EVP_PKEY *pkey;
@@ -670,12 +698,7 @@ MOCK_IMPL(STATIC tor_x509_cert_t *,
   length = i2d_X509(x509_cert, &buf);
   cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
   if (length <= 0 || buf == NULL) {
-    /* LCOV_EXCL_START for the same reason as the exclusion above */
-    tor_free(cert);
-    log_err(LD_CRYPTO, "Couldn't get length of encoded x509 certificate");
-    X509_free(x509_cert);
-    return NULL;
-    /* LCOV_EXCL_STOP */
+    goto err;
   }
   cert->encoded_len = (size_t) length;
   cert->encoded = tor_malloc(length);
@@ -690,13 +713,25 @@ MOCK_IMPL(STATIC tor_x509_cert_t *,
   if ((pkey = X509_get_pubkey(x509_cert)) &&
       (rsa = EVP_PKEY_get1_RSA(pkey))) {
     crypto_pk_t *pk = crypto_new_pk_from_rsa_(rsa);
-    crypto_pk_get_common_digests(pk, &cert->pkey_digests);
+    if (crypto_pk_get_common_digests(pk, &cert->pkey_digests) < 0) {
+      crypto_pk_free(pk);
+      EVP_PKEY_free(pkey);
+      goto err;
+    }
+
     cert->pkey_digests_set = 1;
     crypto_pk_free(pk);
     EVP_PKEY_free(pkey);
   }
 
   return cert;
+ err:
+  /* LCOV_EXCL_START for the same reason as the exclusion above */
+  tor_free(cert);
+  log_err(LD_CRYPTO, "Couldn't wrap encoded X509 certificate.");
+  X509_free(x509_cert);
+  return NULL;
+  /* LCOV_EXCL_STOP */
 }
 
 /** Return a new copy of <b>cert</b>. */
@@ -800,8 +835,8 @@ tor_tls_context_decref(tor_tls_context_t *ctx)
 /** Set *<b>link_cert_out</b> and *<b>id_cert_out</b> to the link certificate
  * and ID certificate that we're currently using for our V3 in-protocol
  * handshake's certificate chain.  If <b>server</b> is true, provide the certs
- * that we use in server mode; otherwise, provide the certs that we use in
- * client mode. */
+ * that we use in server mode (auth, ID); otherwise, provide the certs that we
+ * use in client mode. (link, ID) */
 int
 tor_tls_get_my_certs(int server,
                      const tor_x509_cert_t **link_cert_out,
@@ -831,7 +866,7 @@ tor_tls_get_my_client_auth_key(void)
 
 /**
  * Return a newly allocated copy of the public key that a certificate
- * certifies.  Return NULL if the cert's key is not RSA.
+ * certifies. Watch out! This returns NULL if the cert's key is not RSA.
  */
 crypto_pk_t *
 tor_tls_cert_get_key(tor_x509_cert_t *cert)
@@ -886,6 +921,7 @@ int
 tor_tls_cert_is_valid(int severity,
                       const tor_x509_cert_t *cert,
                       const tor_x509_cert_t *signing_cert,
+                      time_t now,
                       int check_rsa_1024)
 {
   check_no_tls_errors();
@@ -905,7 +941,7 @@ tor_tls_cert_is_valid(int severity,
 
   /* okay, the signature checked out right.  Now let's check the check the
    * lifetime. */
-  if (check_cert_lifetime_internal(severity, cert->cert,
+  if (check_cert_lifetime_internal(severity, cert->cert, now,
                                    48*60*60, 30*24*60*60) < 0)
     goto bad;
 
@@ -1050,6 +1086,8 @@ tor_tls_context_init_one(tor_tls_context_t **ppcontext,
 /** The group we should use for ecdhe when none was selected. */
 #define  NID_tor_default_ecdhe_group NID_X9_62_prime256v1
 
+#define RSA_LINK_KEY_BITS 2048
+
 /** Create a new TLS context for use with Tor TLS handshakes.
  * <b>identity</b> should be set to the identity key used to sign the
  * certificate.
@@ -1075,7 +1113,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
   /* Generate short-term RSA key for use with TLS. */
   if (!(rsa = crypto_pk_new()))
     goto error;
-  if (crypto_pk_generate_key(rsa)<0)
+  if (crypto_pk_generate_key_with_bits(rsa, RSA_LINK_KEY_BITS)<0)
     goto error;
   if (!is_client) {
     /* Generate short-term RSA key for use in the in-protocol ("v3")
@@ -2073,13 +2111,13 @@ tor_tls_get_own_cert,(tor_tls_t *tls))
 
 /** Warn that a certificate lifetime extends through a certain range. */
 static void
-log_cert_lifetime(int severity, const X509 *cert, const char *problem)
+log_cert_lifetime(int severity, const X509 *cert, const char *problem,
+                  time_t now)
 {
   BIO *bio = NULL;
   BUF_MEM *buf;
   char *s1=NULL, *s2=NULL;
   char mytime[33];
-  time_t now = time(NULL);
   struct tm tm;
   size_t n;
 
@@ -2227,6 +2265,7 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity_key)
  */
 int
 tor_tls_check_lifetime(int severity, tor_tls_t *tls,
+                       time_t now,
                        int past_tolerance, int future_tolerance)
 {
   X509 *cert;
@@ -2235,7 +2274,7 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls,
   if (!(cert = SSL_get_peer_certificate(tls->ssl)))
     goto done;
 
-  if (check_cert_lifetime_internal(severity, cert,
+  if (check_cert_lifetime_internal(severity, cert, now,
                                    past_tolerance, future_tolerance) < 0)
     goto done;
 
@@ -2251,30 +2290,48 @@ tor_tls_check_lifetime(int severity, tor_tls_t *tls,
 
 /** Helper: check whether <b>cert</b> is expired give or take
  * <b>past_tolerance</b> seconds, or not-yet-valid give or take
- * <b>future_tolerance</b> seconds.  If it is live, return 0.  If it is not
- * live, log a message and return -1. */
+ * <b>future_tolerance</b> seconds.  (Relative to the current time
+ * <b>now</b>.)  If it is live, return 0.  If it is not live, log a message
+ * and return -1. */
 static int
 check_cert_lifetime_internal(int severity, const X509 *cert,
+                             time_t now,
                              int past_tolerance, int future_tolerance)
 {
-  time_t now, t;
-
-  now = time(NULL);
+  time_t t;
 
   t = now + future_tolerance;
   if (X509_cmp_time(X509_get_notBefore_const(cert), &t) > 0) {
-    log_cert_lifetime(severity, cert, "not yet valid");
+    log_cert_lifetime(severity, cert, "not yet valid", now);
     return -1;
   }
   t = now - past_tolerance;
   if (X509_cmp_time(X509_get_notAfter_const(cert), &t) < 0) {
-    log_cert_lifetime(severity, cert, "already expired");
+    log_cert_lifetime(severity, cert, "already expired", now);
     return -1;
   }
 
   return 0;
 }
 
+#ifdef TOR_UNIT_TESTS
+/* Testing only: return a new x509 cert with the same contents as <b>inp</b>,
+   but with the expiration time <b>new_expiration_time</b>, signed with
+   <b>signing_key</b>. */
+STATIC tor_x509_cert_t *
+tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
+                                 time_t new_expiration_time,
+                                 crypto_pk_t *signing_key)
+{
+  X509 *newc = X509_dup(inp->cert);
+  X509_time_adj(X509_get_notAfter(newc), 0, &new_expiration_time);
+  EVP_PKEY *pk = crypto_pk_get_evp_pkey_(signing_key, 1);
+  tor_assert(X509_sign(newc, pk, EVP_sha256()));
+  EVP_PKEY_free(pk);
+  return tor_x509_cert_new(newc);
+}
+#endif
+
 /** Return the number of bytes available for reading from <b>tls</b>.
  */
 int
@@ -2493,6 +2550,28 @@ tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out))
   return 0;
 }
 
+/** Using the RFC5705 key material exporting construction, and the
+ * provided <b>context</b> (<b>context_len</b> bytes long) and
+ * <b>label</b> (a NUL-terminated string), compute a 32-byte secret in
+ * <b>secrets_out</b> that only the parties to this TLS session can
+ * compute.  Return 0 on success and -1 on failure.
+ */
+MOCK_IMPL(int,
+tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
+                             const uint8_t *context,
+                             size_t context_len,
+                             const char *label))
+{
+  tor_assert(tls);
+  tor_assert(tls->ssl);
+
+  int r = SSL_export_keying_material(tls->ssl,
+                                     secrets_out, DIGEST256_LEN,
+                                     label, strlen(label),
+                                     context, context_len, 1);
+  return (r == 1) ? 0 : -1;
+}
+
 /** Examine the amount of memory used and available for buffers in <b>tls</b>.
  * Set *<b>rbuf_capacity</b> to the amount of storage allocated for the read
  * buffer and *<b>rbuf_bytes</b> to the amount actually used.
diff --git a/src/common/tortls.h b/src/common/tortls.h
index f018c45c82..f430aff70b 100644
--- a/src/common/tortls.h
+++ b/src/common/tortls.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TORTLS_H
@@ -63,12 +63,17 @@ typedef enum {
 } tor_tls_state_t;
 #define tor_tls_state_bitfield_t ENUM_BF(tor_tls_state_t)
 
+struct x509_st;
+struct ssl_st;
+struct ssl_ctx_st;
+struct ssl_session_st;
+
 /** Holds a SSL_CTX object and related state used to configure TLS
  * connections.
  */
 typedef struct tor_tls_context_t {
   int refcnt;
-  SSL_CTX *ctx;
+  struct ssl_ctx_st *ctx;
   tor_x509_cert_t *my_link_cert;
   tor_x509_cert_t *my_id_cert;
   tor_x509_cert_t *my_auth_cert;
@@ -78,7 +83,7 @@ typedef struct tor_tls_context_t {
 
 /** Structure that we use for a single certificate. */
 struct tor_x509_cert_t {
-  X509 *cert;
+  struct x509_st *cert;
   uint8_t *encoded;
   size_t encoded_len;
   unsigned pkey_digests_set : 1;
@@ -92,7 +97,7 @@ struct tor_x509_cert_t {
 struct tor_tls_t {
   uint32_t magic;
   tor_tls_context_t *context; /** A link to the context object for this tls. */
-  SSL *ssl; /**< An OpenSSL SSL object. */
+  struct ssl_st *ssl; /**< An OpenSSL SSL object. */
   int socket; /**< The underlying file descriptor for this TLS connection. */
   char *address; /**< An address to log when describing this connection. */
   tor_tls_state_bitfield_t state : 3; /**< The current SSL state,
@@ -128,35 +133,45 @@ struct tor_tls_t {
 STATIC int tor_errno_to_tls_error(int e);
 STATIC int tor_tls_get_error(tor_tls_t *tls, int r, int extra,
                   const char *doing, int severity, int domain);
-STATIC tor_tls_t *tor_tls_get_by_ssl(const SSL *ssl);
+STATIC tor_tls_t *tor_tls_get_by_ssl(const struct ssl_st *ssl);
 STATIC void tor_tls_allocate_tor_tls_object_ex_data_index(void);
+#ifdef TORTLS_OPENSSL_PRIVATE
 STATIC int always_accept_verify_cb(int preverify_ok, X509_STORE_CTX *x509_ctx);
-STATIC int tor_tls_classify_client_ciphers(const SSL *ssl,
+STATIC int tor_tls_classify_client_ciphers(const struct ssl_st *ssl,
                                            STACK_OF(SSL_CIPHER) *peer_ciphers);
-STATIC int tor_tls_client_is_using_v2_ciphers(const SSL *ssl);
+#endif
+STATIC int tor_tls_client_is_using_v2_ciphers(const struct ssl_st *ssl);
 MOCK_DECL(STATIC void, try_to_extract_certs_from_tls,
-          (int severity, tor_tls_t *tls, X509 **cert_out, X509 **id_cert_out));
+          (int severity, tor_tls_t *tls, struct x509_st **cert_out,
+           struct x509_st **id_cert_out));
 #ifndef HAVE_SSL_SESSION_GET_MASTER_KEY
-STATIC size_t SSL_SESSION_get_master_key(SSL_SESSION *s, uint8_t *out,
+STATIC size_t SSL_SESSION_get_master_key(struct ssl_session_st *s,
+                                         uint8_t *out,
                                          size_t len);
 #endif
-STATIC void tor_tls_debug_state_callback(const SSL *ssl, int type, int val);
-STATIC void tor_tls_server_info_callback(const SSL *ssl, int type, int val);
-STATIC int tor_tls_session_secret_cb(SSL *ssl, void *secret,
+STATIC void tor_tls_debug_state_callback(const struct ssl_st *ssl,
+                                         int type, int val);
+STATIC void tor_tls_server_info_callback(const struct ssl_st *ssl,
+                                         int type, int val);
+#ifdef TORTLS_OPENSSL_PRIVATE
+STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
                             int *secret_len,
                             STACK_OF(SSL_CIPHER) *peer_ciphers,
                             CONST_IF_OPENSSL_1_1_API SSL_CIPHER **cipher,
                             void *arg);
 STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
                              uint16_t cipher);
-MOCK_DECL(STATIC X509*, tor_tls_create_certificate,(crypto_pk_t *rsa,
+#endif
+MOCK_DECL(STATIC struct x509_st *, tor_tls_create_certificate,
+                                                   (crypto_pk_t *rsa,
                                                     crypto_pk_t *rsa_sign,
                                                     const char *cname,
                                                     const char *cname_sign,
                                                   unsigned int cert_lifetime));
 STATIC tor_tls_context_t *tor_tls_context_new(crypto_pk_t *identity,
                    unsigned int key_lifetime, unsigned flags, int is_client);
-MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,(X509 *x509_cert));
+MOCK_DECL(STATIC tor_x509_cert_t *, tor_x509_cert_new,
+          (struct x509_st *x509_cert));
 STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext,
                                     crypto_pk_t *identity,
                                     unsigned int key_lifetime,
@@ -172,10 +187,16 @@ extern tor_tls_context_t *client_tls_context;
 extern uint16_t v2_cipher_list[];
 extern uint64_t total_bytes_written_over_tls;
 extern uint64_t total_bytes_written_by_tls;
+
+STATIC tor_x509_cert_t *tor_x509_cert_replace_expiration(
+                                               const tor_x509_cert_t *inp,
+                                               time_t new_expiration_time,
+                                               crypto_pk_t *signing_key);
 #endif
 
 #endif /* endif TORTLS_PRIVATE */
 
+tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 const char *tor_tls_err_to_string(int err);
 void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz);
 
@@ -197,12 +218,12 @@ void tor_tls_set_renegotiate_callback(tor_tls_t *tls,
 int tor_tls_is_server(tor_tls_t *tls);
 void tor_tls_free(tor_tls_t *tls);
 int tor_tls_peer_has_cert(tor_tls_t *tls);
-tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 MOCK_DECL(tor_x509_cert_t *,tor_tls_get_peer_cert,(tor_tls_t *tls));
 MOCK_DECL(tor_x509_cert_t *,tor_tls_get_own_cert,(tor_tls_t *tls));
 int tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity);
 int tor_tls_check_lifetime(int severity,
-                           tor_tls_t *tls, int past_tolerance,
+                           tor_tls_t *tls, time_t now,
+                           int past_tolerance,
                            int future_tolerance);
 MOCK_DECL(int, tor_tls_read, (tor_tls_t *tls, char *cp, size_t len));
 int tor_tls_write(tor_tls_t *tls, const char *cp, size_t n);
@@ -228,6 +249,11 @@ int tor_tls_used_v1_handshake(tor_tls_t *tls);
 int tor_tls_get_num_server_handshakes(tor_tls_t *tls);
 int tor_tls_server_got_renegotiate(tor_tls_t *tls);
 MOCK_DECL(int,tor_tls_get_tlssecrets,(tor_tls_t *tls, uint8_t *secrets_out));
+MOCK_DECL(int,tor_tls_export_key_material,(
+                     tor_tls_t *tls, uint8_t *secrets_out,
+                     const uint8_t *context,
+                     size_t context_len,
+                     const char *label));
 
 /* Log and abort if there are unhandled TLS errors in OpenSSL's error stack.
  */
@@ -256,6 +282,7 @@ MOCK_DECL(int,tor_tls_cert_matches_key,(const tor_tls_t *tls,
 int tor_tls_cert_is_valid(int severity,
                           const tor_x509_cert_t *cert,
                           const tor_x509_cert_t *signing_cert,
+                          time_t now,
                           int check_rsa_1024);
 const char *tor_tls_get_ciphersuite_name(tor_tls_t *tls);
 
diff --git a/src/common/util.c b/src/common/util.c
index d2cbacde31..5b47028097 100644
--- a/src/common/util.c
+++ b/src/common/util.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -79,7 +79,7 @@
 #include <malloc/malloc.h>
 #endif
 #ifdef HAVE_MALLOC_H
-#if !defined(OPENBSD) && !defined(__FreeBSD__)
+#if !defined(OpenBSD) && !defined(__FreeBSD__)
 /* OpenBSD has a malloc.h, but for our purposes, it only exists in order to
  * scold us for being so stupid as to autodetect its presence.  To be fair,
  * they've done this since 1996, when autoconf was only 5 years old. */
@@ -187,8 +187,9 @@ tor_malloc_zero_(size_t size DMALLOC_PARAMS)
  * 0xfffe0001. */
 #define SQRT_SIZE_MAX_P1 (((size_t)1) << (sizeof(size_t)*4))
 
-/** Return non-zero if and only if the product of the arguments is exact. */
-static inline int
+/** Return non-zero if and only if the product of the arguments is exact,
+ * and cannot overflow. */
+int
 size_mul_check(const size_t x, const size_t y)
 {
   /* This first check is equivalent to
@@ -202,15 +203,6 @@ size_mul_check(const size_t x, const size_t y)
           x <= SIZE_MAX / y);
 }
 
-#ifdef TOR_UNIT_TESTS
-/** Exposed for unit tests only */
-int
-size_mul_check__(const size_t x, const size_t y)
-{
-  return size_mul_check(x,y);
-}
-#endif
-
 /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill
  * the memory with zero bytes, and return a pointer to the result.
  * Log and terminate the process on error.  (Same as
@@ -712,6 +704,19 @@ tor_strisnonupper(const char *s)
   return 1;
 }
 
+/** Return true iff every character in <b>s</b> is whitespace space; else
+ * return false. */
+int
+tor_strisspace(const char *s)
+{
+  while (*s) {
+    if (!TOR_ISSPACE(*s))
+      return 0;
+    s++;
+  }
+  return 1;
+}
+
 /** As strcmp, except that either string may be NULL.  The NULL string is
  * considered to be before any non-NULL string. */
 int
@@ -1803,17 +1808,26 @@ format_iso_time_nospace_usec(char *buf, const struct timeval *tv)
 /** Given an ISO-formatted UTC time value (after the epoch) in <b>cp</b>,
  * parse it and store its value in *<b>t</b>.  Return 0 on success, -1 on
  * failure.  Ignore extraneous stuff in <b>cp</b> after the end of the time
- * string, unless <b>strict</b> is set. */
+ * string, unless <b>strict</b> is set. If <b>nospace</b> is set,
+ * expect the YYYY-MM-DDTHH:MM:SS format. */
 int
-parse_iso_time_(const char *cp, time_t *t, int strict)
+parse_iso_time_(const char *cp, time_t *t, int strict, int nospace)
 {
   struct tm st_tm;
   unsigned int year=0, month=0, day=0, hour=0, minute=0, second=0;
   int n_fields;
-  char extra_char;
-  n_fields = tor_sscanf(cp, "%u-%2u-%2u %2u:%2u:%2u%c", &year, &month,
-                        &day, &hour, &minute, &second, &extra_char);
-  if (strict ? (n_fields != 6) : (n_fields < 6)) {
+  char extra_char, separator_char;
+  n_fields = tor_sscanf(cp, "%u-%2u-%2u%c%2u:%2u:%2u%c",
+                        &year, &month, &day,
+                        &separator_char,
+                        &hour, &minute, &second, &extra_char);
+  if (strict ? (n_fields != 7) : (n_fields < 7)) {
+    char *esc = esc_for_log(cp);
+    log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
+    tor_free(esc);
+    return -1;
+  }
+  if (separator_char != (nospace ? 'T' : ' ')) {
     char *esc = esc_for_log(cp);
     log_warn(LD_GENERAL, "ISO time %s was unparseable", esc);
     tor_free(esc);
@@ -1855,7 +1869,16 @@ parse_iso_time_(const char *cp, time_t *t, int strict)
 int
 parse_iso_time(const char *cp, time_t *t)
 {
-  return parse_iso_time_(cp, t, 1);
+  return parse_iso_time_(cp, t, 1, 0);
+}
+
+/**
+ * As parse_iso_time, but parses a time encoded by format_iso_time_nospace().
+ */
+int
+parse_iso_time_nospace(const char *cp, time_t *t)
+{
+  return parse_iso_time_(cp, t, 1, 1);
 }
 
 /** Given a <b>date</b> in one of the three formats allowed by HTTP (ugh),
@@ -1930,7 +1953,7 @@ parse_http_time(const char *date, struct tm *tm)
 
 /** Given an <b>interval</b> in seconds, try to write it to the
  * <b>out_len</b>-byte buffer in <b>out</b> in a human-readable form.
- * Return 0 on success, -1 on failure.
+ * Returns a non-negative integer on success, -1 on failure.
  */
 int
 format_time_interval(char *out, size_t out_len, long interval)
@@ -2095,7 +2118,7 @@ read_all(tor_socket_t fd, char *buf, size_t count, int isSocket)
     return -1;
   }
 
-  while (numread != count) {
+  while (numread < count) {
     if (isSocket)
       result = tor_socket_recv(fd, buf+numread, count-numread, 0);
     else
@@ -2270,10 +2293,14 @@ check_private_dir,(const char *dirname, cpd_check_t check,
        * permissions on the directory will be checked again below.*/
       fd = open(sandbox_intern_string(dirname), O_NOFOLLOW);
 
-      if (fd == -1)
+      if (fd == -1) {
+        log_warn(LD_FS, "Could not reopen recently created directory %s: %s",
+                 dirname,
+                 strerror(errno));
         return -1;
-      else
+      } else {
         close(fd);
+      }
 
     } else if (!(check & CPD_CHECK)) {
       log_warn(LD_FS, "Directory %s does not exist.", dirname);
@@ -2601,6 +2628,14 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
 
   if (file_data->rename_on_close) {
     tor_assert(file_data->tempname && file_data->filename);
+    if (!abort_write) {
+      tor_assert(strcmp(file_data->filename, file_data->tempname));
+      if (replace_file(file_data->tempname, file_data->filename)) {
+        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
+                 strerror(errno));
+        abort_write = r = -1;
+      }
+    }
     if (abort_write) {
       int res = unlink(file_data->tempname);
       if (res != 0) {
@@ -2609,13 +2644,6 @@ finish_writing_to_file_impl(open_file_t *file_data, int abort_write)
                  file_data->tempname, strerror(errno));
         r = -1;
       }
-    } else {
-      tor_assert(strcmp(file_data->filename, file_data->tempname));
-      if (replace_file(file_data->tempname, file_data->filename)) {
-        log_warn(LD_FS, "Error replacing \"%s\": %s", file_data->filename,
-                 strerror(errno));
-        r = -1;
-      }
     }
   }
 
@@ -3017,135 +3045,39 @@ unescape_string(const char *s, char **result, size_t *size_out)
   }
 }
 
-/** Given a string containing part of a configuration file or similar format,
- * advance past comments and whitespace and try to parse a single line.  If we
- * parse a line successfully, set *<b>key_out</b> to a new string holding the
- * key portion and *<b>value_out</b> to a new string holding the value portion
- * of the line, and return a pointer to the start of the next line.  If we run
- * out of data, return a pointer to the end of the string.  If we encounter an
- * error, return NULL and set *<b>err_out</b> (if provided) to an error
- * message.
- */
-const char *
-parse_config_line_from_str_verbose(const char *line, char **key_out,
-                                   char **value_out,
-                                   const char **err_out)
+/** Removes enclosing quotes from <b>path</b> and unescapes quotes between the
+ * enclosing quotes. Backslashes are not unescaped. Return the unquoted
+ * <b>path</b> on sucess or 0 if <b>path</b> is not quoted correctly. */
+char *
+get_unquoted_path(const char *path)
 {
-  /*
-    See torrc_format.txt for a description of the (silly) format this parses.
-   */
-  const char *key, *val, *cp;
-  int continuation = 0;
-
-  tor_assert(key_out);
-  tor_assert(value_out);
+  size_t len = strlen(path);
 
-  *key_out = *value_out = NULL;
-  key = val = NULL;
-  /* Skip until the first keyword. */
-  while (1) {
-    while (TOR_ISSPACE(*line))
-      ++line;
-    if (*line == '#') {
-      while (*line && *line != '\n')
-        ++line;
-    } else {
-      break;
-    }
+  if (len == 0) {
+    return tor_strdup("");
   }
 
-  if (!*line) { /* End of string? */
-    *key_out = *value_out = NULL;
-    return line;
+  int has_start_quote = (path[0] == '\"');
+  int has_end_quote = (len > 0 && path[len-1] == '\"');
+  if (has_start_quote != has_end_quote || (len == 1 && has_start_quote)) {
+    return NULL;
   }
 
-  /* Skip until the next space or \ followed by newline. */
-  key = line;
-  while (*line && !TOR_ISSPACE(*line) && *line != '#' &&
-         ! (line[0] == '\\' && line[1] == '\n'))
-    ++line;
-  *key_out = tor_strndup(key, line-key);
-
-  /* Skip until the value. */
-  while (*line == ' ' || *line == '\t')
-    ++line;
-
-  val = line;
-
-  /* Find the end of the line. */
-  if (*line == '\"') { // XXX No continuation handling is done here
-    if (!(line = unescape_string(line, value_out, NULL))) {
-      if (err_out)
-        *err_out = "Invalid escape sequence in quoted string";
-      return NULL;
-    }
-    while (*line == ' ' || *line == '\t')
-      ++line;
-    if (*line == '\r' && *(++line) == '\n')
-      ++line;
-    if (*line && *line != '#' && *line != '\n') {
-      if (err_out)
-        *err_out = "Excess data after quoted string";
+  char *unquoted_path = tor_malloc(len - has_start_quote - has_end_quote + 1);
+  char *s = unquoted_path;
+  size_t i;
+  for (i = has_start_quote; i < len - has_end_quote; i++) {
+    if (path[i] == '\"' && (i > 0 && path[i-1] == '\\')) {
+      *(s-1) = path[i];
+    } else if (path[i] != '\"') {
+      *s++ = path[i];
+    } else {  /* unescaped quote */
+      tor_free(unquoted_path);
       return NULL;
     }
-  } else {
-    /* Look for the end of the line. */
-    while (*line && *line != '\n' && (*line != '#' || continuation)) {
-      if (*line == '\\' && line[1] == '\n') {
-        continuation = 1;
-        line += 2;
-      } else if (*line == '#') {
-        do {
-          ++line;
-        } while (*line && *line != '\n');
-        if (*line == '\n')
-          ++line;
-      } else {
-        ++line;
-      }
-    }
-
-    if (*line == '\n') {
-      cp = line++;
-    } else {
-      cp = line;
-    }
-    /* Now back cp up to be the last nonspace character */
-    while (cp>val && TOR_ISSPACE(*(cp-1)))
-      --cp;
-
-    tor_assert(cp >= val);
-
-    /* Now copy out and decode the value. */
-    *value_out = tor_strndup(val, cp-val);
-    if (continuation) {
-      char *v_out, *v_in;
-      v_out = v_in = *value_out;
-      while (*v_in) {
-        if (*v_in == '#') {
-          do {
-            ++v_in;
-          } while (*v_in && *v_in != '\n');
-          if (*v_in == '\n')
-            ++v_in;
-        } else if (v_in[0] == '\\' && v_in[1] == '\n') {
-          v_in += 2;
-        } else {
-          *v_out++ = *v_in++;
-        }
-      }
-      *v_out = '\0';
-    }
   }
-
-  if (*line == '#') {
-    do {
-      ++line;
-    } while (*line && *line != '\n');
-  }
-  while (TOR_ISSPACE(*line)) ++line;
-
-  return line;
+  *s = '\0';
+  return unquoted_path;
 }
 
 /** Expand any homedir prefix on <b>filename</b>; return a newly allocated
@@ -3534,6 +3466,17 @@ smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
   smartlist_add(sl, str);
 }
 
+/** Append a copy of string to sl */
+void
+smartlist_add_strdup(struct smartlist_t *sl, const char *string)
+{
+  char *copy;
+
+  copy = tor_strdup(string);
+
+  smartlist_add(sl, copy);
+}
+
 /** Return a new list containing the filenames in the directory <b>dirname</b>.
  * Return NULL on error or if <b>dirname</b> is not a directory.
  */
@@ -3567,7 +3510,7 @@ tor_listdir, (const char *dirname))
 #endif
     if (strcmp(name, ".") &&
         strcmp(name, "..")) {
-      smartlist_add(result, tor_strdup(name));
+      smartlist_add_strdup(result, name);
     }
     if (!FindNextFile(handle, &findData)) {
       DWORD err;
@@ -3593,7 +3536,7 @@ tor_listdir, (const char *dirname))
     if (!strcmp(de->d_name, ".") ||
         !strcmp(de->d_name, ".."))
       continue;
-    smartlist_add(result, tor_strdup(de->d_name));
+    smartlist_add_strdup(result, de->d_name);
   }
   closedir(d);
 #endif
@@ -4136,10 +4079,10 @@ tor_process_get_stdout_pipe(process_handle_t *process_handle)
 }
 #else
 /* DOCDOC tor_process_get_stdout_pipe */
-FILE *
+int
 tor_process_get_stdout_pipe(process_handle_t *process_handle)
 {
-  return process_handle->stdout_handle;
+  return process_handle->stdout_pipe;
 }
 #endif
 
@@ -4570,10 +4513,6 @@ tor_spawn_background(const char *const filename, const char **argv,
     log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
              "nonblocking in parent process: %s", strerror(errno));
   }
-  /* Open the buffered IO streams */
-  process_handle->stdout_handle = fdopen(process_handle->stdout_pipe, "r");
-  process_handle->stderr_handle = fdopen(process_handle->stderr_pipe, "r");
-  process_handle->stdin_handle = fdopen(process_handle->stdin_pipe, "r");
 
   *process_handle_out = process_handle;
   return process_handle->status;
@@ -4620,14 +4559,9 @@ tor_process_handle_destroy,(process_handle_t *process_handle,
   if (process_handle->stdin_pipe)
     CloseHandle(process_handle->stdin_pipe);
 #else
-  if (process_handle->stdout_handle)
-    fclose(process_handle->stdout_handle);
-
-  if (process_handle->stderr_handle)
-    fclose(process_handle->stderr_handle);
-
-  if (process_handle->stdin_handle)
-    fclose(process_handle->stdin_handle);
+  close(process_handle->stdout_pipe);
+  close(process_handle->stderr_pipe);
+  close(process_handle->stdin_pipe);
 
   clear_waitpid_callback(process_handle->waitpid_cb);
 #endif
@@ -4864,7 +4798,7 @@ get_current_process_environment_variables(void)
 
   char **environ_tmp; /* Not const char ** ? Really? */
   for (environ_tmp = get_environment(); *environ_tmp; ++environ_tmp) {
-    smartlist_add(sl, tor_strdup(*environ_tmp));
+    smartlist_add_strdup(sl, *environ_tmp);
   }
 
   return sl;
@@ -4913,7 +4847,7 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count,
   if (count > SIZE_T_CEILING || count > SSIZE_MAX)
     return -1;
 
-  while (numread != count) {
+  while (numread < count) {
     /* Check if there is anything to read */
     retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
     if (!retval) {
@@ -4959,19 +4893,19 @@ tor_read_all_handle(HANDLE h, char *buf, size_t count,
   return (ssize_t)numread;
 }
 #else
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
+/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
  * <b>process</b> is NULL, the function will return immediately if there is
  * nothing more to read. Otherwise data will be read until end of file, or
  * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
  * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
  * file has been reached. */
 ssize_t
-tor_read_all_handle(FILE *h, char *buf, size_t count,
+tor_read_all_handle(int fd, char *buf, size_t count,
                     const process_handle_t *process,
                     int *eof)
 {
   size_t numread = 0;
-  char *retval;
+  ssize_t result;
 
   if (eof)
     *eof = 0;
@@ -4979,34 +4913,28 @@ tor_read_all_handle(FILE *h, char *buf, size_t count,
   if (count > SIZE_T_CEILING || count > SSIZE_MAX)
     return -1;
 
-  while (numread != count) {
-    /* Use fgets because that is what we use in log_from_pipe() */
-    retval = fgets(buf+numread, (int)(count-numread), h);
-    if (NULL == retval) {
-      if (feof(h)) {
-        log_debug(LD_GENERAL, "fgets() reached end of file");
-        if (eof)
-          *eof = 1;
+  while (numread < count) {
+    result = read(fd, buf+numread, count-numread);
+
+    if (result == 0) {
+      log_debug(LD_GENERAL, "read() reached end of file");
+      if (eof)
+        *eof = 1;
+      break;
+    } else if (result < 0 && errno == EAGAIN) {
+      if (process)
+        continue;
+      else
         break;
-      } else {
-        if (EAGAIN == errno) {
-          if (process)
-            continue;
-          else
-            break;
-        } else {
-          log_warn(LD_GENERAL, "fgets() from handle failed: %s",
-                   strerror(errno));
-          return -1;
-        }
-      }
+    } else if (result < 0) {
+      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
+      return -1;
     }
-    tor_assert(retval != NULL);
-    tor_assert(strlen(retval) + numread <= count);
-    numread += strlen(retval);
+
+    numread += result;
   }
 
-  log_debug(LD_GENERAL, "fgets() read %d bytes from handle", (int)numread);
+  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
   return (ssize_t)numread;
 }
 #endif
@@ -5020,7 +4948,7 @@ tor_read_all_from_process_stdout(const process_handle_t *process_handle,
   return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
                              process_handle);
 #else
-  return tor_read_all_handle(process_handle->stdout_handle, buf, count,
+  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
                              process_handle, NULL);
 #endif
 }
@@ -5034,7 +4962,7 @@ tor_read_all_from_process_stderr(const process_handle_t *process_handle,
   return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
                              process_handle);
 #else
-  return tor_read_all_handle(process_handle->stderr_handle, buf, count,
+  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
                              process_handle, NULL);
 #endif
 }
@@ -5228,11 +5156,10 @@ log_from_handle(HANDLE *pipe, int severity)
 #else
 
 /** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
+ *  <b>fd</b>. Return NULL on error, and set
  *  <b>stream_status_out</b> appropriately. */
 MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (FILE *handle,
-                            enum stream_status *stream_status_out))
+tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
 {
   enum stream_status stream_status;
   char stdout_buf[400];
@@ -5241,13 +5168,13 @@ tor_get_lines_from_handle, (FILE *handle,
   while (1) {
     memset(stdout_buf, 0, sizeof(stdout_buf));
 
-    stream_status = get_string_from_pipe(handle,
+    stream_status = get_string_from_pipe(fd,
                                          stdout_buf, sizeof(stdout_buf) - 1);
     if (stream_status != IO_STREAM_OKAY)
       goto done;
 
     if (!lines) lines = smartlist_new();
-    smartlist_add(lines, tor_strdup(stdout_buf));
+    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
   }
 
  done:
@@ -5255,20 +5182,20 @@ tor_get_lines_from_handle, (FILE *handle,
   return lines;
 }
 
-/** Read from stream, and send lines to log at the specified log level.
+/** Read from fd, and send lines to log at the specified log level.
  * Returns 1 if stream is closed normally, -1 if there is a error reading, and
  * 0 otherwise. Handles lines from tor-fw-helper and
  * tor_spawn_background() specially.
  */
 static int
-log_from_pipe(FILE *stream, int severity, const char *executable,
+log_from_pipe(int fd, int severity, const char *executable,
               int *child_status)
 {
   char buf[256];
   enum stream_status r;
 
   for (;;) {
-    r = get_string_from_pipe(stream, buf, sizeof(buf) - 1);
+    r = get_string_from_pipe(fd, buf, sizeof(buf) - 1);
 
     if (r == IO_STREAM_CLOSED) {
       return 1;
@@ -5293,7 +5220,7 @@ log_from_pipe(FILE *stream, int severity, const char *executable,
 }
 #endif
 
-/** Reads from <b>stream</b> and stores input in <b>buf_out</b> making
+/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
  *  sure it's below <b>count</b> bytes.
  *  If the string has a trailing newline, we strip it off.
  *
@@ -5309,52 +5236,28 @@ log_from_pipe(FILE *stream, int severity, const char *executable,
  * IO_STREAM_OKAY: If everything went okay and we got a string
  *  in <b>buf_out</b>. */
 enum stream_status
-get_string_from_pipe(FILE *stream, char *buf_out, size_t count)
+get_string_from_pipe(int fd, char *buf_out, size_t count)
 {
-  char *retval;
-  size_t len;
+  ssize_t ret;
 
   tor_assert(count <= INT_MAX);
 
-  retval = fgets(buf_out, (int)count, stream);
+  ret = read(fd, buf_out, count);
 
-  if (!retval) {
-    if (feof(stream)) {
-      /* Program has closed stream (probably it exited) */
-      /* TODO: check error */
-      return IO_STREAM_CLOSED;
-    } else {
-      if (EAGAIN == errno) {
-        /* Nothing more to read, try again next time */
-        return IO_STREAM_EAGAIN;
-      } else {
-        /* There was a problem, abandon this child process */
-        return IO_STREAM_TERM;
-      }
-    }
-  } else {
-    len = strlen(buf_out);
-    if (len == 0) {
-      /* this probably means we got a NUL at the start of the string. */
-      return IO_STREAM_EAGAIN;
-    }
-
-    if (buf_out[len - 1] == '\n') {
-      /* Remove the trailing newline */
-      buf_out[len - 1] = '\0';
-    } else {
-      /* No newline; check whether we overflowed the buffer */
-      if (!feof(stream))
-        log_info(LD_GENERAL,
-                 "Line from stream was truncated: %s", buf_out);
-      /* TODO: What to do with this error? */
-    }
+  if (ret == 0)
+    return IO_STREAM_CLOSED;
+  else if (ret < 0 && errno == EAGAIN)
+    return IO_STREAM_EAGAIN;
+  else if (ret < 0)
+    return IO_STREAM_TERM;
 
-    return IO_STREAM_OKAY;
-  }
+  if (buf_out[ret - 1] == '\n') {
+    /* Remove the trailing newline */
+    buf_out[ret - 1] = '\0';
+  } else
+    buf_out[ret] = '\0';
 
-  /* We should never get here */
-  return IO_STREAM_TERM;
+  return IO_STREAM_OKAY;
 }
 
 /** Parse a <b>line</b> from tor-fw-helper and issue an appropriate
@@ -5591,7 +5494,7 @@ tor_check_port_forwarding(const char *filename,
 #ifdef _WIN32
     stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO);
 #else
-    stderr_status = log_from_pipe(child_handle->stderr_handle,
+    stderr_status = log_from_pipe(child_handle->stderr_pipe,
                                   LOG_INFO, filename, &retval);
 #endif
     if (handle_fw_helper_output(filename, child_handle) < 0) {
diff --git a/src/common/util.h b/src/common/util.h
index 479fc8d610..d56abcee2e 100644
--- a/src/common/util.h
+++ b/src/common/util.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -163,9 +163,9 @@ int64_t clamp_double_to_int64(double number);
 void simplify_fraction64(uint64_t *numer, uint64_t *denom);
 
 /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
- * and positive <b>b</b>.  Works on integer types only. Not defined if a+b can
- * overflow. */
-#define CEIL_DIV(a,b) (((a)+(b)-1)/(b))
+ * and positive <b>b</b>.  Works on integer types only. Not defined if a+(b-1)
+ * can overflow. */
+#define CEIL_DIV(a,b) (((a)+((b)-1))/(b))
 
 /* Return <b>v</b> if it's between <b>min</b> and <b>max</b>.  Otherwise
  * return <b>min</b> if <b>v</b> is smaller than <b>min</b>, or <b>max</b> if
@@ -186,6 +186,7 @@ void tor_strlower(char *s) ATTR_NONNULL((1));
 void tor_strupper(char *s) ATTR_NONNULL((1));
 int tor_strisprint(const char *s) ATTR_NONNULL((1));
 int tor_strisnonupper(const char *s) ATTR_NONNULL((1));
+int tor_strisspace(const char *s);
 int strcmp_opt(const char *s1, const char *s2);
 int strcmpstart(const char *s1, const char *s2) ATTR_NONNULL((1,2));
 int strcmp_len(const char *s1, const char *s2, size_t len) ATTR_NONNULL((1,2));
@@ -239,6 +240,7 @@ void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern, ...)
 void smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern,
                              va_list args)
   CHECK_PRINTF(2, 0);
+void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
 
 /* Time helpers */
 long tv_udiff(const struct timeval *start, const struct timeval *end);
@@ -254,8 +256,9 @@ void format_local_iso_time(char *buf, time_t t);
 void format_iso_time(char *buf, time_t t);
 void format_iso_time_nospace(char *buf, time_t t);
 void format_iso_time_nospace_usec(char *buf, const struct timeval *tv);
-int parse_iso_time_(const char *cp, time_t *t, int strict);
+int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace);
 int parse_iso_time(const char *buf, time_t *t);
+int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
@@ -319,7 +322,7 @@ enum stream_status {
 
 const char *stream_status_to_string(enum stream_status stream_status);
 
-enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count);
+enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
 
 MOCK_DECL(int,tor_unlink,(const char *pathname));
 
@@ -386,9 +389,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
                                  size_t *sz_out)
   ATTR_MALLOC;
 const char *unescape_string(const char *s, char **result, size_t *size_out);
-const char *parse_config_line_from_str_verbose(const char *line,
-                                       char **key_out, char **value_out,
-                                       const char **err_out);
+char *get_unquoted_path(const char *path);
 char *expand_filename(const char *filename);
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 int path_is_relative(const char *filename);
@@ -460,9 +461,6 @@ struct process_handle_t {
   int stdin_pipe;
   int stdout_pipe;
   int stderr_pipe;
-  FILE *stdin_handle;
-  FILE *stdout_handle;
-  FILE *stderr_handle;
   pid_t pid;
   /** If the process has not given us a SIGCHLD yet, this has the
    * waitpid_callback_t that gets invoked once it has. Otherwise this
@@ -485,7 +483,7 @@ int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
 ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
                             const process_handle_t *process);
 #else
-ssize_t tor_read_all_handle(FILE *h, char *buf, size_t count,
+ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
                             const process_handle_t *process,
                             int *eof);
 #endif
@@ -499,7 +497,7 @@ int tor_process_get_pid(process_handle_t *process_handle);
 #ifdef _WIN32
 HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
 #else
-FILE *tor_process_get_stdout_pipe(process_handle_t *process_handle);
+int tor_process_get_stdout_pipe(process_handle_t *process_handle);
 #endif
 
 #ifdef _WIN32
@@ -508,7 +506,7 @@ tor_get_lines_from_handle,(HANDLE *handle,
                            enum stream_status *stream_status));
 #else
 MOCK_DECL(struct smartlist_t *,
-tor_get_lines_from_handle,(FILE *handle,
+tor_get_lines_from_handle,(int fd,
                            enum stream_status *stream_status));
 #endif
 
@@ -551,9 +549,7 @@ STATIC int format_helper_exit_status(unsigned char child_state,
 
 #endif
 
-#ifdef TOR_UNIT_TESTS
-int size_mul_check__(const size_t x, const size_t y);
-#endif
+int size_mul_check(const size_t x, const size_t y);
 
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
diff --git a/src/common/util_bug.c b/src/common/util_bug.c
index 08aba47974..3d990e3700 100644
--- a/src/common/util_bug.c
+++ b/src/common/util_bug.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,7 +44,7 @@ static void
 add_captured_bug(const char *s)
 {
   --n_bugs_to_capture;
-  smartlist_add(bug_messages, tor_strdup(s));
+  smartlist_add_strdup(bug_messages, s);
 }
 /** Set a callback to be invoked when we get any tor_bug_occurred_
  * invocation. We use this in the unit tests so that a nonfatal
diff --git a/src/common/util_bug.h b/src/common/util_bug.h
index 0695806911..ae7e7a37fd 100644
--- a/src/common/util_bug.h
+++ b/src/common/util_bug.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/util_format.c b/src/common/util_format.c
index aef9db85c8..1f7b8b03aa 100644
--- a/src/common/util_format.c
+++ b/src/common/util_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,13 +22,16 @@
 #include <stdlib.h>
 
 /* Return the base32 encoded size in bytes using the source length srclen.
- * The NUL terminated byte is added as well since every base32 encoding
- * requires enough space for it. */
+ *
+ * (WATCH OUT: This API counts the terminating NUL byte, but
+ * base64_encode_size does not.)
+ */
 size_t
 base32_encoded_size(size_t srclen)
 {
   size_t enclen;
-  enclen = CEIL_DIV(srclen*8, 5) + 1;
+  tor_assert(srclen < SIZE_T_CEILING / 8);
+  enclen = BASE32_NOPAD_BUFSIZE(srclen);
   tor_assert(enclen < INT_MAX && enclen > srclen);
   return enclen;
 }
@@ -41,7 +44,6 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   size_t nbits = srclen * 8;
   size_t bit;
 
-  tor_assert(srclen < SIZE_T_CEILING/8);
   /* We need enough space for the encoded data and the extra NUL byte. */
   tor_assert(base32_encoded_size(srclen) <= destlen);
   tor_assert(destlen < SIZE_T_CEILING);
@@ -51,9 +53,10 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 
   for (i=0,bit=0; bit < nbits; ++i, bit+=5) {
     /* set v to the 16-bit value starting at src[bits/8], 0-padded. */
-    v = ((uint8_t)src[bit/8]) << 8;
-    if (bit+5<nbits)
-      v += (uint8_t)src[(bit/8)+1];
+    size_t idx = bit / 8;
+    v = ((uint8_t)src[idx]) << 8;
+    if (idx+1 < srclen)
+      v += (uint8_t)src[idx+1];
     /* set u to the 5-bit value at the bit'th bit of buf. */
     u = (v >> (11-(bit%8))) & 0x1F;
     dest[i] = BASE32_CHARS[u];
@@ -133,6 +136,9 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
 /** Return the Base64 encoded size of <b>srclen</b> bytes of data in
  * bytes.
  *
+ * (WATCH OUT: This API <em>does not</em> count the terminating NUL byte,
+ * but base32_encoded_size does.)
+ *
  * If <b>flags</b>&amp;BASE64_ENCODE_MULTILINE is true, return the size
  * of the encoded output as multiline output (64 character, `\n' terminated
  * lines).
@@ -141,19 +147,16 @@ size_t
 base64_encode_size(size_t srclen, int flags)
 {
   size_t enclen;
+
+  /* Use INT_MAX for overflow checking because base64_encode() returns int. */
   tor_assert(srclen < INT_MAX);
+  tor_assert(CEIL_DIV(srclen, 3) < INT_MAX / 4);
 
-  if (srclen == 0)
-    return 0;
+  enclen = BASE64_LEN(srclen);
+  if (flags & BASE64_ENCODE_MULTILINE)
+    enclen += CEIL_DIV(enclen, BASE64_OPENSSL_LINELEN);
 
-  enclen = ((srclen - 1) / 3) * 4 + 4;
-  if (flags & BASE64_ENCODE_MULTILINE) {
-    size_t remainder = enclen % BASE64_OPENSSL_LINELEN;
-    enclen += enclen / BASE64_OPENSSL_LINELEN;
-    if (remainder)
-      enclen++;
-  }
-  tor_assert(enclen < INT_MAX && enclen > srclen);
+  tor_assert(enclen < INT_MAX && (enclen == 0 || enclen > srclen));
   return enclen;
 }
 
@@ -310,39 +313,6 @@ base64_encode_nopad(char *dest, size_t destlen,
   return (int)(out - dest);
 }
 
-/** As base64_decode, but do not require any padding on the input */
-int
-base64_decode_nopad(uint8_t *dest, size_t destlen,
-                    const char *src, size_t srclen)
-{
-  if (srclen > SIZE_T_CEILING - 4)
-    return -1;
-  char *buf = tor_malloc(srclen + 4);
-  memcpy(buf, src, srclen+1);
-  size_t buflen;
-  switch (srclen % 4)
-    {
-    case 0:
-    default:
-      buflen = srclen;
-      break;
-    case 1:
-      tor_free(buf);
-      return -1;
-    case 2:
-      memcpy(buf+srclen, "==", 3);
-      buflen = srclen + 2;
-      break;
-    case 3:
-      memcpy(buf+srclen, "=", 2);
-      buflen = srclen + 1;
-      break;
-  }
-  int n = base64_decode((char*)dest, destlen, buf, buflen);
-  tor_free(buf);
-  return n;
-}
-
 #undef BASE64_OPENSSL_LINELEN
 
 /** @{ */
@@ -392,15 +362,9 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   const char *eos = src+srclen;
   uint32_t n=0;
   int n_idx=0;
-  char *dest_orig = dest;
+  size_t di = 0;
 
-  /* Max number of bits == srclen*6.
-   * Number of bytes required to hold all bits == (srclen*6)/8.
-   * Yes, we want to round down: anything that hangs over the end of a
-   * byte is padding. */
-  if (destlen < (srclen*3)/4)
-    return -1;
-  if (destlen > SIZE_T_CEILING)
+  if (destlen > INT_MAX)
     return -1;
 
   /* Make sure we leave no uninitialized data in the destination buffer. */
@@ -428,9 +392,11 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
         n = (n<<6) | v;
         if ((++n_idx) == 4) {
           /* We've accumulated 24 bits in n. Flush them. */
-          *dest++ = (n>>16);
-          *dest++ = (n>>8) & 0xff;
-          *dest++ = (n) & 0xff;
+          if (destlen < 3 || di > destlen - 3)
+            return -1;
+          dest[di++] = (n>>16);
+          dest[di++] = (n>>8) & 0xff;
+          dest[di++] = (n) & 0xff;
           n_idx = 0;
           n = 0;
         }
@@ -448,18 +414,21 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen)
       return -1;
     case 2:
       /* 12 leftover bits: The last 4 are padding and the first 8 are data. */
-      *dest++ = n >> 4;
+      if (destlen < 1 || di > destlen - 1)
+        return -1;
+      dest[di++] = n >> 4;
       break;
     case 3:
       /* 18 leftover bits: The last 2 are padding and the first 16 are data. */
-      *dest++ = n >> 10;
-      *dest++ = n >> 2;
+      if (destlen < 2 || di > destlen - 2)
+        return -1;
+      dest[di++] = n >> 10;
+      dest[di++] = n >> 2;
   }
 
-  tor_assert((dest-dest_orig) <= (ssize_t)destlen);
-  tor_assert((dest-dest_orig) <= INT_MAX);
+  tor_assert(di <= destlen);
 
-  return (int)(dest-dest_orig);
+  return (int)di;
 }
 #undef X
 #undef SP
@@ -475,7 +444,8 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
   const char *end;
   char *cp;
 
-  tor_assert(destlen >= srclen*2+1);
+  tor_assert(srclen < SIZE_T_CEILING / 2 - 1);
+  tor_assert(destlen >= BASE16_BUFSIZE(srclen));
   tor_assert(destlen < SIZE_T_CEILING);
 
   /* Make sure we leave no uninitialized data in the destination buffer. */
diff --git a/src/common/util_format.h b/src/common/util_format.h
index 20ac711d10..4af8832bbe 100644
--- a/src/common/util_format.h
+++ b/src/common/util_format.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_UTIL_FORMAT_H
@@ -10,6 +10,26 @@
 #include "testsupport.h"
 #include "torint.h"
 
+/** @{ */
+/** These macros don't check for overflow.  Use them only for constant inputs
+ * (like array declarations).  The *_LEN macros are the raw encoding lengths
+ * (without terminating NUL), while the *_BUFSIZE macros count the terminating
+ * NUL. */
+#define BASE64_LEN(n) (CEIL_DIV((n), 3) * 4)
+#define BASE32_LEN(n) (CEIL_DIV((n), 5) * 8)
+#define BASE16_LEN(n) ((n) * 2)
+
+#define BASE64_BUFSIZE(n) (BASE64_LEN(n) + 1)
+#define BASE32_BUFSIZE(n) (BASE32_LEN(n) + 1)
+#define BASE16_BUFSIZE(n) (BASE16_LEN(n) + 1)
+
+#define BASE64_NOPAD_LEN(n) (CEIL_DIV((n) * 4, 3))
+#define BASE32_NOPAD_LEN(n) (CEIL_DIV((n) * 8, 5))
+
+#define BASE64_NOPAD_BUFSIZE(n) (BASE64_NOPAD_LEN(n) + 1)
+#define BASE32_NOPAD_BUFSIZE(n) (BASE32_NOPAD_LEN(n) + 1)
+/** @} */
+
 #define BASE64_ENCODE_MULTILINE 1
 size_t base64_encode_size(size_t srclen, int flags);
 int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
@@ -17,8 +37,6 @@ int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
 int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base64_encode_nopad(char *dest, size_t destlen,
                         const uint8_t *src, size_t srclen);
-int base64_decode_nopad(uint8_t *dest, size_t destlen,
-                        const char *src, size_t srclen);
 
 /** Characters that can appear (case-insensitively) in a base32 encoding. */
 #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567"
diff --git a/src/common/util_process.c b/src/common/util_process.c
index abda63720c..9e9679b099 100644
--- a/src/common/util_process.c
+++ b/src/common/util_process.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/util_process.h b/src/common/util_process.h
index d38301a354..c3a63498b5 100644
--- a/src/common/util_process.h
+++ b/src/common/util_process.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/common/workqueue.c b/src/common/workqueue.c
index e1fb663a2a..42723224d3 100644
--- a/src/common/workqueue.c
+++ b/src/common/workqueue.c
@@ -25,11 +25,19 @@
 #include "orconfig.h"
 #include "compat.h"
 #include "compat_threads.h"
+#include "crypto.h"
 #include "util.h"
 #include "workqueue.h"
 #include "tor_queue.h"
 #include "torlog.h"
 
+#define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH
+#define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW
+#define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1)
+
+TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s);
+typedef struct work_tailq_t work_tailq_t;
+
 struct threadpool_s {
   /** An array of pointers to workerthread_t: one for each running worker
    * thread. */
@@ -38,8 +46,12 @@ struct threadpool_s {
   /** Condition variable that we wait on when we have no work, and which
    * gets signaled when our queue becomes nonempty. */
   tor_cond_t condition;
-  /** Queue of pending work that we have to do. */
-  TOR_TAILQ_HEAD(, workqueue_entry_s) work;
+  /** Queues of pending work that we have to do. The queue with priority
+   * <b>p</b> is work[p]. */
+  work_tailq_t work[WORKQUEUE_N_PRIORITIES];
+
+  /** Weak RNG, used to decide when to ignore priority. */
+  tor_weak_rng_t weak_rng;
 
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
@@ -66,6 +78,11 @@ struct threadpool_s {
   void *new_thread_state_arg;
 };
 
+/** Used to put a workqueue_priority_t value into a bitfield. */
+#define workqueue_priority_bitfield_t ENUM_BF(workqueue_priority_t)
+/** Number of bits needed to hold all legal values of workqueue_priority_t */
+#define WORKQUEUE_PRIORITY_BITS 2
+
 struct workqueue_entry_s {
   /** The next workqueue_entry_t that's pending on the same thread or
    * reply queue. */
@@ -76,6 +93,8 @@ struct workqueue_entry_s {
   struct threadpool_s *on_pool;
   /** True iff this entry is waiting for a worker to start processing it. */
   uint8_t pending;
+  /** Priority of this entry. */
+  workqueue_priority_bitfield_t priority : WORKQUEUE_PRIORITY_BITS;
   /** Function to run in the worker thread. */
   workqueue_reply_t (*fn)(void *state, void *arg);
   /** Function to run while processing the reply queue. */
@@ -94,9 +113,7 @@ struct replyqueue_s {
   alert_sockets_t alert;
 };
 
-/** A worker thread represents a single thread in a thread pool.  To avoid
- * contention, each gets its own queue. This breaks the guarantee that that
- * queued work will get executed strictly in order. */
+/** A worker thread represents a single thread in a thread pool. */
 typedef struct workerthread_s {
   /** Which thread it this?  In range 0..in_pool->n_threads-1 */
   int index;
@@ -109,6 +126,8 @@ typedef struct workerthread_s {
   replyqueue_t *reply_queue;
   /** The current update generation of this thread */
   unsigned generation;
+  /** One over the probability of taking work from a lower-priority queue. */
+  int32_t lower_priority_chance;
 } workerthread_t;
 
 static void queue_reply(replyqueue_t *queue, workqueue_entry_t *work);
@@ -125,6 +144,7 @@ workqueue_entry_new(workqueue_reply_t (*fn)(void*, void*),
   ent->fn = fn;
   ent->reply_fn = reply_fn;
   ent->arg = arg;
+  ent->priority = WQ_PRI_HIGH;
   return ent;
 }
 
@@ -161,8 +181,9 @@ workqueue_entry_cancel(workqueue_entry_t *ent)
   int cancelled = 0;
   void *result = NULL;
   tor_mutex_acquire(&ent->on_pool->lock);
+  workqueue_priority_t prio = ent->priority;
   if (ent->pending) {
-    TOR_TAILQ_REMOVE(&ent->on_pool->work, ent, next_work);
+    TOR_TAILQ_REMOVE(&ent->on_pool->work[prio], ent, next_work);
     cancelled = 1;
     result = ent->arg;
   }
@@ -180,8 +201,46 @@ workqueue_entry_cancel(workqueue_entry_t *ent)
 static int
 worker_thread_has_work(workerthread_t *thread)
 {
-  return !TOR_TAILQ_EMPTY(&thread->in_pool->work) ||
-    thread->generation != thread->in_pool->generation;
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    if (!TOR_TAILQ_EMPTY(&thread->in_pool->work[i]))
+        return 1;
+  }
+  return thread->generation != thread->in_pool->generation;
+}
+
+/** Extract the next workqueue_entry_t from the the thread's pool, removing
+ * it from the relevant queues and marking it as non-pending.
+ *
+ * The caller must hold the lock. */
+static workqueue_entry_t *
+worker_thread_extract_next_work(workerthread_t *thread)
+{
+  threadpool_t *pool = thread->in_pool;
+  work_tailq_t *queue = NULL, *this_queue;
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    this_queue = &pool->work[i];
+    if (!TOR_TAILQ_EMPTY(this_queue)) {
+      queue = this_queue;
+      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+                                     thread->lower_priority_chance)) {
+        /* Usually we'll just break now, so that we can get out of the loop
+         * and use the queue where we found work. But with a small
+         * probability, we'll keep looking for lower priority work, so that
+         * we don't ignore our low-priority queues entirely. */
+        break;
+      }
+    }
+  }
+
+  if (queue == NULL)
+    return NULL;
+
+  workqueue_entry_t *work = TOR_TAILQ_FIRST(queue);
+  TOR_TAILQ_REMOVE(queue, work, next_work);
+  work->pending = 0;
+  return work;
 }
 
 /**
@@ -217,9 +276,9 @@ worker_thread_main(void *thread_)
         tor_mutex_acquire(&pool->lock);
         continue;
       }
-      work = TOR_TAILQ_FIRST(&pool->work);
-      TOR_TAILQ_REMOVE(&pool->work, work, next_work);
-      work->pending = 0;
+      work = worker_thread_extract_next_work(thread);
+      if (BUG(work == NULL))
+        break;
       tor_mutex_release(&pool->lock);
 
       /* We run the work function without holding the thread lock. This
@@ -268,12 +327,14 @@ queue_reply(replyqueue_t *queue, workqueue_entry_t *work)
 /** Allocate and start a new worker thread to use state object <b>state</b>,
  * and send responses to <b>replyqueue</b>. */
 static workerthread_t *
-workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue)
+workerthread_new(int32_t lower_priority_chance,
+                 void *state, threadpool_t *pool, replyqueue_t *replyqueue)
 {
   workerthread_t *thr = tor_malloc_zero(sizeof(workerthread_t));
   thr->state = state;
   thr->reply_queue = replyqueue;
   thr->in_pool = pool;
+  thr->lower_priority_chance = lower_priority_chance;
 
   if (spawn_func(worker_thread_main, thr) < 0) {
     //LCOV_EXCL_START
@@ -299,24 +360,34 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue)
  * function's responsibility to free the work object.
  *
  * On success, return a workqueue_entry_t object that can be passed to
- * workqueue_entry_cancel(). On failure, return NULL.
+ * workqueue_entry_cancel(). On failure, return NULL.  (Failure is not
+ * currently possible, but callers should check anyway.)
+ *
+ * Items are executed in a loose priority order -- each thread will usually
+ * take from the queued work with the highest prioirity, but will occasionally
+ * visit lower-priority queues to keep them from starving completely.
  *
- * Note that because each thread has its own work queue, work items may not
+ * Note that because of priorities and thread behavior, work items may not
  * be executed strictly in order.
  */
 workqueue_entry_t *
-threadpool_queue_work(threadpool_t *pool,
-                      workqueue_reply_t (*fn)(void *, void *),
-                      void (*reply_fn)(void *),
-                      void *arg)
+threadpool_queue_work_priority(threadpool_t *pool,
+                               workqueue_priority_t prio,
+                               workqueue_reply_t (*fn)(void *, void *),
+                               void (*reply_fn)(void *),
+                               void *arg)
 {
+  tor_assert(((int)prio) >= WORKQUEUE_PRIORITY_FIRST &&
+             ((int)prio) <= WORKQUEUE_PRIORITY_LAST);
+
   workqueue_entry_t *ent = workqueue_entry_new(fn, reply_fn, arg);
   ent->on_pool = pool;
   ent->pending = 1;
+  ent->priority = prio;
 
   tor_mutex_acquire(&pool->lock);
 
-  TOR_TAILQ_INSERT_TAIL(&pool->work, ent, next_work);
+  TOR_TAILQ_INSERT_TAIL(&pool->work[prio], ent, next_work);
 
   tor_cond_signal_one(&pool->condition);
 
@@ -325,6 +396,16 @@ threadpool_queue_work(threadpool_t *pool,
   return ent;
 }
 
+/** As threadpool_queue_work_priority(), but assumes WQ_PRI_HIGH */
+workqueue_entry_t *
+threadpool_queue_work(threadpool_t *pool,
+                      workqueue_reply_t (*fn)(void *, void *),
+                      void (*reply_fn)(void *),
+                      void *arg)
+{
+  return threadpool_queue_work_priority(pool, WQ_PRI_HIGH, fn, reply_fn, arg);
+}
+
 /**
  * Queue a copy of a work item for every thread in a pool.  This can be used,
  * for example, to tell the threads to update some parameter in their states.
@@ -388,6 +469,14 @@ threadpool_queue_update(threadpool_t *pool,
 /** Don't have more than this many threads per pool. */
 #define MAX_THREADS 1024
 
+/** For half of our threads, choose lower priority queues with probability
+ * 1/N for each of these values. Both are chosen somewhat arbitrarily.  If
+ * CHANCE_PERMISSIVE is too low, then we have a risk of low-priority tasks
+ * stalling forever.  If it's too high, we have a risk of low-priority tasks
+ * grabbing half of the threads. */
+#define CHANCE_PERMISSIVE 37
+#define CHANCE_STRICT INT32_MAX
+
 /** Launch threads until we have <b>n</b>. */
 static int
 threadpool_start_threads(threadpool_t *pool, int n)
@@ -404,8 +493,14 @@ threadpool_start_threads(threadpool_t *pool, int n)
                                      sizeof(workerthread_t*), n);
 
   while (pool->n_threads < n) {
+    /* For half of our threads, we'll choose lower priorities permissively;
+     * for the other half, we'll stick more strictly to higher priorities.
+     * This keeps slow low-priority tasks from taking over completely. */
+    int32_t chance = (pool->n_threads & 1) ? CHANCE_STRICT : CHANCE_PERMISSIVE;
+
     void *state = pool->new_thread_state_fn(pool->new_thread_state_arg);
-    workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue);
+    workerthread_t *thr = workerthread_new(chance,
+                                           state, pool, pool->reply_queue);
 
     if (!thr) {
       //LCOV_EXCL_START
@@ -441,7 +536,15 @@ threadpool_new(int n_threads,
   pool = tor_malloc_zero(sizeof(threadpool_t));
   tor_mutex_init_nonrecursive(&pool->lock);
   tor_cond_init(&pool->condition);
-  TOR_TAILQ_INIT(&pool->work);
+  unsigned i;
+  for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
+    TOR_TAILQ_INIT(&pool->work[i]);
+  }
+  {
+    unsigned seed;
+    crypto_rand((void*)&seed, sizeof(seed));
+    tor_init_weak_random(&pool->weak_rng, seed);
+  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -510,12 +613,13 @@ replyqueue_get_socket(replyqueue_t *rq)
 void
 replyqueue_process(replyqueue_t *queue)
 {
-  if (queue->alert.drain_fn(queue->alert.read_fd) < 0) {
+  int r = queue->alert.drain_fn(queue->alert.read_fd);
+  if (r < 0) {
     //LCOV_EXCL_START
     static ratelim_t warn_limit = RATELIM_INIT(7200);
     log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL,
                  "Failure from drain_fd: %s",
-                 tor_socket_strerror(tor_socket_errno(queue->alert.read_fd)));
+                   tor_socket_strerror(-r));
     //LCOV_EXCL_STOP
   }
 
diff --git a/src/common/workqueue.h b/src/common/workqueue.h
index 54276767b0..d2508f5329 100644
--- a/src/common/workqueue.h
+++ b/src/common/workqueue.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_WORKQUEUE_H
@@ -16,12 +16,26 @@ typedef struct threadpool_s threadpool_t;
 typedef struct workqueue_entry_s workqueue_entry_t;
 
 /** Possible return value from a work function: */
-typedef enum {
+typedef enum workqueue_reply_t {
   WQ_RPL_REPLY = 0, /** indicates success */
   WQ_RPL_ERROR = 1, /** indicates fatal error */
   WQ_RPL_SHUTDOWN = 2, /** indicates thread is shutting down */
 } workqueue_reply_t;
 
+/** Possible priorities for work.  Lower numeric values are more important. */
+typedef enum workqueue_priority_t {
+  WQ_PRI_HIGH = 0,
+  WQ_PRI_MED  = 1,
+  WQ_PRI_LOW  = 2,
+} workqueue_priority_t;
+
+workqueue_entry_t *threadpool_queue_work_priority(threadpool_t *pool,
+                                    workqueue_priority_t prio,
+                                    workqueue_reply_t (*fn)(void *,
+                                                            void *),
+                                    void (*reply_fn)(void *),
+                                    void *arg);
+
 workqueue_entry_t *threadpool_queue_work(threadpool_t *pool,
                                          workqueue_reply_t (*fn)(void *,
                                                                  void *),
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index 5328206da9..8f3597f3f6 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -95,7 +95,12 @@
 
 ## If you have multiple network interfaces, you can specify one for
 ## outgoing traffic to use.
-# OutboundBindAddress 10.0.0.5
+## OutboundBindAddressExit will be used for all exit traffic, while
+## OutboundBindAddressOR will be used for all other connections.
+## If you do not wish to differentiate, use OutboundBindAddress to
+## specify the same address for both in a single line.
+#OutboundBindAddressExit 10.0.0.4
+#OutboundBindAddressOR 10.0.0.5
 
 ## A handle for your relay, so people don't have to refer to it by key.
 ## Nicknames must be between 1 and 19 characters inclusive, and must
@@ -204,3 +209,12 @@
 ## address manually to your friends, uncomment this line:
 #PublishServerDescriptor 0
 
+## Configuration options can be imported from files or folders using the %include
+## option with the value being a path. If the path is a file, the options from the
+## file will be parsed as if they were written where the %include option is. If
+## the path is a folder, all files on that folder will be parsed following lexical
+## order. Files starting with a dot are ignored. Files on subfolders are ignored.
+## The %include option can be used recursively.
+#%include /etc/torrc.d/
+#%include /etc/torrc.custom
+
diff --git a/src/ext/byteorder.h b/src/ext/byteorder.h
new file mode 100644
index 0000000000..c8ba52184b
--- /dev/null
+++ b/src/ext/byteorder.h
@@ -0,0 +1,67 @@
+/* <MIT License>
+ Copyright (c) 2013-2014  Marek Majkowski <marek@popcount.org>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ </MIT License>
+
+ Original location:
+    https://github.com/majek/csiphash/
+
+ Solution inspired by code from:
+    Samuel Neves (supercop/crypto_auth/siphash24/little)
+    djb (supercop/crypto_auth/siphash24/little2)
+    Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c)
+*/
+
+/* This code is extracted from csiphash.h */
+
+#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) &&      \
+	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#  define _le64toh(x) ((uint64_t)(x))
+#elif defined(_WIN32)
+/* Windows is always little endian, unless you're on xbox360
+   http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
+#  define _le64toh(x) ((uint64_t)(x))
+#elif defined(__APPLE__)
+#  include <libkern/OSByteOrder.h>
+#  define _le64toh(x) OSSwapLittleToHostInt64(x)
+#elif defined(sun) || defined(__sun)
+#  include <sys/byteorder.h>
+#  define _le64toh(x) LE_64(x)
+
+#else
+
+/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
+#  if defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD)
+#    include <sys/endian.h>
+#  else
+#    include <endian.h>
+#  endif
+#  if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
+	__BYTE_ORDER == __LITTLE_ENDIAN
+#    define _le64toh(x) ((uint64_t)(x))
+#  else
+#    if defined(OpenBSD)
+#      define _le64toh(x) letoh64(x)
+#    else
+#      define _le64toh(x) le64toh(x)
+#    endif
+#  endif
+
+#endif
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index 49a6dc4778..508e4f6ceb 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -35,41 +35,7 @@
 #include "util.h"
 /* for memcpy */
 #include <string.h>
-
-#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
-	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#  define _le64toh(x) ((uint64_t)(x))
-#elif defined(_WIN32)
-/* Windows is always little endian, unless you're on xbox360
-   http://msdn.microsoft.com/en-us/library/b0084kay(v=vs.80).aspx */
-#  define _le64toh(x) ((uint64_t)(x))
-#elif defined(__APPLE__)
-#  include <libkern/OSByteOrder.h>
-#  define _le64toh(x) OSSwapLittleToHostInt64(x)
-#elif defined(sun) || defined(__sun)
-#  include <sys/byteorder.h>
-#  define _le64toh(x) LE_64(x)
-
-#else
-
-/* See: http://sourceforge.net/p/predef/wiki/Endianness/ */
-#  if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
-#    include <sys/endian.h>
-#  else
-#    include <endian.h>
-#  endif
-#  if defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \
-	__BYTE_ORDER == __LITTLE_ENDIAN
-#    define _le64toh(x) ((uint64_t)(x))
-#  else
-#    if defined(__OpenBSD__)
-#      define _le64toh(x) letoh64(x)
-#    else
-#      define _le64toh(x) le64toh(x)
-#    endif
-#  endif
-
-#endif
+#include "byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
diff --git a/src/ext/ed25519/donna/ed25519-hash-custom.h b/src/ext/ed25519/donna/ed25519-hash-custom.h
index 7dc249129d..609451abd5 100644
--- a/src/ext/ed25519/donna/ed25519-hash-custom.h
+++ b/src/ext/ed25519/donna/ed25519-hash-custom.h
@@ -9,3 +9,34 @@
 	void ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen);
 */
 
+#include "crypto.h"
+
+typedef struct ed25519_hash_context {
+  crypto_digest_t *ctx;
+} ed25519_hash_context;
+
+
+static void
+ed25519_hash_init(ed25519_hash_context *ctx)
+{
+  ctx->ctx = crypto_digest512_new(DIGEST_SHA512);
+}
+static void
+ed25519_hash_update(ed25519_hash_context *ctx, const uint8_t *in, size_t inlen)
+{
+  crypto_digest_add_bytes(ctx->ctx, (const char *)in, inlen);
+}
+static void
+ed25519_hash_final(ed25519_hash_context *ctx, uint8_t *hash)
+{
+  crypto_digest_get_digest(ctx->ctx, (char *)hash, DIGEST512_LEN);
+  crypto_digest_free(ctx->ctx);
+  ctx->ctx = NULL;
+}
+static void
+ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen)
+{
+  crypto_digest512((char *)hash, (const char *)in, inlen,
+		   DIGEST_SHA512);
+}
+
diff --git a/src/ext/ed25519/ref10/crypto_hash_sha512.h b/src/ext/ed25519/ref10/crypto_hash_sha512.h
index 0278571522..5dad935c79 100644
--- a/src/ext/ed25519/ref10/crypto_hash_sha512.h
+++ b/src/ext/ed25519/ref10/crypto_hash_sha512.h
@@ -1,30 +1,32 @@
 /* Added for Tor. */
-#include <openssl/sha.h>
+#include "crypto.h"
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len'-byte string in 'inp' */
 #define crypto_hash_sha512(out, inp, len) \
-  SHA512((inp), (len), (out))
+  crypto_digest512((char *)(out), (const char *)(inp), (len), DIGEST_SHA512)
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len1'-byte string in 'inp1',
  * concatenated with the 'len2'-byte string in 'inp2'. */
 #define crypto_hash_sha512_2(out, inp1, len1, inp2, len2)               \
   do {                                                                  \
-      SHA512_CTX sha_ctx_;                                              \
-      SHA512_Init(&sha_ctx_);                                           \
-      SHA512_Update(&sha_ctx_, (inp1), (len1));                         \
-      SHA512_Update(&sha_ctx_, (inp2), (len2));                         \
-      SHA512_Final((out), &sha_ctx_);                                   \
- } while(0)
+    crypto_digest_t *sha_ctx_;						\
+    sha_ctx_ = crypto_digest512_new(DIGEST_SHA512);			\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp1), (len1));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp2), (len2));	\
+    crypto_digest_get_digest(sha_ctx_, (char *)out, DIGEST512_LEN);	\
+    crypto_digest_free(sha_ctx_);					\
+  } while (0)
 
 /* Set 'out' to the 512-bit SHA512 hash of the 'len1'-byte string in 'inp1',
  * concatenated with the 'len2'-byte string in 'inp2', concatenated with
  * the 'len3'-byte string in 'len3'. */
 #define crypto_hash_sha512_3(out, inp1, len1, inp2, len2, inp3, len3)   \
   do {                                                                  \
-      SHA512_CTX sha_ctx_;                                              \
-      SHA512_Init(&sha_ctx_);                                           \
-      SHA512_Update(&sha_ctx_, (inp1), (len1));                         \
-      SHA512_Update(&sha_ctx_, (inp2), (len2));                         \
-      SHA512_Update(&sha_ctx_, (inp3), (len3));                         \
-      SHA512_Final((out), &sha_ctx_);                                   \
+    crypto_digest_t *sha_ctx_;						\
+    sha_ctx_ = crypto_digest512_new(DIGEST_SHA512);			\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp1), (len1));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp2), (len2));	\
+    crypto_digest_add_bytes(sha_ctx_, (const char *)(inp3), (len3));	\
+    crypto_digest_get_digest(sha_ctx_, (char *)out, DIGEST512_LEN);	\
+    crypto_digest_free(sha_ctx_);					\
  } while(0)
diff --git a/src/ext/ht.h b/src/ext/ht.h
index a441d0b685..caa420e9b5 100644
--- a/src/ext/ht.h
+++ b/src/ext/ht.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2002, Christopher Clark.
  * Copyright (c) 2005-2006, Nick Mathewson.
- * Copyright (c) 2007-2015, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See license at end. */
 
 /* Based on ideas by Christopher Clark and interfaces from Niels Provos. */
diff --git a/src/ext/include.am b/src/ext/include.am
index f00f3e031e..7ec2e43312 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -5,6 +5,7 @@ EXTRA_DIST += src/ext/README
 
 EXTHEADERS = \
   src/ext/ht.h		\
+  src/ext/byteorder.h   \
   src/ext/tinytest.h	\
   src/ext/tor_readpassphrase.h \
   src/ext/strlcat.c	\
@@ -100,6 +101,7 @@ noinst_LIBRARIES += $(LIBED25519_REF10)
 src_ext_ed25519_donna_libed25519_donna_a_CFLAGS=\
   @CFLAGS_CONSTTIME@ \
   -DED25519_CUSTOMRANDOM \
+  -DED25519_CUSTOMHASH \
   -DED25519_SUFFIX=_donna
 
 src_ext_ed25519_donna_libed25519_donna_a_SOURCES= \
diff --git a/src/ext/keccak-tiny/keccak-tiny-unrolled.c b/src/ext/keccak-tiny/keccak-tiny-unrolled.c
index d1342c3601..d8d7fe335a 100644
--- a/src/ext/keccak-tiny/keccak-tiny-unrolled.c
+++ b/src/ext/keccak-tiny/keccak-tiny-unrolled.c
@@ -10,28 +10,21 @@
 
 #include <string.h>
 #include "crypto.h"
+#include "byteorder.h"
 
 /******** Endianness conversion helpers ********/
 
 static inline uint64_t
 loadu64le(const unsigned char *x) {
-  uint64_t r = 0;
-  size_t i;
-
-  for (i = 0; i < 8; ++i) {
-    r |= (uint64_t)x[i] << 8 * i;
-  }
-  return r;
+  uint64_t r;
+  memcpy(&r, x, sizeof(r));
+  return _le64toh(r);
 }
 
 static inline void
 storeu64le(uint8_t *x, uint64_t u) {
-  size_t i;
-
-  for(i=0; i<8; ++i) {
-    x[i] = u;
-    u >>= 8;
-  }
+  uint64_t val = _le64toh(u);
+  memcpy(x, &val, sizeof(u));
 }
 
 /******** The Keccak-f[1600] permutation ********/
diff --git a/src/ext/rust b/src/ext/rust
new file mode 160000
+Subproject 240296800824e40b10cb8c16da0e71156335394
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 3ffde6e09b..85c847b3fd 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,11 +1,11 @@
-/* trunnel-impl.h -- copied from Trunnel v1.4.6
+/* trunnel-impl.h -- copied from Trunnel v1.5.1
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel-impl.h -- Implementation helpers for trunnel, included by
  * generated trunnel files
  *
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2017, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  */
 
@@ -17,6 +17,7 @@
 #include "trunnel.h"
 #include <assert.h>
 #include <string.h>
+#include <stdlib.h>
 
 #if defined(_MSC_VER) && (_MSC_VER < 1600)
 #define uint8_t unsigned char
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3994422643..6a42417241 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,10 +1,10 @@
-/* trunnel.c -- copied from Trunnel v1.4.6
+/* trunnel.c -- copied from Trunnel v1.5.1
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel.c -- Helper functions to implement trunnel.
  *
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2017, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  *
  * See trunnel-impl.h for documentation of these functions.
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 41068b8fb3..dd78553c77 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,11 +1,11 @@
-/* trunnel.h -- copied from Trunnel v1.4.6
+/* trunnel.h -- copied from Trunnel v1.5.1
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
 /* trunnel.h -- Public declarations for trunnel, to be included
  * in trunnel header files.
 
- * Copyright 2014-2015, The Tor Project, Inc.
+ * Copyright 2014-2017, The Tor Project, Inc.
  * See license at the end of this file for copying information.
  */
 
diff --git a/src/include.am b/src/include.am
index c468af3649..90ecf90d45 100644
--- a/src/include.am
+++ b/src/include.am
@@ -2,8 +2,10 @@ include src/ext/include.am
 include src/trunnel/include.am
 include src/common/include.am
 include src/or/include.am
+include src/rust/include.am
 include src/test/include.am
 include src/tools/include.am
 include src/win32/include.am
 include src/config/include.am
-
+include src/test/fuzz/include.am
+include src/trace/include.am
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake
index 2ac98cd372..429ae67858 100644
--- a/src/or/Makefile.nmake
+++ b/src/or/Makefile.nmake
@@ -14,6 +14,7 @@ LIBTOR_OBJECTS = \
   addressmap.obj \
   buffers.obj \
   channel.obj \
+  channelpadding.obj \
   channeltls.obj \
   circpathbias.obj \
   circuitbuild.obj \
diff --git a/src/or/addressmap.c b/src/or/addressmap.c
index 33fd7e0f4a..c92af38254 100644
--- a/src/or/addressmap.c
+++ b/src/or/addressmap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -376,29 +376,38 @@ addressmap_rewrite(char *address, size_t maxlen,
   char *addr_orig = tor_strdup(address);
   char *log_addr_orig = NULL;
 
+  /* We use a loop here to limit the total number of rewrites we do,
+   * so that we can't hit an infinite loop. */
   for (rewrites = 0; rewrites < 16; rewrites++) {
     int exact_match = 0;
     log_addr_orig = tor_strdup(escaped_safe_str_client(address));
 
+    /* First check to see if there's an exact match for this address */
     ent = strmap_get(addressmap, address);
 
     if (!ent || !ent->new_address) {
+      /* And if we don't have an exact match, try to check whether
+       * we have a pattern-based match.
+       */
       ent = addressmap_match_superdomains(address);
     } else {
       if (ent->src_wildcard && !ent->dst_wildcard &&
           !strcasecmp(address, ent->new_address)) {
-        /* This is a rule like *.example.com example.com, and we just got
-         * "example.com" */
+        /* This is a rule like "rewrite *.example.com to example.com", and we
+         * just got "example.com". Instead of calling it an infinite loop,
+         * call it complete. */
         goto done;
       }
-
       exact_match = 1;
     }
 
     if (!ent || !ent->new_address) {
+      /* We still have no match at all.  We're done! */
       goto done;
     }
 
+    /* Check wither the flags we were passed tell us not to use this
+     * mapping. */
     switch (ent->source) {
       case ADDRMAPSRC_DNS:
         {
@@ -431,6 +440,8 @@ addressmap_rewrite(char *address, size_t maxlen,
         goto done;
     }
 
+    /* Now fill in the address with the new address. That might be via
+     * appending some new stuff to the end, or via just replacing it. */
     if (ent->dst_wildcard && !exact_match) {
       strlcat(address, ".", maxlen);
       strlcat(address, ent->new_address, maxlen);
@@ -438,6 +449,7 @@ addressmap_rewrite(char *address, size_t maxlen,
       strlcpy(address, ent->new_address, maxlen);
     }
 
+    /* Is this now a .exit address?  If so, remember where we got it.*/
     if (!strcmpend(address, ".exit") &&
         strcmpend(addr_orig, ".exit") &&
         exit_source == ADDRMAPSRC_NONE) {
diff --git a/src/or/addressmap.h b/src/or/addressmap.h
index 67648d0518..80f453b4c1 100644
--- a/src/or/addressmap.h
+++ b/src/or/addressmap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ADDRESSMAP_H
diff --git a/src/or/bridges.c b/src/or/bridges.c
new file mode 100644
index 0000000000..0818fb0812
--- /dev/null
+++ b/src/or/bridges.c
@@ -0,0 +1,889 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.c
+ * \brief Code to manage bridges and bridge selection.
+ *
+ * Bridges are fixed entry nodes, used for censorship circumvention.
+ **/
+
+#include "or.h"
+#include "bridges.h"
+#include "circuitbuild.h"
+#include "config.h"
+#include "connection.h"
+#include "directory.h"
+#include "entrynodes.h"
+#include "nodelist.h"
+#include "policies.h"
+#include "router.h"
+#include "routerlist.h"
+#include "routerset.h"
+#include "transports.h"
+
+/** Information about a configured bridge. Currently this just matches the
+ * ones in the torrc file, but one day we may be able to learn about new
+ * bridges on our own, and remember them in the state file. */
+struct bridge_info_t {
+  /** Address and port of the bridge, as configured by the user.*/
+  tor_addr_port_t addrport_configured;
+  /** Address of the bridge. */
+  tor_addr_t addr;
+  /** TLS port for the bridge. */
+  uint16_t port;
+  /** Boolean: We are re-parsing our bridge list, and we are going to remove
+   * this one if we don't find it in the list of configured bridges. */
+  unsigned marked_for_removal : 1;
+  /** Expected identity digest, or all zero bytes if we don't know what the
+   * digest should be. */
+  char identity[DIGEST_LEN];
+
+  /** Name of pluggable transport protocol taken from its config line. */
+  char *transport_name;
+
+  /** When should we next try to fetch a descriptor for this bridge? */
+  download_status_t fetch_status;
+
+  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
+      transports are used for this bridge. */
+  smartlist_t *socks_args;
+};
+
+static void bridge_free(bridge_info_t *bridge);
+
+/** A list of configured bridges. Whenever we actually get a descriptor
+ * for one, we add it as an entry guard.  Note that the order of bridges
+ * in this list does not necessarily correspond to the order of bridges
+ * in the torrc. */
+static smartlist_t *bridge_list = NULL;
+
+/** Mark every entry of the bridge list to be removed on our next call to
+ * sweep_bridge_list unless it has first been un-marked. */
+void
+mark_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
+                    b->marked_for_removal = 1);
+}
+
+/** Remove every entry of the bridge list that was marked with
+ * mark_bridge_list if it has not subsequently been un-marked. */
+void
+sweep_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+    if (b->marked_for_removal) {
+      SMARTLIST_DEL_CURRENT(bridge_list, b);
+      bridge_free(b);
+    }
+  } SMARTLIST_FOREACH_END(b);
+}
+
+/** Initialize the bridge list to empty, creating it if needed. */
+static void
+clear_bridge_list(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
+  smartlist_clear(bridge_list);
+}
+
+/** Free the bridge <b>bridge</b>. */
+static void
+bridge_free(bridge_info_t *bridge)
+{
+  if (!bridge)
+    return;
+
+  tor_free(bridge->transport_name);
+  if (bridge->socks_args) {
+    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
+    smartlist_free(bridge->socks_args);
+  }
+
+  tor_free(bridge);
+}
+
+/** Return a list of all the configured bridges, as bridge_info_t pointers. */
+const smartlist_t *
+bridge_list_get(void)
+{
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+  return bridge_list;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its RSA identity digest, or
+ * NULL if we don't know one for it.
+ */
+const uint8_t *
+bridge_get_rsa_id_digest(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  if (tor_digest_is_zero(bridge->identity))
+    return NULL;
+  else
+    return (const uint8_t *) bridge->identity;
+}
+
+/**
+ * Given a <b>bridge</b>, return a pointer to its configured addr:port
+ * combination.
+ */
+const tor_addr_port_t *
+bridge_get_addr_port(const bridge_info_t *bridge)
+{
+  tor_assert(bridge);
+  return &bridge->addrport_configured;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches any of the
+ * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
+ * NULL. */
+static bridge_info_t *
+get_configured_bridge_by_orports_digest(const char *digest,
+                                        const smartlist_t *orports)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if (tor_digest_is_zero(bridge->identity)) {
+        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
+          {
+            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
+                bridge->port == ap->port)
+              return bridge;
+          }
+        SMARTLIST_FOREACH_END(ap);
+      }
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+bridge_info_t *
+get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
+          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+          bridge->port == port)
+        return bridge;
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    }
+  SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/**
+ * As get_configured_bridge_by_addr_port, but require that the
+ * address match <b>addr</b>:<b>port</b>, and that the ID digest match
+ * <b>digest</b>.  (The other function will ignore the address if the
+ * digest matches.)
+ */
+bridge_info_t *
+get_configured_bridge_by_exact_addr_port_digest(const tor_addr_t *addr,
+                                                uint16_t port,
+                                                const char *digest)
+{
+  if (!bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (!tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
+        bridge->port == port) {
+
+      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+      else if (!digest || tor_digest_is_zero(bridge->identity))
+        return bridge;
+    }
+
+  } SMARTLIST_FOREACH_END(bridge);
+  return NULL;
+}
+
+/** If we have a bridge configured whose digest matches <b>digest</b>, or a
+ * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
+ * return 1.  Else return 0. If <b>digest</b> is NULL, check for
+ * address/port matches only. */
+int
+addr_is_a_configured_bridge(const tor_addr_t *addr,
+                                uint16_t port,
+                                const char *digest)
+{
+  tor_assert(addr);
+  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
+}
+
+/** If we have a bridge configured whose digest matches
+ * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
+ * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
+ * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
+int
+extend_info_is_a_configured_bridge(const extend_info_t *ei)
+{
+  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
+  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
+}
+
+/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
+ * it up via router descriptor <b>ri</b>. */
+static bridge_info_t *
+get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
+{
+  bridge_info_t *bi = NULL;
+  smartlist_t *orports = router_get_all_orports(ri);
+  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
+                                               orports);
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return bi;
+}
+
+/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
+int
+routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
+{
+  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
+}
+
+/** Return 1 if <b>node</b> is one of our configured bridges, else 0. */
+int
+node_is_a_configured_bridge(const node_t *node)
+{
+  int retval = 0;
+  smartlist_t *orports = node_get_all_orports(node);
+  retval = get_configured_bridge_by_orports_digest(node->identity,
+                                                   orports) != NULL;
+  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
+  smartlist_free(orports);
+  return retval;
+}
+
+/** We made a connection to a router at <b>addr</b>:<b>port</b>
+ * without knowing its digest. Its digest turned out to be <b>digest</b>.
+ * If it was a bridge, and we still don't know its digest, record it.
+ */
+void
+learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                        const char *digest,
+                        const ed25519_public_key_t *ed_id)
+{
+  // XXXX prop220 use ed_id here, once there is some way to specify
+  (void)ed_id;
+  int learned = 0;
+  bridge_info_t *bridge =
+    get_configured_bridge_by_exact_addr_port_digest(addr, port, digest);
+  if (bridge && tor_digest_is_zero(bridge->identity)) {
+    memcpy(bridge->identity, digest, DIGEST_LEN);
+    learned = 1;
+  }
+  /* XXXX prop220 remember bridge ed25519 identities -- add a field */
+#if 0
+  if (bridge && ed_id &&
+      ed25519_public_key_is_zero(&bridge->ed25519_identity) &&
+      !ed25519_public_key_is_zero(ed_id)) {
+    memcpy(&bridge->ed25519_identity, ed_id, sizeof(*ed_id));
+    learned = 1;
+  }
+#endif
+  if (learned) {
+    char *transport_info = NULL;
+    const char *transport_name =
+      find_transport_name_by_bridge_addrport(addr, port);
+    if (transport_name)
+      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
+
+    // XXXX prop220 log both fingerprints.
+    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
+               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
+               transport_info ? transport_info : "");
+    tor_free(transport_info);
+    entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                                        (const uint8_t *)digest);
+  }
+}
+
+/** Return true if <b>bridge</b> has the same identity digest as
+ *  <b>digest</b>. If <b>digest</b> is NULL, it matches
+ *  bridges with unspecified identity digests. */
+static int
+bridge_has_digest(const bridge_info_t *bridge, const char *digest)
+{
+  if (digest)
+    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
+  else
+    return tor_digest_is_zero(bridge->identity);
+}
+
+/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
+ * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
+ * existing bridge with the same address and port, and warn the user as
+ * appropriate.
+ */
+static void
+bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
+                         const char *digest, const char *transport_name)
+{
+  /* Iterate the already-registered bridge list:
+
+     If you find a bridge with the same adress and port, mark it for
+     removal. It doesn't make sense to have two active bridges with
+     the same IP:PORT. If the bridge in question has a different
+     digest or transport than <b>digest</b>/<b>transport_name</b>,
+     it's probably a misconfiguration and we should warn the user.
+  */
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    if (bridge->marked_for_removal)
+      continue;
+
+    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
+
+      bridge->marked_for_removal = 1;
+
+      if (!bridge_has_digest(bridge, digest) ||
+          strcmp_opt(bridge->transport_name, transport_name)) {
+        /* warn the user */
+        char *bridge_description_new, *bridge_description_old;
+        tor_asprintf(&bridge_description_new, "%s:%s:%s",
+                     fmt_addrport(addr, port),
+                     digest ? hex_str(digest, DIGEST_LEN) : "",
+                     transport_name ? transport_name : "");
+        tor_asprintf(&bridge_description_old, "%s:%s:%s",
+                     fmt_addrport(&bridge->addr, bridge->port),
+                     tor_digest_is_zero(bridge->identity) ?
+                     "" : hex_str(bridge->identity,DIGEST_LEN),
+                     bridge->transport_name ? bridge->transport_name : "");
+
+        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
+                 " with the already registered bridge '%s'. We will discard"
+                 " the old bridge and keep '%s'. If this is not what you"
+                 " wanted, please change your configuration file accordingly.",
+                 bridge_description_new, bridge_description_old,
+                 bridge_description_new);
+
+        tor_free(bridge_description_new);
+        tor_free(bridge_description_old);
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+}
+
+/** Return True if we have a bridge that uses a transport with name
+ *  <b>transport_name</b>. */
+MOCK_IMPL(int,
+transport_is_needed, (const char *transport_name))
+{
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (bridge->transport_name &&
+        !strcmp(bridge->transport_name, transport_name))
+      return 1;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return 0;
+}
+
+/** Register the bridge information in <b>bridge_line</b> to the
+ *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
+void
+bridge_add_from_config(bridge_line_t *bridge_line)
+{
+  bridge_info_t *b;
+
+  // XXXX prop220 add a way to specify ed25519 ID to bridge_line_t.
+
+  { /* Log the bridge we are about to register: */
+    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
+              fmt_addrport(&bridge_line->addr, bridge_line->port),
+              bridge_line->transport_name ?
+              bridge_line->transport_name : "no transport",
+              tor_digest_is_zero(bridge_line->digest) ?
+              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
+
+    if (bridge_line->socks_args) { /* print socks arguments */
+      int i = 0;
+
+      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
+
+      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
+                smartlist_len(bridge_line->socks_args));
+      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
+                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
+    }
+  }
+
+  bridge_resolve_conflicts(&bridge_line->addr,
+                           bridge_line->port,
+                           bridge_line->digest,
+                           bridge_line->transport_name);
+
+  b = tor_malloc_zero(sizeof(bridge_info_t));
+  tor_addr_copy(&b->addrport_configured.addr, &bridge_line->addr);
+  b->addrport_configured.port = bridge_line->port;
+  tor_addr_copy(&b->addr, &bridge_line->addr);
+  b->port = bridge_line->port;
+  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
+  if (bridge_line->transport_name)
+    b->transport_name = bridge_line->transport_name;
+  b->fetch_status.schedule = DL_SCHED_BRIDGE;
+  b->fetch_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL;
+  b->socks_args = bridge_line->socks_args;
+  if (!bridge_list)
+    bridge_list = smartlist_new();
+
+  tor_free(bridge_line); /* Deallocate bridge_line now. */
+
+  smartlist_add(bridge_list, b);
+}
+
+/** If <b>digest</b> is one of our known bridges, return it. */
+bridge_info_t *
+find_bridge_by_digest(const char *digest)
+{
+  if (! bridge_list)
+    return NULL;
+  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
+    {
+      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
+        return bridge;
+    });
+  return NULL;
+}
+
+/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
+ *  supports a pluggable transport, return its name. Otherwise, return
+ *  NULL. */
+const char *
+find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  if (!bridge_list)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port))
+      return bridge->transport_name;
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return NULL;
+}
+
+/** If <b>addr</b> and <b>port</b> match the address and port of a
+ * bridge of ours that uses pluggable transports, place its transport
+ * in <b>transport</b>.
+ *
+ * Return 0 on success (found a transport, or found a bridge with no
+ * transport, or found no bridge); return -1 if we should be using a
+ * transport, but the transport could not be found.
+ */
+int
+get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                  const transport_t **transport)
+{
+  *transport = NULL;
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
+    if (tor_addr_eq(&bridge->addr, addr) &&
+        (bridge->port == port)) { /* bridge matched */
+      if (bridge->transport_name) { /* it also uses pluggable transports */
+        *transport = transport_get_by_name(bridge->transport_name);
+        if (*transport == NULL) { /* it uses pluggable transports, but
+                                     the transport could not be found! */
+          return -1;
+        }
+        return 0;
+      } else { /* bridge matched, but it doesn't use transports. */
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+
+  *transport = NULL;
+  return 0;
+}
+
+/** Return a smartlist containing all the SOCKS arguments that we
+ *  should pass to the SOCKS proxy. */
+const smartlist_t *
+get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+{
+  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
+                                                                    port,
+                                                                    NULL);
+  return bridge ? bridge->socks_args : NULL;
+}
+
+/** We need to ask <b>bridge</b> for its server descriptor. */
+static void
+launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
+{
+  const or_options_t *options = get_options();
+  circuit_guard_state_t *guard_state = NULL;
+
+  if (connection_get_by_type_addr_port_purpose(
+      CONN_TYPE_DIR, &bridge->addr, bridge->port,
+      DIR_PURPOSE_FETCH_SERVERDESC))
+    return; /* it's already on the way */
+
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+    download_status_mark_impossible(&bridge->fetch_status);
+    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+    return;
+  }
+
+  /* Until we get a descriptor for the bridge, we only know one address for
+   * it. */
+  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                            FIREWALL_OR_CONNECTION, 0, 0)) {
+    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
+               "bridge, but that bridge is not reachable through our "
+               "firewall.");
+    return;
+  }
+
+  tor_addr_port_t bridge_addrport;
+  memcpy(&bridge_addrport.addr, &bridge->addr, sizeof(tor_addr_t));
+  bridge_addrport.port = bridge->port;
+
+  guard_state = get_guard_state_for_bridge_desc_fetch(bridge->identity);
+
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+  directory_request_set_or_addr_port(req, &bridge_addrport);
+  directory_request_set_directory_id_digest(req, bridge->identity);
+  directory_request_set_router_purpose(req, ROUTER_PURPOSE_BRIDGE);
+  directory_request_set_resource(req, "authority.z");
+  if (guard_state) {
+    directory_request_set_guard_state(req, guard_state);
+  }
+  directory_initiate_request(req);
+  directory_request_free(req);
+}
+
+/** Fetching the bridge descriptor from the bridge authority returned a
+ * "not found". Fall back to trying a direct fetch. */
+void
+retry_bridge_descriptor_fetch_directly(const char *digest)
+{
+  bridge_info_t *bridge = find_bridge_by_digest(digest);
+  if (!bridge)
+    return; /* not found? oh well. */
+
+  launch_direct_bridge_descriptor_fetch(bridge);
+}
+
+/** For each bridge in our list for which we don't currently have a
+ * descriptor, fetch a new copy of its descriptor -- either directly
+ * from the bridge or via a bridge authority. */
+void
+fetch_bridge_descriptors(const or_options_t *options, time_t now)
+{
+  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
+  int ask_bridge_directly;
+  int can_use_bridge_authority;
+
+  if (!bridge_list)
+    return;
+
+  /* If we still have unconfigured managed proxies, don't go and
+     connect to a bridge. */
+  if (pt_proxies_configuration_pending())
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
+    {
+      if (!download_status_is_ready(&bridge->fetch_status, now,
+                                    IMPOSSIBLE_TO_DOWNLOAD))
+        continue; /* don't bother, no need to retry yet */
+      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
+        download_status_mark_impossible(&bridge->fetch_status);
+        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
+                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
+        continue;
+      }
+
+      /* schedule another fetch as if this one will fail, in case it does */
+      download_status_failed(&bridge->fetch_status, 0);
+
+      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
+                                 num_bridge_auths;
+      ask_bridge_directly = !can_use_bridge_authority ||
+                            !options->UpdateBridgesFromAuthority;
+      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
+                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
+                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
+
+      if (ask_bridge_directly &&
+          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
+                                                FIREWALL_OR_CONNECTION, 0,
+                                                0)) {
+        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
+                   "firewall policy. %s.",
+                   fmt_addrport(&bridge->addr, bridge->port),
+                   can_use_bridge_authority ?
+                     "Asking bridge authority instead" : "Skipping");
+        if (can_use_bridge_authority)
+          ask_bridge_directly = 0;
+        else
+          continue;
+      }
+
+      if (ask_bridge_directly) {
+        /* we need to ask the bridge itself for its descriptor. */
+        launch_direct_bridge_descriptor_fetch(bridge);
+      } else {
+        /* We have a digest and we want to ask an authority. We could
+         * combine all the requests into one, but that may give more
+         * hints to the bridge authority than we want to give. */
+        char resource[10 + HEX_DIGEST_LEN];
+        memcpy(resource, "fp/", 3);
+        base16_encode(resource+3, HEX_DIGEST_LEN+1,
+                      bridge->identity, DIGEST_LEN);
+        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
+        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
+                 resource);
+        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
+                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
+      }
+    }
+  SMARTLIST_FOREACH_END(bridge);
+}
+
+/** If our <b>bridge</b> is configured to be a different address than
+ * the bridge gives in <b>node</b>, rewrite the routerinfo
+ * we received to use the address we meant to use. Now we handle
+ * multihomed bridges better.
+ */
+static void
+rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
+{
+  /* XXXX move this function. */
+  /* XXXX overridden addresses should really live in the node_t, so that the
+   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
+   *   do that safely if we know that no function that connects to an OR
+   *   does so through an address from any source other than node_get_addr().
+   */
+  tor_addr_t addr;
+  const or_options_t *options = get_options();
+
+  if (node->ri) {
+    routerinfo_t *ri = node->ri;
+    tor_addr_from_ipv4h(&addr, ri->addr);
+
+    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+         bridge->port == ri->or_port) ||
+        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
+         bridge->port == ri->ipv6_orport)) {
+      /* they match, so no need to do anything */
+    } else {
+      if (tor_addr_family(&bridge->addr) == AF_INET) {
+        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
+        ri->or_port = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s:%d.",
+                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
+      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
+        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
+        ri->ipv6_orport = bridge->port;
+        log_info(LD_DIR,
+                 "Adjusted bridge routerinfo for '%s' to match configured "
+                 "address %s.",
+                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
+      } else {
+        log_err(LD_BUG, "Address family not supported: %d.",
+                tor_addr_family(&bridge->addr));
+        return;
+      }
+    }
+
+    if (options->ClientPreferIPv6ORPort == -1) {
+      /* Mark which address to use based on which bridge_t we got. */
+      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    } else {
+      /* Mark which address to use based on user preference */
+      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
+                              !tor_addr_is_null(&node->ri->ipv6_addr));
+    }
+
+    /* XXXipv6 we lack support for falling back to another address for
+       the same relay, warn the user */
+    if (!tor_addr_is_null(&ri->ipv6_addr)) {
+      tor_addr_port_t ap;
+      node_get_pref_orport(node, &ap);
+      log_notice(LD_CONFIG,
+                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
+                 "Will prefer using its %s address (%s) based on %s.",
+                 ri->nickname,
+                 node->ipv6_preferred ? "IPv6" : "IPv4",
+                 fmt_addrport(&ap.addr, ap.port),
+                 options->ClientPreferIPv6ORPort == -1 ?
+                 "the configured Bridge address" :
+                 "ClientPreferIPv6ORPort");
+    }
+  }
+  if (node->rs) {
+    routerstatus_t *rs = node->rs;
+    tor_addr_from_ipv4h(&addr, rs->addr);
+
+    if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
+        bridge->port == rs->or_port) {
+      /* they match, so no need to do anything */
+    } else {
+      rs->addr = tor_addr_to_ipv4h(&bridge->addr);
+      rs->or_port = bridge->port;
+      log_info(LD_DIR,
+               "Adjusted bridge routerstatus for '%s' to match "
+               "configured address %s.",
+               rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
+    }
+  }
+}
+
+/** We just learned a descriptor for a bridge. See if that
+ * digest is in our entry guard list, and add it if not. */
+void
+learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
+{
+  tor_assert(ri);
+  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
+  if (get_options()->UseBridges) {
+    int first = num_bridges_usable() <= 1;
+    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
+    time_t now = time(NULL);
+    router_set_status(ri->cache_info.identity_digest, 1);
+
+    if (bridge) { /* if we actually want to use this one */
+      node_t *node;
+      /* it's here; schedule its re-fetch for a long time from now. */
+      if (!from_cache)
+        download_status_reset(&bridge->fetch_status);
+
+      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
+      tor_assert(node);
+      rewrite_node_address_for_bridge(bridge, node);
+      if (tor_digest_is_zero(bridge->identity)) {
+        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
+        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
+                   hex_str(bridge->identity, DIGEST_LEN),
+                   fmt_and_decorate_addr(&bridge->addr),
+                   (int) bridge->port);
+      }
+      entry_guard_learned_bridge_identity(&bridge->addrport_configured,
+                              (const uint8_t*)ri->cache_info.identity_digest);
+
+      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
+                 from_cache ? "cached" : "fresh", router_describe(ri));
+      /* set entry->made_contact so if it goes down we don't drop it from
+       * our entry node list */
+      if (first) {
+        routerlist_retry_directory_downloads(now);
+      }
+    }
+  }
+}
+
+/** Return the number of bridges that have descriptors that
+ * are marked with purpose 'bridge' and are running.
+ *
+ * We use this function to decide if we're ready to start building
+ * circuits through our bridges, or if we need to wait until the
+ * directory "server/authority" requests finish. */
+int
+any_bridge_descriptors_known(void)
+{
+  tor_assert(get_options()->UseBridges);
+
+  if (!bridge_list)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
+    const node_t *node;
+    if (!tor_digest_is_zero(bridge->identity) &&
+        (node = node_get_by_id(bridge->identity)) != NULL &&
+        node->ri) {
+      return 1;
+    }
+  } SMARTLIST_FOREACH_END(bridge);
+
+  return 0;
+}
+
+/** Return a smartlist containing all bridge identity digests */
+MOCK_IMPL(smartlist_t *,
+list_bridge_identities, (void))
+{
+  smartlist_t *result = NULL;
+  char *digest_tmp;
+
+  if (get_options()->UseBridges && bridge_list) {
+    result = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      digest_tmp = tor_malloc(DIGEST_LEN);
+      memcpy(digest_tmp, b->identity, DIGEST_LEN);
+      smartlist_add(result, digest_tmp);
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return result;
+}
+
+/** Get the download status for a bridge descriptor given its identity */
+MOCK_IMPL(download_status_t *,
+get_bridge_dl_status_by_id, (const char *digest))
+{
+  download_status_t *dl = NULL;
+
+  if (digest && get_options()->UseBridges && bridge_list) {
+    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
+      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
+        dl = &(b->fetch_status);
+        break;
+      }
+    } SMARTLIST_FOREACH_END(b);
+  }
+
+  return dl;
+}
+
+/** Release all storage held in bridges.c */
+void
+bridges_free_all(void)
+{
+  clear_bridge_list();
+  smartlist_free(bridge_list);
+  bridge_list = NULL;
+}
+
diff --git a/src/or/bridges.h b/src/or/bridges.h
new file mode 100644
index 0000000000..3bfc782f9a
--- /dev/null
+++ b/src/or/bridges.h
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file bridges.h
+ * \brief Header file for circuitbuild.c.
+ **/
+
+#ifndef TOR_BRIDGES_H
+#define TOR_BRIDGES_H
+
+struct bridge_line_t;
+
+/* Opaque handle to a configured bridge */
+typedef struct bridge_info_t bridge_info_t;
+
+void mark_bridge_list(void);
+void sweep_bridge_list(void);
+const smartlist_t *bridge_list_get(void);
+bridge_info_t *find_bridge_by_digest(const char *digest);
+const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge);
+const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge);
+bridge_info_t *get_configured_bridge_by_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+bridge_info_t *get_configured_bridge_by_exact_addr_port_digest(
+                                          const tor_addr_t *addr,
+                                          uint16_t port,
+                                          const char *digest);
+
+int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
+                                const char *digest);
+int extend_info_is_a_configured_bridge(const extend_info_t *ei);
+int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
+int node_is_a_configured_bridge(const node_t *node);
+void learned_router_identity(const tor_addr_t *addr, uint16_t port,
+                             const char *digest,
+                             const ed25519_public_key_t *ed_id);
+
+void bridge_add_from_config(struct bridge_line_t *bridge_line);
+void retry_bridge_descriptor_fetch_directly(const char *digest);
+void fetch_bridge_descriptors(const or_options_t *options, time_t now);
+void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
+int any_bridge_descriptors_known(void);
+const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
+                                                     uint16_t port);
+
+int any_bridges_dont_support_microdescriptors(void);
+
+const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
+                                                   uint16_t port);
+struct transport_t;
+int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
+                                      const struct transport_t **transport);
+
+MOCK_DECL(int, transport_is_needed, (const char *transport_name));
+int validate_pluggable_transports_config(void);
+
+MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
+MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
+          (const char *digest));
+
+void bridges_free_all(void);
+
+#endif
+
diff --git a/src/or/buffers.c b/src/or/buffers.c
index 89382d1d8e..12a6c0239b 100644
--- a/src/or/buffers.c
+++ b/src/or/buffers.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -83,7 +83,11 @@ static int parse_socks_client(const uint8_t *data, size_t datalen,
 #define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0])
 
 /* We leave this many NUL bytes at the end of the buffer. */
+#ifdef DISABLE_MEMORY_SENTINELS
+#define SENTINEL_LEN 0
+#else
 #define SENTINEL_LEN 4
+#endif
 
 /* Header size plus NUL bytes at the end */
 #define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
@@ -97,18 +101,22 @@ static int parse_socks_client(const uint8_t *data, size_t datalen,
 
 #define DEBUG_SENTINEL
 
-#ifdef DEBUG_SENTINEL
+#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS)
 #define DBG_S(s) s
 #else
 #define DBG_S(s) (void)0
 #endif
 
+#ifdef DISABLE_MEMORY_SENTINELS
+#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
+#else
 #define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
     uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
     DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
     DBG_S(tor_assert(a == b));                                          \
     memset(a,0,SENTINEL_LEN);                                           \
   } while (0)
+#endif
 
 /** Return the next character in <b>chunk</b> onto which data can be appended.
  * If the chunk is full, this might be off the end of chunk->mem. */
@@ -281,6 +289,7 @@ buf_pullup(buf_t *buf, size_t bytes)
 }
 
 #ifdef TOR_UNIT_TESTS
+/* Return the data from the first chunk of buf in cp, and its length in sz. */
 void
 buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz)
 {
@@ -292,6 +301,53 @@ buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz)
     *sz = buf->head->datalen;
   }
 }
+
+/* Write sz bytes from cp into a newly allocated buffer buf.
+ * Returns NULL when passed a NULL cp or zero sz.
+ * Asserts on failure: only for use in unit tests.
+ * buf must be freed using buf_free(). */
+buf_t *
+buf_new_with_data(const char *cp, size_t sz)
+{
+  /* Validate arguments */
+  if (!cp || sz <= 0) {
+    return NULL;
+  }
+
+  tor_assert(sz < SSIZE_T_CEILING);
+
+  /* Allocate a buffer */
+  buf_t *buf = buf_new_with_capacity(sz);
+  tor_assert(buf);
+  assert_buf_ok(buf);
+  tor_assert(!buf->head);
+
+  /* Allocate a chunk that is sz bytes long */
+  buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz));
+  buf->tail = buf->head;
+  tor_assert(buf->head);
+  assert_buf_ok(buf);
+  tor_assert(buf_allocation(buf) >= sz);
+
+  /* Copy the data and size the buffers */
+  tor_assert(sz <= buf_slack(buf));
+  tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head));
+  memcpy(&buf->head->mem[0], cp, sz);
+  buf->datalen = sz;
+  buf->head->datalen = sz;
+  buf->head->data = &buf->head->mem[0];
+  assert_buf_ok(buf);
+
+  /* Make sure everything is large enough */
+  tor_assert(buf_allocation(buf) >= sz);
+  tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf));
+  /* Does the buffer implementation allocate more than the requested size?
+   * (for example, by rounding up). If so, these checks will fail. */
+  tor_assert(buf_datalen(buf) == sz);
+  tor_assert(buf_slack(buf) == 0);
+
+  return buf;
+}
 #endif
 
 /** Remove the first <b>n</b> bytes from buf. */
@@ -562,6 +618,11 @@ read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof,
   tor_assert(reached_eof);
   tor_assert(SOCKET_OK(s));
 
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
   while (at_most > total_read) {
     size_t readlen = at_most - total_read;
     chunk_t *chunk;
@@ -619,6 +680,11 @@ read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf)
 
   check();
 
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
   while (at_most > total_read) {
     size_t readlen = at_most - total_read;
     chunk_t *chunk;
@@ -813,6 +879,11 @@ write_to_buf(const char *string, size_t string_len, buf_t *buf)
     return (int)buf->datalen;
   check();
 
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - string_len))
+    return -1;
+
   while (string_len) {
     size_t copy;
     if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
@@ -962,6 +1033,12 @@ move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
   /* We can do way better here, but this doesn't turn up in any profiles. */
   char b[4096];
   size_t cp, len;
+
+  if (BUG(buf_out->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+    return -1;
+
   len = *buf_flushlen;
   if (len > buf_in->datalen)
     len = buf_in->datalen;
@@ -1090,6 +1167,52 @@ buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
   return -1;
 }
 
+/**
+ * Scan the HTTP headers in the <b>headerlen</b>-byte memory range at
+ * <b>headers</b>, looking for a "Content-Length" header.  Try to set
+ * *<b>result_out</b> to the numeric value of that header if possible.
+ * Return -1 if the header was malformed, 0 if it was missing, and 1 if
+ * it was present and well-formed.
+ */
+STATIC int
+buf_http_find_content_length(const char *headers, size_t headerlen,
+                             size_t *result_out)
+{
+  const char *p, *newline;
+  char *len_str, *eos=NULL;
+  size_t remaining, result;
+  int ok;
+  *result_out = 0; /* The caller shouldn't look at this unless the
+                    * return value is 1, but let's prevent confusion */
+
+#define CONTENT_LENGTH "\r\nContent-Length: "
+  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
+  if (p == NULL)
+    return 0;
+
+  tor_assert(p >= headers && p < headers+headerlen);
+  remaining = (headers+headerlen)-p;
+  p += strlen(CONTENT_LENGTH);
+  remaining -= strlen(CONTENT_LENGTH);
+
+  newline = memchr(p, '\n', remaining);
+  if (newline == NULL)
+    return -1;
+
+  len_str = tor_memdup_nulterm(p, newline-p);
+  /* We limit the size to INT_MAX because other parts of the buffer.c
+   * code don't like buffers to be any bigger than that. */
+  result = (size_t) tor_parse_uint64(len_str, 10, 0, INT_MAX, &ok, &eos);
+  if (eos && !tor_strisspace(eos)) {
+    ok = 0;
+  } else {
+    *result_out = result;
+  }
+  tor_free(len_str);
+
+  return ok ? 1 : -1;
+}
+
 /** There is a (possibly incomplete) http statement on <b>buf</b>, of the
  * form "\%s\\r\\n\\r\\n\%s", headers, body. (body may contain NULs.)
  * If a) the headers include a Content-Length field and all bytes in
@@ -1115,9 +1238,10 @@ fetch_from_buf_http(buf_t *buf,
                     char **body_out, size_t *body_used, size_t max_bodylen,
                     int force_complete)
 {
-  char *headers, *p;
-  size_t headerlen, bodylen, contentlen;
+  char *headers;
+  size_t headerlen, bodylen, contentlen=0;
   int crlf_offset;
+  int r;
 
   check();
   if (!buf->head)
@@ -1153,17 +1277,12 @@ fetch_from_buf_http(buf_t *buf,
     return -1;
   }
 
-#define CONTENT_LENGTH "\r\nContent-Length: "
-  p = (char*) tor_memstr(headers, headerlen, CONTENT_LENGTH);
-  if (p) {
-    int i;
-    i = atoi(p+strlen(CONTENT_LENGTH));
-    if (i < 0) {
-      log_warn(LD_PROTOCOL, "Content-Length is less than zero; it looks like "
-               "someone is trying to crash us.");
-      return -1;
-    }
-    contentlen = i;
+  r = buf_http_find_content_length(headers, headerlen, &contentlen);
+  if (r == -1) {
+    log_warn(LD_PROTOCOL, "Content-Length is bogus; maybe "
+             "someone is trying to crash us.");
+    return -1;
+  } else if (r == 1) {
     /* if content-length is malformed, then our body length is 0. fine. */
     log_debug(LD_HTTP,"Got a contentlen of %d.",(int)contentlen);
     if (bodylen < contentlen) {
@@ -1176,7 +1295,11 @@ fetch_from_buf_http(buf_t *buf,
       bodylen = contentlen;
       log_debug(LD_HTTP,"bodylen reduced to %d.",(int)bodylen);
     }
+  } else {
+    tor_assert(r == 0);
+    /* Leave bodylen alone */
   }
+
   /* all happy. copy into the appropriate places, and return 1 */
   if (headers_out) {
     *headers_out = tor_malloc(headerlen+1);
@@ -1196,7 +1319,7 @@ fetch_from_buf_http(buf_t *buf,
 
 /**
  * Wait this many seconds before warning the user about using SOCKS unsafely
- * again (requires that WarnUnsafeSocks is turned on). */
+ * again. */
 #define SOCKS_WARN_INTERVAL 5
 
 /** Warn that the user application has made an unsafe socks request using
@@ -1208,9 +1331,6 @@ log_unsafe_socks_warning(int socks_protocol, const char *address,
 {
   static ratelim_t socks_ratelim = RATELIM_INIT(SOCKS_WARN_INTERVAL);
 
-  const or_options_t *options = get_options();
-  if (! options->WarnUnsafeSocks)
-    return;
   if (safe_socks) {
     log_fn_ratelim(&socks_ratelim, LOG_WARN, LD_APP,
              "Your application (using socks%d to port %d) is giving "
@@ -1966,13 +2086,13 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len)
 }
 
 /** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the
- * zlib state <b>state</b>, appending the result to <b>buf</b>.  If
+ * compression state <b>state</b>, appending the result to <b>buf</b>.  If
  * <b>done</b> is true, flush the data in the state and finish the
  * compression/uncompression.  Return -1 on failure, 0 on success. */
 int
-write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
-                  const char *data, size_t data_len,
-                  int done)
+write_to_buf_compress(buf_t *buf, tor_compress_state_t *state,
+                      const char *data, size_t data_len,
+                      const int done)
 {
   char *next;
   size_t old_avail, avail;
@@ -1986,22 +2106,31 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
     }
     next = CHUNK_WRITE_PTR(buf->tail);
     avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
-    switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) {
-      case TOR_ZLIB_DONE:
+    switch (tor_compress_process(state, &next, &avail,
+                                 &data, &data_len, done)) {
+      case TOR_COMPRESS_DONE:
         over = 1;
         break;
-      case TOR_ZLIB_ERR:
+      case TOR_COMPRESS_ERROR:
         return -1;
-      case TOR_ZLIB_OK:
-        if (data_len == 0)
+      case TOR_COMPRESS_OK:
+        if (data_len == 0) {
+          tor_assert_nonfatal(!done);
           over = 1;
+        }
         break;
-      case TOR_ZLIB_BUF_FULL:
+      case TOR_COMPRESS_BUFFER_FULL:
         if (avail) {
-          /* Zlib says we need more room (ZLIB_BUF_FULL).  Start a new chunk
-           * automatically, whether were going to or not. */
+          /* The compression module says we need more room
+           * (TOR_COMPRESS_BUFFER_FULL).  Start a new chunk automatically,
+           * whether were going to or not. */
           need_new_chunk = 1;
         }
+        if (data_len == 0 && !done) {
+          /* We've consumed all the input data, though, so there's no
+           * point in forging ahead right now. */
+          over = 1;
+        }
         break;
     }
     buf->datalen += old_avail - avail;
diff --git a/src/or/buffers.h b/src/or/buffers.h
index 52b21d5885..23b58a571a 100644
--- a/src/or/buffers.h
+++ b/src/or/buffers.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,8 +36,8 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen);
 int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen);
 
 int write_to_buf(const char *string, size_t string_len, buf_t *buf);
-int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state,
-                      const char *data, size_t data_len, int done);
+int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state,
+                          const char *data, size_t data_len, int done);
 int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
 int fetch_from_buf(char *string, size_t string_len, buf_t *buf);
 int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto);
@@ -64,7 +64,10 @@ void assert_buf_ok(buf_t *buf);
 #ifdef BUFFERS_PRIVATE
 STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
 STATIC void buf_pullup(buf_t *buf, size_t bytes);
+#ifdef TOR_UNIT_TESTS
 void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz);
+buf_t *buf_new_with_data(const char *cp, size_t sz);
+#endif
 STATIC size_t preferred_chunk_size(size_t target);
 
 #define DEBUG_CHUNK_ALLOC
@@ -97,5 +100,10 @@ struct buf_t {
 };
 #endif
 
+#ifdef BUFFERS_PRIVATE
+STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
+                                        size_t *result_out);
+#endif
+
 #endif
 
diff --git a/src/or/channel.c b/src/or/channel.c
index 54e10666d2..9f652b5845 100644
--- a/src/or/channel.c
+++ b/src/or/channel.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,6 +49,7 @@
 #include "or.h"
 #include "channel.h"
 #include "channeltls.h"
+#include "channelpadding.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuitstats.h"
@@ -63,6 +64,9 @@
 #include "router.h"
 #include "routerlist.h"
 #include "scheduler.h"
+#include "compat_time.h"
+#include "networkstatus.h"
+#include "rendservice.h"
 
 /* Global lists of channels */
 
@@ -84,6 +88,28 @@ static smartlist_t *active_listeners = NULL;
 /* All channel_listener_t instances in LISTENING state */
 static smartlist_t *finished_listeners = NULL;
 
+/** Map from channel->global_identifier to channel.  Contains the same
+ * elements as all_channels. */
+static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER();
+
+static unsigned
+channel_id_hash(const channel_t *chan)
+{
+  return (unsigned) chan->global_identifier;
+}
+static int
+channel_id_eq(const channel_t *a, const channel_t *b)
+{
+  return a->global_identifier == b->global_identifier;
+}
+HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node,
+             channel_id_hash, channel_id_eq)
+HT_GENERATE2(channel_gid_map, channel_s, gidmap_node,
+             channel_id_hash, channel_id_eq,
+             0.6, tor_reallocarray_, tor_free_)
+
+HANDLE_IMPL(channel, channel_s,)
+
 /* Counter for ID numbers */
 static uint64_t n_channels_allocated = 0;
 /*
@@ -429,6 +455,7 @@ void
 channel_register(channel_t *chan)
 {
   tor_assert(chan);
+  tor_assert(chan->global_identifier);
 
   /* No-op if already registered */
   if (chan->registered) return;
@@ -443,6 +470,8 @@ channel_register(channel_t *chan)
   /* Make sure we have all_channels, then add it */
   if (!all_channels) all_channels = smartlist_new();
   smartlist_add(all_channels, chan);
+  channel_t *oldval = HT_REPLACE(channel_gid_map, &channel_gid_map, chan);
+  tor_assert(! oldval);
 
   /* Is it finished? */
   if (CHANNEL_FINISHED(chan)) {
@@ -498,7 +527,9 @@ channel_unregister(channel_t *chan)
   }
 
   /* Get it out of all_channels */
- if (all_channels) smartlist_remove(all_channels, chan);
+  if (all_channels) smartlist_remove(all_channels, chan);
+  channel_t *oldval = HT_REMOVE(channel_gid_map, &channel_gid_map, chan);
+  tor_assert(oldval == NULL || oldval == chan);
 
   /* Mark it as unregistered */
   chan->registered = 0;
@@ -533,7 +564,7 @@ channel_listener_register(channel_listener_t *chan_l)
             channel_listener_state_to_string(chan_l->state),
             chan_l->state);
 
-  /* Make sure we have all_channels, then add it */
+  /* Make sure we have all_listeners, then add it */
   if (!all_listeners) all_listeners = smartlist_new();
   smartlist_add(all_listeners, chan_l);
 
@@ -578,7 +609,7 @@ channel_listener_unregister(channel_listener_t *chan_l)
     if (active_listeners) smartlist_remove(active_listeners, chan_l);
   }
 
-  /* Get it out of all_channels */
+  /* Get it out of all_listeners */
  if (all_listeners) smartlist_remove(all_listeners, chan_l);
 
   /* Mark it as unregistered */
@@ -719,41 +750,74 @@ channel_remove_from_digest_map(channel_t *chan)
 channel_t *
 channel_find_by_global_id(uint64_t global_identifier)
 {
+  channel_t lookup;
   channel_t *rv = NULL;
 
-  if (all_channels && smartlist_len(all_channels) > 0) {
-    SMARTLIST_FOREACH_BEGIN(all_channels, channel_t *, curr) {
-      if (curr->global_identifier == global_identifier) {
-        rv = curr;
-        break;
-      }
-    } SMARTLIST_FOREACH_END(curr);
+  lookup.global_identifier = global_identifier;
+  rv = HT_FIND(channel_gid_map, &channel_gid_map, &lookup);
+  if (rv) {
+    tor_assert(rv->global_identifier == global_identifier);
   }
 
   return rv;
 }
 
+/** Return true iff <b>chan</b> matches <b>rsa_id_digest</b> and <b>ed_id</b>.
+ * as its identity keys.  If either is NULL, do not check for a match. */
+static int
+channel_remote_identity_matches(const channel_t *chan,
+                                const char *rsa_id_digest,
+                                const ed25519_public_key_t *ed_id)
+{
+  if (BUG(!chan))
+    return 0;
+  if (rsa_id_digest) {
+    if (tor_memneq(rsa_id_digest, chan->identity_digest, DIGEST_LEN))
+      return 0;
+  }
+  if (ed_id) {
+    if (tor_memneq(ed_id->pubkey, chan->ed25519_identity.pubkey,
+                   ED25519_PUBKEY_LEN))
+      return 0;
+  }
+  return 1;
+}
+
 /**
- * Find channel by digest of the remote endpoint
+ * Find channel by RSA/Ed25519 identity of of the remote endpoint
+ *
+ * This function looks up a channel by the digest of its remote endpoint's RSA
+ * identity key.  If <b>ed_id</b> is provided and nonzero, only a channel
+ * matching the <b>ed_id</b> will be returned.
  *
- * This function looks up a channel by the digest of its remote endpoint in
- * the channel digest map.  It's possible that more than one channel to a
- * given endpoint exists.  Use channel_next_with_digest() to walk the list.
+ * It's possible that more than one channel to a given endpoint exists.  Use
+ * channel_next_with_rsa_identity() to walk the list of channels; make sure
+ * to test for Ed25519 identity match too (as appropriate)
  */
-
 channel_t *
-channel_find_by_remote_digest(const char *identity_digest)
+channel_find_by_remote_identity(const char *rsa_id_digest,
+                                const ed25519_public_key_t *ed_id)
 {
   channel_t *rv = NULL;
   channel_idmap_entry_t *ent, search;
 
-  tor_assert(identity_digest);
+  tor_assert(rsa_id_digest); /* For now, we require that every channel have
+                              * an RSA identity, and that every lookup
+                              * contain an RSA identity */
+  if (ed_id && ed25519_public_key_is_zero(ed_id)) {
+    /* Treat zero as meaning "We don't care about the presence or absence of
+     * an Ed key", not "There must be no Ed key". */
+    ed_id = NULL;
+  }
 
-  memcpy(search.digest, identity_digest, DIGEST_LEN);
+  memcpy(search.digest, rsa_id_digest, DIGEST_LEN);
   ent = HT_FIND(channel_idmap, &channel_identity_map, &search);
   if (ent) {
     rv = TOR_LIST_FIRST(&ent->channel_list);
   }
+  while (rv && ! channel_remote_identity_matches(rv, rsa_id_digest, ed_id)) {
+    rv = channel_next_with_rsa_identity(rv);
+  }
 
   return rv;
 }
@@ -766,7 +830,7 @@ channel_find_by_remote_digest(const char *identity_digest)
  */
 
 channel_t *
-channel_next_with_digest(channel_t *chan)
+channel_next_with_rsa_identity(channel_t *chan)
 {
   tor_assert(chan);
 
@@ -774,6 +838,83 @@ channel_next_with_digest(channel_t *chan)
 }
 
 /**
+ * Relays run this once an hour to look over our list of channels to other
+ * relays. It prints out some statistics if there are multiple connections
+ * to many relays.
+ *
+ * This function is similar to connection_or_set_bad_connections(),
+ * and probably could be adapted to replace it, if it was modified to actually
+ * take action on any of these connections.
+ */
+void
+channel_check_for_duplicates(void)
+{
+  channel_idmap_entry_t **iter;
+  channel_t *chan;
+  int total_relay_connections = 0, total_relays = 0, total_canonical = 0;
+  int total_half_canonical = 0;
+  int total_gt_one_connection = 0, total_gt_two_connections = 0;
+  int total_gt_four_connections = 0;
+
+  HT_FOREACH(iter, channel_idmap, &channel_identity_map) {
+    int connections_to_relay = 0;
+
+    /* Only consider relay connections */
+    if (!connection_or_digest_is_known_relay((char*)(*iter)->digest))
+      continue;
+
+    total_relays++;
+
+    for (chan = TOR_LIST_FIRST(&(*iter)->channel_list); chan;
+        chan = channel_next_with_rsa_identity(chan)) {
+
+      if (CHANNEL_CONDEMNED(chan) || !CHANNEL_IS_OPEN(chan))
+        continue;
+
+      connections_to_relay++;
+      total_relay_connections++;
+
+      if (chan->is_canonical(chan, 0)) total_canonical++;
+
+      if (!chan->is_canonical_to_peer && chan->is_canonical(chan, 0)
+          && chan->is_canonical(chan, 1)) {
+        total_half_canonical++;
+      }
+    }
+
+    if (connections_to_relay > 1) total_gt_one_connection++;
+    if (connections_to_relay > 2) total_gt_two_connections++;
+    if (connections_to_relay > 4) total_gt_four_connections++;
+  }
+
+#define MIN_RELAY_CONNECTIONS_TO_WARN 5
+
+  /* If we average 1.5 or more connections per relay, something is wrong */
+  if (total_relays > MIN_RELAY_CONNECTIONS_TO_WARN &&
+          total_relay_connections >= 1.5*total_relays) {
+    log_notice(LD_OR,
+        "Your relay has a very large number of connections to other relays. "
+        "Is your outbound address the same as your relay address? "
+        "Found %d connections to %d relays. Found %d current canonical "
+        "connections, in %d of which we were a non-canonical peer. "
+        "%d relays had more than 1 connection, %d had more than 2, and "
+        "%d had more than 4 connections.",
+        total_relay_connections, total_relays, total_canonical,
+        total_half_canonical, total_gt_one_connection,
+        total_gt_two_connections, total_gt_four_connections);
+  } else {
+    log_info(LD_OR, "Performed connection pruning. "
+        "Found %d connections to %d relays. Found %d current canonical "
+        "connections, in %d of which we were a non-canonical peer. "
+        "%d relays had more than 1 connection, %d had more than 2, and "
+        "%d had more than 4 connections.",
+        total_relay_connections, total_relays, total_canonical,
+        total_half_canonical, total_gt_one_connection,
+        total_gt_two_connections, total_gt_four_connections);
+  }
+}
+
+/**
  * Initialize a channel
  *
  * This function should be called by subclasses to set up some per-channel
@@ -787,7 +928,7 @@ channel_init(channel_t *chan)
   tor_assert(chan);
 
   /* Assign an ID and bump the counter */
-  chan->global_identifier = n_channels_allocated++;
+  chan->global_identifier = ++n_channels_allocated;
 
   /* Init timestamp */
   chan->timestamp_last_had_circuits = time(NULL);
@@ -826,7 +967,7 @@ channel_init_listener(channel_listener_t *chan_l)
   tor_assert(chan_l);
 
   /* Assign an ID and bump the counter */
-  chan_l->global_identifier = n_channels_allocated++;
+  chan_l->global_identifier = ++n_channels_allocated;
 
   /* Timestamp it */
   channel_listener_timestamp_created(chan_l);
@@ -863,6 +1004,11 @@ channel_free(channel_t *chan)
     circuitmux_set_policy(chan->cmux, NULL);
   }
 
+  /* Remove all timers and associated handle entries now */
+  timer_free(chan->padding_timer);
+  channel_handle_free(chan->timer_handle);
+  channel_handles_clear(chan);
+
   /* Call a free method if there is one */
   if (chan->free_fn) chan->free_fn(chan);
 
@@ -941,6 +1087,11 @@ channel_force_free(channel_t *chan)
     circuitmux_set_policy(chan->cmux, NULL);
   }
 
+  /* Remove all timers and associated handle entries now */
+  timer_free(chan->padding_timer);
+  channel_handle_free(chan->timer_handle);
+  channel_handles_clear(chan);
+
   /* Call a free method if there is one */
   if (chan->free_fn) chan->free_fn(chan);
 
@@ -1433,10 +1584,10 @@ channel_clear_identity_digest(channel_t *chan)
  * This function sets the identity digest of the remote endpoint for a
  * channel; this is intended for use by the lower layer.
  */
-
 void
 channel_set_identity_digest(channel_t *chan,
-                            const char *identity_digest)
+                            const char *identity_digest,
+                            const ed25519_public_key_t *ed_identity)
 {
   int was_in_digest_map, should_be_in_digest_map, state_not_in_map;
 
@@ -1475,6 +1626,11 @@ channel_set_identity_digest(channel_t *chan,
     memset(chan->identity_digest, 0,
            sizeof(chan->identity_digest));
   }
+  if (ed_identity) {
+    memcpy(&chan->ed25519_identity, ed_identity, sizeof(*ed_identity));
+  } else {
+    memset(&chan->ed25519_identity, 0, sizeof(*ed_identity));
+  }
 
   /* Put it in the digest map if we should */
   if (should_be_in_digest_map)
@@ -1738,7 +1894,7 @@ channel_get_cell_queue_entry_size(channel_t *chan, cell_queue_entry_t *q)
       rv = get_cell_network_size(chan->wide_circ_ids);
       break;
     default:
-      tor_assert(1);
+      tor_assert_nonfatal_unreached_once();
   }
 
   return rv;
@@ -1838,45 +1994,58 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q)
   }
 }
 
-/**
- * Write a cell to a channel
+/** Write a generic cell type to a channel
  *
- * Write a fixed-length cell to a channel using the write_cell() method.
- * This is equivalent to the pre-channels connection_or_write_cell_to_buf();
- * it is called by the transport-independent code to deliver a cell to a
- * channel for transmission.
+ * Write a generic cell to a channel. It is called by channel_write_cell(),
+ * channel_write_var_cell() and channel_write_packed_cell() in order to reduce
+ * code duplication. Notice that it takes cell as pointer of type void,
+ * this can be dangerous because no type check is performed.
  */
 
 void
-channel_write_cell(channel_t *chan, cell_t *cell)
+channel_write_cell_generic_(channel_t *chan, const char *cell_type,
+                            void *cell, cell_queue_entry_t *q)
 {
-  cell_queue_entry_t q;
 
   tor_assert(chan);
   tor_assert(cell);
 
   if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding cell_t %p on closing channel %p with "
-              "global ID "U64_FORMAT, cell, chan,
+    log_debug(LD_CHANNEL, "Discarding %c %p on closing channel %p with "
+              "global ID "U64_FORMAT, *cell_type, cell, chan,
               U64_PRINTF_ARG(chan->global_identifier));
     tor_free(cell);
     return;
   }
-
   log_debug(LD_CHANNEL,
-            "Writing cell_t %p to channel %p with global ID "
-            U64_FORMAT,
+            "Writing %c %p to channel %p with global ID "
+            U64_FORMAT, *cell_type,
             cell, chan, U64_PRINTF_ARG(chan->global_identifier));
 
-  q.type = CELL_QUEUE_FIXED;
-  q.u.fixed.cell = cell;
-  channel_write_cell_queue_entry(chan, &q);
-
+  channel_write_cell_queue_entry(chan, q);
   /* Update the queue size estimate */
   channel_update_xmit_queue_size(chan);
 }
 
 /**
+ * Write a cell to a channel
+ *
+ * Write a fixed-length cell to a channel using the write_cell() method.
+ * This is equivalent to the pre-channels connection_or_write_cell_to_buf();
+ * it is called by the transport-independent code to deliver a cell to a
+ * channel for transmission.
+ */
+
+void
+channel_write_cell(channel_t *chan, cell_t *cell)
+{
+  cell_queue_entry_t q;
+  q.type = CELL_QUEUE_FIXED;
+  q.u.fixed.cell = cell;
+  channel_write_cell_generic_(chan, "cell_t", cell, &q);
+}
+
+/**
  * Write a packed cell to a channel
  *
  * Write a packed cell to a channel using the write_cell() method.  This is
@@ -1888,30 +2057,9 @@ void
 channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell)
 {
   cell_queue_entry_t q;
-
-  tor_assert(chan);
-  tor_assert(packed_cell);
-
-  if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding packed_cell_t %p on closing channel %p "
-              "with global ID "U64_FORMAT, packed_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    packed_cell_free(packed_cell);
-    return;
-  }
-
-  log_debug(LD_CHANNEL,
-            "Writing packed_cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            packed_cell, chan,
-            U64_PRINTF_ARG(chan->global_identifier));
-
   q.type = CELL_QUEUE_PACKED;
   q.u.packed.packed_cell = packed_cell;
-  channel_write_cell_queue_entry(chan, &q);
-
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
+  channel_write_cell_generic_(chan, "packed_cell_t", packed_cell, &q);
 }
 
 /**
@@ -1927,30 +2075,9 @@ void
 channel_write_var_cell(channel_t *chan, var_cell_t *var_cell)
 {
   cell_queue_entry_t q;
-
-  tor_assert(chan);
-  tor_assert(var_cell);
-
-  if (CHANNEL_IS_CLOSING(chan)) {
-    log_debug(LD_CHANNEL, "Discarding var_cell_t %p on closing channel %p "
-              "with global ID "U64_FORMAT, var_cell, chan,
-              U64_PRINTF_ARG(chan->global_identifier));
-    var_cell_free(var_cell);
-    return;
-  }
-
-  log_debug(LD_CHANNEL,
-            "Writing var_cell_t %p to channel %p with global ID "
-            U64_FORMAT,
-            var_cell, chan,
-            U64_PRINTF_ARG(chan->global_identifier));
-
   q.type = CELL_QUEUE_VAR;
   q.u.var.var_cell = var_cell;
-  channel_write_cell_queue_entry(chan, &q);
-
-  /* Update the queue size estimate */
-  channel_update_xmit_queue_size(chan);
+  channel_write_cell_generic_(chan, "var_cell_t", var_cell, &q);
 }
 
 /**
@@ -2307,121 +2434,120 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan,
       free_q = 0;
       handed_off = 0;
 
-      if (1) {
-        /* Figure out how big it is for statistical purposes */
-        cell_size = channel_get_cell_queue_entry_size(chan, q);
-        /*
-         * Okay, we have a good queue entry, try to give it to the lower
-         * layer.
-         */
-        switch (q->type) {
-          case CELL_QUEUE_FIXED:
-            if (q->u.fixed.cell) {
-              if (chan->write_cell(chan,
-                    q->u.fixed.cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_FIXED "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-         case CELL_QUEUE_PACKED:
-            if (q->u.packed.packed_cell) {
-              if (chan->write_packed_cell(chan,
-                    q->u.packed.packed_cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_PACKED "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-         case CELL_QUEUE_VAR:
-            if (q->u.var.var_cell) {
-              if (chan->write_var_cell(chan,
-                    q->u.var.var_cell)) {
-                ++flushed;
-                channel_timestamp_xmit(chan);
-                ++(chan->n_cells_xmitted);
-                chan->n_bytes_xmitted += cell_size;
-                free_q = 1;
-                handed_off = 1;
-              }
-              /* Else couldn't write it; leave it on the queue */
-            } else {
-              /* This shouldn't happen */
-              log_info(LD_CHANNEL,
-                       "Saw broken cell queue entry of type CELL_QUEUE_VAR "
-                       "with no cell on channel %p "
-                       "(global ID " U64_FORMAT ").",
-                       chan, U64_PRINTF_ARG(chan->global_identifier));
-              /* Throw it away */
-              free_q = 1;
-              handed_off = 0;
-            }
-            break;
-          default:
-            /* Unknown type, log and free it */
-            log_info(LD_CHANNEL,
-                     "Saw an unknown cell queue entry type %d on channel %p "
-                     "(global ID " U64_FORMAT "; ignoring it."
-                     "  Someone should fix this.",
-                     q->type, chan, U64_PRINTF_ARG(chan->global_identifier));
+      /* Figure out how big it is for statistical purposes */
+      cell_size = channel_get_cell_queue_entry_size(chan, q);
+      /*
+       * Okay, we have a good queue entry, try to give it to the lower
+       * layer.
+       */
+      switch (q->type) {
+      case CELL_QUEUE_FIXED:
+        if (q->u.fixed.cell) {
+          if (chan->write_cell(chan,
+                               q->u.fixed.cell)) {
+            ++flushed;
+            channel_timestamp_xmit(chan);
+            ++(chan->n_cells_xmitted);
+            chan->n_bytes_xmitted += cell_size;
+            free_q = 1;
+            handed_off = 1;
+          }
+          /* Else couldn't write it; leave it on the queue */
+        } else {
+          /* This shouldn't happen */
+          log_info(LD_CHANNEL,
+                   "Saw broken cell queue entry of type CELL_QUEUE_FIXED "
+                   "with no cell on channel %p "
+                   "(global ID " U64_FORMAT ").",
+                   chan, U64_PRINTF_ARG(chan->global_identifier));
+          /* Throw it away */
+          free_q = 1;
+          handed_off = 0;
+        }
+        break;
+      case CELL_QUEUE_PACKED:
+        if (q->u.packed.packed_cell) {
+          if (chan->write_packed_cell(chan,
+                                      q->u.packed.packed_cell)) {
+            ++flushed;
+            channel_timestamp_xmit(chan);
+            ++(chan->n_cells_xmitted);
+            chan->n_bytes_xmitted += cell_size;
+            free_q = 1;
+            handed_off = 1;
+          }
+          /* Else couldn't write it; leave it on the queue */
+        } else {
+          /* This shouldn't happen */
+          log_info(LD_CHANNEL,
+                   "Saw broken cell queue entry of type CELL_QUEUE_PACKED "
+                   "with no cell on channel %p "
+                   "(global ID " U64_FORMAT ").",
+                   chan, U64_PRINTF_ARG(chan->global_identifier));
+          /* Throw it away */
+          free_q = 1;
+          handed_off = 0;
+        }
+        break;
+      case CELL_QUEUE_VAR:
+        if (q->u.var.var_cell) {
+          if (chan->write_var_cell(chan,
+                                   q->u.var.var_cell)) {
+            ++flushed;
+            channel_timestamp_xmit(chan);
+            ++(chan->n_cells_xmitted);
+            chan->n_bytes_xmitted += cell_size;
             free_q = 1;
-            handed_off = 0;
+            handed_off = 1;
+          }
+          /* Else couldn't write it; leave it on the queue */
+        } else {
+          /* This shouldn't happen */
+          log_info(LD_CHANNEL,
+                   "Saw broken cell queue entry of type CELL_QUEUE_VAR "
+                   "with no cell on channel %p "
+                   "(global ID " U64_FORMAT ").",
+                   chan, U64_PRINTF_ARG(chan->global_identifier));
+          /* Throw it away */
+          free_q = 1;
+          handed_off = 0;
         }
+        break;
+      default:
+        /* Unknown type, log and free it */
+        log_info(LD_CHANNEL,
+                 "Saw an unknown cell queue entry type %d on channel %p "
+                 "(global ID " U64_FORMAT "; ignoring it."
+                 "  Someone should fix this.",
+                 q->type, chan, U64_PRINTF_ARG(chan->global_identifier));
+        free_q = 1;
+        handed_off = 0;
+      }
 
+      /*
+       * if free_q is set, we used it and should remove the queue entry;
+       * we have to do the free down here so TOR_SIMPLEQ_REMOVE_HEAD isn't
+       * accessing freed memory
+       */
+      if (free_q) {
+        TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next);
         /*
-         * if free_q is set, we used it and should remove the queue entry;
-         * we have to do the free down here so TOR_SIMPLEQ_REMOVE_HEAD isn't
-         * accessing freed memory
+         * ...and we handed a cell off to the lower layer, so we should
+         * update the counters.
          */
-        if (free_q) {
-          TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next);
-          /*
-           * ...and we handed a cell off to the lower layer, so we should
-           * update the counters.
-           */
-          ++n_channel_cells_passed_to_lower_layer;
-          --n_channel_cells_in_queues;
-          n_channel_bytes_passed_to_lower_layer += cell_size;
-          n_channel_bytes_in_queues -= cell_size;
-          channel_assert_counter_consistency();
-          /* Update the channel's queue size too */
-          chan->bytes_in_queue -= cell_size;
-          /* Finally, free q */
-          cell_queue_entry_free(q, handed_off);
-          q = NULL;
-        }
+        ++n_channel_cells_passed_to_lower_layer;
+        --n_channel_cells_in_queues;
+        n_channel_bytes_passed_to_lower_layer += cell_size;
+        n_channel_bytes_in_queues -= cell_size;
+        channel_assert_counter_consistency();
+        /* Update the channel's queue size too */
+        chan->bytes_in_queue -= cell_size;
+        /* Finally, free q */
+        cell_queue_entry_free(q, handed_off);
+        q = NULL;
+      } else {
         /* No cell removed from list, so we can't go on any further */
-        else break;
+        break;
       }
     }
   }
@@ -2567,20 +2693,10 @@ channel_do_open_actions(channel_t *chan)
   if (started_here) {
     circuit_build_times_network_is_live(get_circuit_build_times_mutable());
     rep_hist_note_connect_succeeded(chan->identity_digest, now);
-    if (entry_guard_register_connect_status(
-          chan->identity_digest, 1, 0, now) < 0) {
-      /* Close any circuits pending on this channel. We leave it in state
-       * 'open' though, because it didn't actually *fail* -- we just
-       * chose not to use it. */
-      log_debug(LD_OR,
-                "New entry guard was reachable, but closing this "
-                "connection so we can retry the earlier entry guards.");
-      close_origin_circuits = 1;
-    }
     router_set_status(chan->identity_digest, 1);
   } else {
     /* only report it to the geoip module if it's not a known router */
-    if (!router_get_by_id_digest(chan->identity_digest)) {
+    if (!connection_or_digest_is_known_relay(chan->identity_digest)) {
       if (channel_get_addr_if_possible(chan, &remote_addr)) {
         char *transport_name = NULL;
         channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
@@ -2600,6 +2716,32 @@ channel_do_open_actions(channel_t *chan)
     }
   }
 
+  /* Disable or reduce padding according to user prefs. */
+  if (chan->padding_enabled || get_options()->ConnectionPadding == 1) {
+    if (!get_options()->ConnectionPadding) {
+      /* Disable if torrc disabled */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (get_options()->Tor2webMode &&
+            !networkstatus_get_param(NULL,
+                                     CHANNELPADDING_TOR2WEB_PARAM,
+                                     CHANNELPADDING_TOR2WEB_DEFAULT, 0, 1)) {
+      /* Disable if we're using tor2web and the consensus disabled padding
+       * for tor2web */
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (rend_service_allow_non_anonymous_connection(get_options()) &&
+               !networkstatus_get_param(NULL,
+                                        CHANNELPADDING_SOS_PARAM,
+                                        CHANNELPADDING_SOS_DEFAULT, 0, 1)) {
+      /* Disable if we're using RSOS and the consensus disabled padding
+       * for RSOS*/
+      channelpadding_disable_padding_on_channel(chan);
+    } else if (get_options()->ReducedConnectionPadding) {
+      /* Padding can be forced and/or reduced by clients, regardless of if
+       * the channel supports it */
+      channelpadding_reduce_padding_on_channel(chan);
+    }
+  }
+
   circuit_n_chan_done(chan, 1, close_origin_circuits);
 }
 
@@ -3237,6 +3379,11 @@ channel_free_all(void)
   /* Geez, anything still left over just won't die ... let it leak then */
   HT_CLEAR(channel_idmap, &channel_identity_map);
 
+  /* Same with channel_gid_map */
+  log_debug(LD_CHANNEL,
+            "Freeing channel_gid_map");
+  HT_CLEAR(channel_gid_map, &channel_gid_map);
+
   log_debug(LD_CHANNEL,
             "Done cleaning up after channels");
 }
@@ -3254,9 +3401,10 @@ channel_free_all(void)
 
 channel_t *
 channel_connect(const tor_addr_t *addr, uint16_t port,
-                const char *id_digest)
+                const char *id_digest,
+                const ed25519_public_key_t *ed_id)
 {
-  return channel_tls_connect(addr, port, id_digest);
+  return channel_tls_connect(addr, port, id_digest, ed_id);
 }
 
 /**
@@ -3271,22 +3419,20 @@ channel_connect(const tor_addr_t *addr, uint16_t port,
  */
 
 int
-channel_is_better(time_t now, channel_t *a, channel_t *b,
-                  int forgive_new_connections)
+channel_is_better(channel_t *a, channel_t *b)
 {
-  int a_grace, b_grace;
   int a_is_canonical, b_is_canonical;
-  int a_has_circs, b_has_circs;
-
-  /*
-   * Do not definitively deprecate a new channel with no circuits on it
-   * until this much time has passed.
-   */
-#define NEW_CHAN_GRACE_PERIOD (15*60)
 
   tor_assert(a);
   tor_assert(b);
 
+  /* If one channel is bad for new circuits, and the other isn't,
+   * use the one that is still good. */
+  if (!channel_is_bad_for_new_circs(a) && channel_is_bad_for_new_circs(b))
+    return 1;
+  if (channel_is_bad_for_new_circs(a) && !channel_is_bad_for_new_circs(b))
+    return 0;
+
   /* Check if one is canonical and the other isn't first */
   a_is_canonical = channel_is_canonical(a);
   b_is_canonical = channel_is_canonical(b);
@@ -3294,26 +3440,31 @@ channel_is_better(time_t now, channel_t *a, channel_t *b,
   if (a_is_canonical && !b_is_canonical) return 1;
   if (!a_is_canonical && b_is_canonical) return 0;
 
+  /* Check if we suspect that one of the channels will be preferred
+   * by the peer */
+  if (a->is_canonical_to_peer && !b->is_canonical_to_peer) return 1;
+  if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0;
+
   /*
-   * Okay, if we're here they tied on canonicity. Next we check if
-   * they have any circuits, and if one does and the other doesn't,
-   * we prefer the one that does, unless we are forgiving and the
-   * one that has no circuits is in its grace period.
+   * Okay, if we're here they tied on canonicity, the prefer the older
+   * connection, so that the adversary can't create a new connection
+   * and try to switch us over to it (which will leak information
+   * about long-lived circuits). Additionally, switching connections
+   * too often makes us more vulnerable to attacks like Torscan and
+   * passive netflow-based equivalents.
+   *
+   * Connections will still only live for at most a week, due to
+   * the check in connection_or_group_set_badness() against
+   * TIME_BEFORE_OR_CONN_IS_TOO_OLD, which marks old connections as
+   * unusable for new circuits after 1 week. That check sets
+   * is_bad_for_new_circs, which is checked in channel_get_for_extend().
+   *
+   * We check channel_is_bad_for_new_circs() above here anyway, for safety.
    */
+  if (channel_when_created(a) < channel_when_created(b)) return 1;
+  else if (channel_when_created(a) > channel_when_created(b)) return 0;
 
-  a_has_circs = (channel_num_circuits(a) > 0);
-  b_has_circs = (channel_num_circuits(b) > 0);
-  a_grace = (forgive_new_connections &&
-             (now < channel_when_created(a) + NEW_CHAN_GRACE_PERIOD));
-  b_grace = (forgive_new_connections &&
-             (now < channel_when_created(b) + NEW_CHAN_GRACE_PERIOD));
-
-  if (a_has_circs && !b_has_circs && !b_grace) return 1;
-  if (!a_has_circs && b_has_circs && !a_grace) return 0;
-
-  /* They tied on circuits too; just prefer whichever is newer */
-
-  if (channel_when_created(a) > channel_when_created(b)) return 1;
+  if (channel_num_circuits(a) > channel_num_circuits(b)) return 1;
   else return 0;
 }
 
@@ -3329,7 +3480,8 @@ channel_is_better(time_t now, channel_t *a, channel_t *b,
  */
 
 channel_t *
-channel_get_for_extend(const char *digest,
+channel_get_for_extend(const char *rsa_id_digest,
+                       const ed25519_public_key_t *ed_id,
                        const tor_addr_t *target_addr,
                        const char **msg_out,
                        int *launch_out)
@@ -3337,19 +3489,18 @@ channel_get_for_extend(const char *digest,
   channel_t *chan, *best = NULL;
   int n_inprogress_goodaddr = 0, n_old = 0;
   int n_noncanonical = 0, n_possible = 0;
-  time_t now = approx_time();
 
   tor_assert(msg_out);
   tor_assert(launch_out);
 
-  chan = channel_find_by_remote_digest(digest);
+  chan = channel_find_by_remote_identity(rsa_id_digest, ed_id);
 
   /* Walk the list, unrefing the old one and refing the new at each
    * iteration.
    */
-  for (; chan; chan = channel_next_with_digest(chan)) {
+  for (; chan; chan = channel_next_with_rsa_identity(chan)) {
     tor_assert(tor_memeq(chan->identity_digest,
-                         digest, DIGEST_LEN));
+                         rsa_id_digest, DIGEST_LEN));
 
    if (CHANNEL_CONDEMNED(chan))
       continue;
@@ -3360,6 +3511,11 @@ channel_get_for_extend(const char *digest,
       continue;
     }
 
+    /* The Ed25519 key has to match too */
+    if (!channel_remote_identity_matches(chan, rsa_id_digest, ed_id)) {
+      continue;
+    }
+
     /* Never return a non-open connection. */
     if (!CHANNEL_IS_OPEN(chan)) {
       /* If the address matches, don't launch a new connection for this
@@ -3402,7 +3558,7 @@ channel_get_for_extend(const char *digest,
       continue;
     }
 
-    if (channel_is_better(now, chan, best, 0))
+    if (channel_is_better(chan, best))
       best = chan;
   }
 
@@ -4184,8 +4340,12 @@ channel_timestamp_active(channel_t *chan)
   time_t now = time(NULL);
 
   tor_assert(chan);
+  chan->timestamp_xfer_ms = monotime_coarse_absolute_msec();
 
   chan->timestamp_active = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  chan->next_padding_time_ms = 0;
 }
 
 /**
@@ -4268,11 +4428,14 @@ void
 channel_timestamp_recv(channel_t *chan)
 {
   time_t now = time(NULL);
-
   tor_assert(chan);
+  chan->timestamp_xfer_ms = monotime_coarse_absolute_msec();
 
   chan->timestamp_active = now;
   chan->timestamp_recv = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  chan->next_padding_time_ms = 0;
 }
 
 /**
@@ -4285,11 +4448,15 @@ void
 channel_timestamp_xmit(channel_t *chan)
 {
   time_t now = time(NULL);
-
   tor_assert(chan);
 
+  chan->timestamp_xfer_ms = monotime_coarse_absolute_msec();
+
   chan->timestamp_active = now;
   chan->timestamp_xmit = now;
+
+  /* Clear any potential netflow padding timer. We're active */
+  chan->next_padding_time_ms = 0;
 }
 
 /***************************************************************
@@ -4531,6 +4698,81 @@ channel_set_circid_type,(channel_t *chan,
   }
 }
 
+/** Helper for channel_update_bad_for_new_circs(): Perform the
+ * channel_update_bad_for_new_circs operation on all channels in <b>lst</b>,
+ * all of which MUST have the same RSA ID.  (They MAY have different
+ * Ed25519 IDs.) */
+static void
+channel_rsa_id_group_set_badness(struct channel_list_s *lst, int force)
+{
+  /*XXXX This function should really be about channels. 15056 */
+  channel_t *chan;
+
+  /* First, get a minimal list of the ed25519 identites */
+  smartlist_t *ed_identities = smartlist_new();
+  TOR_LIST_FOREACH(chan, lst, next_with_same_id) {
+    uint8_t *id_copy =
+      tor_memdup(&chan->ed25519_identity.pubkey, DIGEST256_LEN);
+    smartlist_add(ed_identities, id_copy);
+  }
+  smartlist_sort_digests256(ed_identities);
+  smartlist_uniq_digests256(ed_identities);
+
+  /* Now, for each Ed identity, build a smartlist and find the best entry on
+   * it.  */
+  smartlist_t *or_conns = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(ed_identities, const uint8_t *, ed_id) {
+    TOR_LIST_FOREACH(chan, lst, next_with_same_id) {
+      channel_tls_t *chantls = BASE_CHAN_TO_TLS(chan);
+      if (tor_memneq(ed_id, &chan->ed25519_identity.pubkey, DIGEST256_LEN))
+        continue;
+      or_connection_t *orconn = chantls->conn;
+      if (orconn) {
+        tor_assert(orconn->chan == chantls);
+        smartlist_add(or_conns, orconn);
+      }
+    }
+
+    connection_or_group_set_badness_(or_conns, force);
+    smartlist_clear(or_conns);
+  } SMARTLIST_FOREACH_END(ed_id);
+
+  /* XXXX 15056 we may want to do something special with connections that have
+   * no set Ed25519 identity! */
+
+  smartlist_free(or_conns);
+
+  SMARTLIST_FOREACH(ed_identities, uint8_t *, ed_id, tor_free(ed_id));
+  smartlist_free(ed_identities);
+}
+
+/** Go through all the channels (or if <b>digest</b> is non-NULL, just
+ * the OR connections with that digest), and set the is_bad_for_new_circs
+ * flag based on the rules in connection_or_group_set_badness() (or just
+ * always set it if <b>force</b> is true).
+ */
+void
+channel_update_bad_for_new_circs(const char *digest, int force)
+{
+  if (digest) {
+    channel_idmap_entry_t *ent;
+    channel_idmap_entry_t search;
+    memset(&search, 0, sizeof(search));
+    memcpy(search.digest, digest, DIGEST_LEN);
+    ent = HT_FIND(channel_idmap, &channel_identity_map, &search);
+    if (ent) {
+      channel_rsa_id_group_set_badness(&ent->channel_list, force);
+    }
+    return;
+  }
+
+  /* no digest; just look at everything. */
+  channel_idmap_entry_t **iter;
+  HT_FOREACH(iter, channel_idmap, &channel_identity_map) {
+    channel_rsa_id_group_set_badness(&(*iter)->channel_list, force);
+  }
+}
+
 /**
  * Update the estimated number of bytes queued to transmit for this channel,
  * and notify the scheduler.  The estimate includes both the channel queue and
diff --git a/src/or/channel.h b/src/or/channel.h
index bcd345e8d2..264743691f 100644
--- a/src/or/channel.h
+++ b/src/or/channel.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,8 @@
 
 #include "or.h"
 #include "circuitmux.h"
+#include "timers.h"
+#include "handles.h"
 
 /* Channel handler function pointer typedefs */
 typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *);
@@ -22,6 +24,17 @@ TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s);
 typedef struct chan_cell_queue chan_cell_queue_t;
 
 /**
+ * This enum is used by channelpadding to decide when to pad channels.
+ * Don't add values to it without updating the checks in
+ * channelpadding_decide_to_pad_channel().
+ */
+typedef enum {
+    CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS = 0,
+    CHANNEL_USED_FOR_FULL_CIRCS,
+    CHANNEL_USED_FOR_USER_TRAFFIC,
+} channel_usage_info_t;
+
+/**
  * Channel struct; see the channel_t typedef in or.h.  A channel is an
  * abstract interface for the OR-to-OR connection, similar to connection_or_t,
  * but without the strong coupling to the underlying TLS implementation.  They
@@ -34,11 +47,17 @@ struct channel_s {
   /** Magic number for type-checking cast macros */
   uint32_t magic;
 
+  /** List entry for hashtable for global-identifier lookup. */
+  HT_ENTRY(channel_s) gidmap_node;
+
+  /** Handle entry for handle-based lookup */
+  HANDLE_ENTRY(channel, channel_s);
+
   /** Current channel state */
   channel_state_t state;
 
   /** Globally unique ID number for a channel over the lifetime of a Tor
-   * process.
+   * process.  This may not be 0.
    */
   uint64_t global_identifier;
 
@@ -48,6 +67,61 @@ struct channel_s {
   /** has this channel ever been open? */
   unsigned int has_been_open:1;
 
+  /**
+   * This field indicates if the other side has enabled or disabled
+   * padding via either the link protocol version or
+   * channelpadding_negotiate cells.
+   *
+   * Clients can override this with ConnectionPadding in torrc to
+   * disable or force padding to relays, but relays cannot override the
+   * client's request.
+   */
+  unsigned int padding_enabled:1;
+
+  /** Cached value of our decision to pad (to avoid expensive
+   * checks during critical path statistics counting). */
+  unsigned int currently_padding:1;
+
+  /** Is there a pending netflow padding callback? */
+  unsigned int pending_padding_callback:1;
+
+  /** Is our peer likely to consider this channel canonical? */
+  unsigned int is_canonical_to_peer:1;
+
+  /** Has this channel ever been used for non-directory traffic?
+   * Used to decide what channels to pad, and when. */
+  channel_usage_info_t channel_usage;
+
+  /** When should we send a cell for netflow padding, in absolute
+   *  milliseconds since monotime system start. 0 means no padding
+   *  is scheduled. */
+  uint64_t next_padding_time_ms;
+
+  /** The callback pointer for the padding callbacks */
+  tor_timer_t *padding_timer;
+  /** The handle to this channel (to free on canceled timers) */
+  struct channel_handle_t *timer_handle;
+
+  /**
+   * These two fields specify the minimum and maximum negotiated timeout
+   * values for inactivity (send or receive) before we decide to pad a
+   * channel. These fields can be set either via a PADDING_NEGOTIATE cell,
+   * or the torrc option ReducedConnectionPadding. The consensus parameters
+   * nf_ito_low and nf_ito_high are used to ensure that padding can only be
+   * negotiated to be less frequent than what is specified in the consensus.
+   * (This is done to prevent wingnut clients from requesting excessive
+   * padding).
+   *
+   * The actual timeout value is randomly chosen between these two values
+   * as per the table in channelpadding_get_netflow_inactive_timeout_ms(),
+   * after ensuring that these values do not specify lower timeouts than
+   * the consensus parameters.
+   *
+   * If these are 0, we have not negotiated or specified custom padding
+   * times, and instead use consensus defaults. */
+  uint16_t padding_timeout_low_ms;
+  uint16_t padding_timeout_high_ms;
+
   /** Why did we close?
    */
   enum {
@@ -87,6 +161,18 @@ struct channel_s {
   time_t timestamp_created; /* Channel created */
   time_t timestamp_active; /* Any activity */
 
+  /**
+   * This is a high-resolution monotonic timestamp that marks when we
+   * believe the channel has actually sent or received data to/from
+   * the wire. Right now, it is used to determine when we should send
+   * a padding cell for channelpadding.
+   *
+   * XXX: Are we setting timestamp_xfer_ms in the right places to
+   * accurately reflect actual network data transfer? Or might this be
+   * very wrong wrt when bytes actually go on the wire?
+   */
+  uint64_t timestamp_xfer_ms;
+
   /* Methods implemented by the lower layer */
 
   /** Free a channel */
@@ -153,16 +239,32 @@ struct channel_s {
   int (*write_var_cell)(channel_t *, var_cell_t *);
 
   /**
-   * Hash of the public RSA key for the other side's identity key, or
-   * zeroes if the other side hasn't shown us a valid identity key.
+   * Hash of the public RSA key for the other side's RSA identity key -- or
+   * zeroes if we don't have an RSA identity in mind for the other side, and
+   * it hasn't shown us one.
+   *
+   * Note that this is the RSA identity that we hope the other side has -- not
+   * necessarily its true identity.  Don't believe this identity unless
+   * authentication has happened.
    */
   char identity_digest[DIGEST_LEN];
+  /**
+   * Ed25519 key for the other side of this channel -- or zeroes if we don't
+   * have an Ed25519 identity in mind for the other side, and it hasn't shown
+   * us one.
+   *
+   * Note that this is the identity that we hope the other side has -- not
+   * necessarily its true identity.  Don't believe this identity unless
+   * authentication has happened.
+   */
+  ed25519_public_key_t ed25519_identity;
+
   /** Nickname of the OR on the other side, or NULL if none. */
   char *nickname;
 
   /**
-   * Linked list of channels with the same identity digest, for the
-   * digest->channel map
+   * Linked list of channels with the same RSA identity digest, for use with
+   * the digest->channel map
    */
   TOR_LIST_ENTRY(channel_s) next_with_same_id;
 
@@ -198,8 +300,8 @@ struct channel_s {
   unsigned int is_bad_for_new_circs:1;
 
   /** True iff we have decided that the other end of this connection
-   * is a client.  Channels with this flag set should never be used
-   * to satisfy an EXTEND request.  */
+   * is a client or bridge relay.  Connections with this flag set should never
+   * be used to satisfy an EXTEND request. */
   unsigned int is_client:1;
 
   /** Set if the channel was initiated remotely (came from a listener) */
@@ -382,6 +484,9 @@ struct cell_queue_entry_s {
 STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue);
 
 STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off);
+
+void channel_write_cell_generic_(channel_t *chan, const char *cell_type,
+                                 void *cell, cell_queue_entry_t *q);
 #endif
 
 /* Channel operations for subclasses and internal use only */
@@ -424,7 +529,8 @@ void channel_mark_incoming(channel_t *chan);
 void channel_mark_outgoing(channel_t *chan);
 void channel_mark_remote(channel_t *chan);
 void channel_set_identity_digest(channel_t *chan,
-                                 const char *identity_digest);
+                                 const char *identity_digest,
+                                 const ed25519_public_key_t *ed_identity);
 void channel_set_remote_end(channel_t *chan,
                             const char *identity_digest,
                             const char *nickname);
@@ -486,27 +592,29 @@ int channel_send_destroy(circid_t circ_id, channel_t *chan,
  */
 
 channel_t * channel_connect(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest);
+                            const char *rsa_id_digest,
+                            const ed25519_public_key_t *ed_id);
 
-channel_t * channel_get_for_extend(const char *digest,
+channel_t * channel_get_for_extend(const char *rsa_id_digest,
+                                   const ed25519_public_key_t *ed_id,
                                    const tor_addr_t *target_addr,
                                    const char **msg_out,
                                    int *launch_out);
 
 /* Ask which of two channels is better for circuit-extension purposes */
-int channel_is_better(time_t now,
-                      channel_t *a, channel_t *b,
-                      int forgive_new_connections);
+int channel_is_better(channel_t *a, channel_t *b);
 
 /** Channel lookups
  */
 
 channel_t * channel_find_by_global_id(uint64_t global_identifier);
-channel_t * channel_find_by_remote_digest(const char *identity_digest);
+channel_t * channel_find_by_remote_identity(const char *rsa_id_digest,
+                                            const ed25519_public_key_t *ed_id);
 
 /** For things returned by channel_find_by_remote_digest(), walk the list.
+ * The RSA key will match for all returned elements; the Ed25519 key might not.
  */
-channel_t * channel_next_with_digest(channel_t *chan);
+channel_t * channel_next_with_rsa_identity(channel_t *chan);
 
 /*
  * Helper macros to lookup state of given channel.
@@ -578,6 +686,9 @@ void channel_listener_dump_statistics(channel_listener_t *chan_l,
                                       int severity);
 void channel_listener_dump_transport_statistics(channel_listener_t *chan_l,
                                                 int severity);
+void channel_check_for_duplicates(void);
+
+void channel_update_bad_for_new_circs(const char *digest, int force);
 
 /* Flow control queries */
 uint64_t channel_get_global_queue_estimate(void);
@@ -605,5 +716,8 @@ int packed_cell_is_destroy(channel_t *chan,
                            const packed_cell_t *packed_cell,
                            circid_t *circid_out);
 
+/* Declare the handle helpers */
+HANDLE_DECL(channel, channel_s,)
+
 #endif
 
diff --git a/src/or/channelpadding.c b/src/or/channelpadding.c
new file mode 100644
index 0000000000..ccaf5b4ec8
--- /dev/null
+++ b/src/or/channelpadding.c
@@ -0,0 +1,793 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of
+ * channelpadding_channel_to_channelinfo() */
+#define TOR_CHANNEL_INTERNAL_
+
+#include "or.h"
+#include "channel.h"
+#include "channelpadding.h"
+#include "channeltls.h"
+#include "config.h"
+#include "networkstatus.h"
+#include "connection.h"
+#include "connection_or.h"
+#include "main.h"
+#include "rephist.h"
+#include "router.h"
+#include "compat_time.h"
+#include <event2/event.h>
+#include "rendservice.h"
+
+STATIC int channelpadding_get_netflow_inactive_timeout_ms(const channel_t *);
+STATIC int channelpadding_send_disable_command(channel_t *);
+STATIC int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *);
+
+/** The total number of pending channelpadding timers */
+static uint64_t total_timers_pending;
+
+/** These are cached consensus parameters for netflow */
+/** The timeout lower bound that is allowed before sending padding */
+static int consensus_nf_ito_low;
+/** The timeout upper bound that is allowed before sending padding */
+static int consensus_nf_ito_high;
+/** The timeout lower bound that is allowed before sending reduced padding */
+static int consensus_nf_ito_low_reduced;
+/** The timeout upper bound that is allowed before sending reduced padding */
+static int consensus_nf_ito_high_reduced;
+/** The connection timeout between relays */
+static int consensus_nf_conntimeout_relays;
+/** The connection timeout for client connections */
+static int consensus_nf_conntimeout_clients;
+/** Should we pad before circuits are actually used for client data? */
+static int consensus_nf_pad_before_usage;
+/** Should we pad relay-to-relay connections? */
+static int consensus_nf_pad_relays;
+/** Should we pad tor2web connections? */
+static int consensus_nf_pad_tor2web;
+/** Should we pad rosos connections? */
+static int consensus_nf_pad_single_onion;
+
+#define TOR_MSEC_PER_SEC 1000
+#define TOR_USEC_PER_MSEC 1000
+
+/**
+ * How often do we get called by the connection housekeeping (ie: once
+ * per second) */
+#define TOR_HOUSEKEEPING_CALLBACK_MSEC 1000
+/**
+ * Additional extra time buffer on the housekeeping callback, since
+ * it can be delayed. This extra slack is used to decide if we should
+ * schedule a timer or wait for the next callback. */
+#define TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC 100
+
+/**
+ * This macro tells us if either end of the channel is connected to a client.
+ * (If we're not a server, we're definitely a client. If the channel thinks
+ *  its a client, use that. Then finally verify in the consensus).
+ */
+#define CHANNEL_IS_CLIENT(chan, options) \
+  (!public_server_mode((options)) || (chan)->is_client || \
+      !connection_or_digest_is_known_relay((chan)->identity_digest))
+
+/**
+ * This function is called to update cached consensus parameters every time
+ * there is a consensus update. This allows us to move the consensus param
+ * search off of the critical path, so it does not need to be evaluated
+ * for every single connection, every second.
+ */
+void
+channelpadding_new_consensus_params(networkstatus_t *ns)
+{
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW 1500
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH 9500
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN 0
+#define DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX 60000
+  consensus_nf_ito_low = networkstatus_get_param(ns, "nf_ito_low",
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_LOW,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MIN,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX);
+  consensus_nf_ito_high = networkstatus_get_param(ns, "nf_ito_high",
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH,
+      consensus_nf_ito_low,
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX);
+
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW 9000
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH 14000
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN 0
+#define DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX 60000
+  consensus_nf_ito_low_reduced =
+    networkstatus_get_param(ns, "nf_ito_low_reduced",
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_LOW,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MIN,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX);
+
+  consensus_nf_ito_high_reduced =
+    networkstatus_get_param(ns, "nf_ito_high_reduced",
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_HIGH,
+        consensus_nf_ito_low_reduced,
+        DFLT_NETFLOW_REDUCED_KEEPALIVE_MAX);
+
+#define CONNTIMEOUT_RELAYS_DFLT (60*60) // 1 hour
+#define CONNTIMEOUT_RELAYS_MIN 60
+#define CONNTIMEOUT_RELAYS_MAX (7*24*60*60) // 1 week
+  consensus_nf_conntimeout_relays =
+    networkstatus_get_param(ns, "nf_conntimeout_relays",
+        CONNTIMEOUT_RELAYS_DFLT,
+        CONNTIMEOUT_RELAYS_MIN,
+        CONNTIMEOUT_RELAYS_MAX);
+
+#define CIRCTIMEOUT_CLIENTS_DFLT (30*60) // 30 minutes
+#define CIRCTIMEOUT_CLIENTS_MIN 60
+#define CIRCTIMEOUT_CLIENTS_MAX (24*60*60) // 24 hours
+  consensus_nf_conntimeout_clients =
+    networkstatus_get_param(ns, "nf_conntimeout_clients",
+        CIRCTIMEOUT_CLIENTS_DFLT,
+        CIRCTIMEOUT_CLIENTS_MIN,
+        CIRCTIMEOUT_CLIENTS_MAX);
+
+  consensus_nf_pad_before_usage =
+    networkstatus_get_param(ns, "nf_pad_before_usage", 1, 0, 1);
+
+  consensus_nf_pad_relays =
+    networkstatus_get_param(ns, "nf_pad_relays", 0, 0, 1);
+
+  consensus_nf_pad_tor2web =
+    networkstatus_get_param(ns,
+                            CHANNELPADDING_TOR2WEB_PARAM,
+                            CHANNELPADDING_TOR2WEB_DEFAULT, 0, 1);
+
+  consensus_nf_pad_single_onion =
+    networkstatus_get_param(ns,
+                            CHANNELPADDING_SOS_PARAM,
+                            CHANNELPADDING_SOS_DEFAULT, 0, 1);
+}
+
+/**
+ * Get a random netflow inactive timeout keepalive period in milliseconds,
+ * the range for which is determined by consensus parameters, negotiation,
+ * configuration, or default values. The consensus parameters enforce the
+ * minimum possible value, to avoid excessively frequent padding.
+ *
+ * The ranges for this value were chosen to be low enough to ensure that
+ * routers do not emit a new netflow record for a connection due to it
+ * being idle.
+ *
+ * Specific timeout values for major routers are listed in Proposal 251.
+ * No major router appeared capable of setting an inactive timeout below 10
+ * seconds, so we set the defaults below that value, since we can always
+ * scale back if it ends up being too much padding.
+ *
+ * Returns the next timeout period (in milliseconds) after which we should
+ * send a padding packet, or 0 if padding is disabled.
+ */
+STATIC int
+channelpadding_get_netflow_inactive_timeout_ms(const channel_t *chan)
+{
+  int low_timeout = consensus_nf_ito_low;
+  int high_timeout = consensus_nf_ito_high;
+  int X1, X2;
+
+  if (low_timeout == 0 && low_timeout == high_timeout)
+    return 0; // No padding
+
+  /* If we have negotiated different timeout values, use those, but
+   * don't allow them to be lower than the consensus ones */
+  if (chan->padding_timeout_low_ms && chan->padding_timeout_high_ms) {
+    low_timeout = MAX(low_timeout, chan->padding_timeout_low_ms);
+    high_timeout = MAX(high_timeout, chan->padding_timeout_high_ms);
+  }
+
+  if (low_timeout == high_timeout)
+    return low_timeout; // No randomization
+
+  /*
+   * This MAX() hack is here because we apply the timeout on both the client
+   * and the server. This creates the situation where the total time before
+   * sending a packet in either direction is actually
+   * min(client_timeout,server_timeout).
+   *
+   * If X is a random variable uniform from 0..R-1 (where R=high-low),
+   * then Y=max(X,X) has Prob(Y == i) = (2.0*i + 1)/(R*R).
+   *
+   * If we create a third random variable Z=min(Y,Y), then it turns out that
+   * Exp[Z] ~= Exp[X]. Here's a table:
+   *
+   *    R     Exp[X]    Exp[Z]    Exp[min(X,X)]   Exp[max(X,X)]
+   *  2000     999.5    1066        666.2           1332.8
+   *  3000    1499.5    1599.5      999.5           1999.5
+   *  5000    2499.5    2666       1666.2           3332.8
+   *  6000    2999.5    3199.5     1999.5           3999.5
+   *  7000    3499.5    3732.8     2332.8           4666.2
+   *  8000    3999.5    4266.2     2666.2           5332.8
+   *  10000   4999.5    5328       3332.8           6666.2
+   *  15000   7499.5    7995       4999.5           9999.5
+   *  20000   9900.5    10661      6666.2           13332.8
+   *
+   * In other words, this hack makes it so that when both the client and
+   * the guard are sending this padding, then the averages work out closer
+   * to the midpoint of the range, making the overhead easier to tune.
+   * If only one endpoint is padding (for example: if the relay does not
+   * support padding, but the client has set ConnectionPadding 1; or
+   * if the relay does support padding, but the client has set
+   * ReducedConnectionPadding 1), then the defense will still prevent
+   * record splitting, but with less overhead than the midpoint
+   * (as seen by the Exp[max(X,X)] column).
+   *
+   * To calculate average padding packet frequency (and thus overhead),
+   * index into the table by picking a row based on R = high-low. Then,
+   * use the appropriate column (Exp[Z] for two-sided padding, and
+   * Exp[max(X,X)] for one-sided padding). Finally, take this value
+   * and add it to the low timeout value. This value is the average
+   * frequency which padding packets will be sent.
+   */
+
+  X1 = crypto_rand_int(high_timeout - low_timeout);
+  X2 = crypto_rand_int(high_timeout - low_timeout);
+  return low_timeout + MAX(X1, X2);
+}
+
+/**
+ * Update this channel's padding settings based on the PADDING_NEGOTIATE
+ * contents.
+ *
+ * Returns -1 on error; 1 on success.
+ */
+int
+channelpadding_update_padding_for_channel(channel_t *chan,
+                const channelpadding_negotiate_t *pad_vars)
+{
+  if (pad_vars->version != 0) {
+    static ratelim_t version_limit = RATELIM_INIT(600);
+
+    log_fn_ratelim(&version_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL,
+           "Got a PADDING_NEGOTIATE cell with an unknown version. Ignoring.");
+    return -1;
+  }
+
+  // We should not allow malicious relays to disable or reduce padding for
+  // us as clients. In fact, we should only accept this cell at all if we're
+  // operating as a relay. Bridges should not accept it from relays, either
+  // (only from their clients).
+  if ((get_options()->BridgeRelay &&
+       connection_or_digest_is_known_relay(chan->identity_digest)) ||
+      !get_options()->ORPort_set) {
+    static ratelim_t relay_limit = RATELIM_INIT(600);
+
+    log_fn_ratelim(&relay_limit,LOG_PROTOCOL_WARN,LD_PROTOCOL,
+           "Got a PADDING_NEGOTIATE from relay at %s (%s). "
+           "This should not happen.",
+           chan->get_remote_descr(chan, 0),
+           hex_str(chan->identity_digest, DIGEST_LEN));
+    return -1;
+  }
+
+  chan->padding_enabled = (pad_vars->command == CHANNELPADDING_COMMAND_START);
+
+  /* Min must not be lower than the current consensus parameter
+     nf_ito_low. */
+  chan->padding_timeout_low_ms = MAX(consensus_nf_ito_low,
+                                     pad_vars->ito_low_ms);
+
+  /* Max must not be lower than ito_low_ms */
+  chan->padding_timeout_high_ms = MAX(chan->padding_timeout_low_ms,
+                                      pad_vars->ito_high_ms);
+
+  log_fn(LOG_INFO,LD_OR,
+         "Negotiated padding=%d, lo=%d, hi=%d on "U64_FORMAT,
+         chan->padding_enabled, chan->padding_timeout_low_ms,
+         chan->padding_timeout_high_ms,
+         U64_PRINTF_ARG(chan->global_identifier));
+
+  return 1;
+}
+
+/**
+ * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side not
+ * to send padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+STATIC int
+channelpadding_send_disable_command(channel_t *chan)
+{
+  channelpadding_negotiate_t disable;
+  cell_t cell;
+
+  tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >=
+             MIN_LINK_PROTO_FOR_CHANNEL_PADDING);
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&disable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP);
+
+  if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                                      &disable) < 0)
+    return -1;
+
+  if (chan->write_cell(chan, &cell) == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Sends a CELL_PADDING_NEGOTIATE on the channel to tell the other side to
+ * resume sending padding at some rate.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+int
+channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout,
+                                   uint16_t high_timeout)
+{
+  channelpadding_negotiate_t enable;
+  cell_t cell;
+
+  tor_assert(BASE_CHAN_TO_TLS(chan)->conn->link_proto >=
+             MIN_LINK_PROTO_FOR_CHANNEL_PADDING);
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&enable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&enable, CHANNELPADDING_COMMAND_START);
+  channelpadding_negotiate_set_ito_low_ms(&enable, low_timeout);
+  channelpadding_negotiate_set_ito_high_ms(&enable, high_timeout);
+
+  if (channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                                      &enable) < 0)
+    return -1;
+
+  if (chan->write_cell(chan, &cell) == 1)
+    return 0;
+  else
+    return -1;
+}
+
+/**
+ * Sends a CELL_PADDING cell on a channel if it has been idle since
+ * our callback was scheduled.
+ *
+ * This function also clears the pending padding timer and the callback
+ * flags.
+ */
+static void
+channelpadding_send_padding_cell_for_callback(channel_t *chan)
+{
+  cell_t cell;
+
+  /* Check that the channel is still valid and open */
+  if (!chan || chan->state != CHANNEL_STATE_OPEN) {
+    if (chan) chan->pending_padding_callback = 0;
+    log_fn(LOG_INFO,LD_OR,
+           "Scheduled a netflow padding cell, but connection already closed.");
+    return;
+  }
+
+  /* We should have a pending callback flag set. */
+  if (BUG(chan->pending_padding_callback == 0))
+    return;
+
+  chan->pending_padding_callback = 0;
+
+  if (!chan->next_padding_time_ms ||
+      chan->has_queued_writes(chan)) {
+    /* We must have been active before the timer fired */
+    chan->next_padding_time_ms = 0;
+    return;
+  }
+
+  {
+    uint64_t now = monotime_coarse_absolute_msec();
+
+    log_fn(LOG_INFO,LD_OR,
+        "Sending netflow keepalive on "U64_FORMAT" to %s (%s) after "
+        I64_FORMAT" ms. Delta "I64_FORMAT"ms",
+        U64_PRINTF_ARG(chan->global_identifier),
+        safe_str_client(chan->get_remote_descr(chan, 0)),
+        safe_str_client(hex_str(chan->identity_digest, DIGEST_LEN)),
+        U64_PRINTF_ARG(now - chan->timestamp_xfer_ms),
+        U64_PRINTF_ARG(now - chan->next_padding_time_ms));
+  }
+
+  /* Clear the timer */
+  chan->next_padding_time_ms = 0;
+
+  /* Send the padding cell. This will cause the channel to get a
+   * fresh timestamp_active */
+  memset(&cell, 0, sizeof(cell));
+  cell.command = CELL_PADDING;
+  chan->write_cell(chan, &cell);
+}
+
+/**
+ * tor_timer callback function for us to send padding on an idle channel.
+ *
+ * This function just obtains the channel from the callback handle, ensures
+ * it is still valid, and then hands it off to
+ * channelpadding_send_padding_cell_for_callback(), which checks if
+ * the channel is still idle before sending padding.
+ */
+static void
+channelpadding_send_padding_callback(tor_timer_t *timer, void *args,
+                                     const struct monotime_t *when)
+{
+  channel_t *chan = channel_handle_get((struct channel_handle_t*)args);
+  (void)timer; (void)when;
+
+  if (chan && CHANNEL_CAN_HANDLE_CELLS(chan)) {
+    /* Hrmm.. It might be nice to have an equivalent to assert_connection_ok
+     * for channels. Then we could get rid of the channeltls dependency */
+    tor_assert(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn)->magic ==
+               OR_CONNECTION_MAGIC);
+    assert_connection_ok(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), approx_time());
+
+    channelpadding_send_padding_cell_for_callback(chan);
+  } else {
+     log_fn(LOG_INFO,LD_OR,
+            "Channel closed while waiting for timer.");
+  }
+
+  total_timers_pending--;
+}
+
+/**
+ * Schedules a callback to send padding on a channel in_ms milliseconds from
+ * now.
+ *
+ * Returns CHANNELPADDING_WONTPAD on error, CHANNELPADDING_PADDING_SENT if we
+ * sent the packet immediately without a timer, and
+ * CHANNELPADDING_PADDING_SCHEDULED if we decided to schedule a timer.
+ */
+static channelpadding_decision_t
+channelpadding_schedule_padding(channel_t *chan, int in_ms)
+{
+  struct timeval timeout;
+  tor_assert(!chan->pending_padding_callback);
+
+  if (in_ms <= 0) {
+    chan->pending_padding_callback = 1;
+    channelpadding_send_padding_cell_for_callback(chan);
+    return CHANNELPADDING_PADDING_SENT;
+  }
+
+  timeout.tv_sec = in_ms/TOR_MSEC_PER_SEC;
+  timeout.tv_usec = (in_ms%TOR_USEC_PER_MSEC)*TOR_USEC_PER_MSEC;
+
+  if (!chan->timer_handle) {
+    chan->timer_handle = channel_handle_new(chan);
+  }
+
+  if (chan->padding_timer) {
+    timer_set_cb(chan->padding_timer,
+                 channelpadding_send_padding_callback,
+                 chan->timer_handle);
+  } else {
+    chan->padding_timer = timer_new(channelpadding_send_padding_callback,
+                                    chan->timer_handle);
+  }
+  timer_schedule(chan->padding_timer, &timeout);
+
+  rep_hist_padding_count_timers(++total_timers_pending);
+
+  chan->pending_padding_callback = 1;
+  return CHANNELPADDING_PADDING_SCHEDULED;
+}
+
+/**
+ * Calculates the number of milliseconds from now to schedule a padding cell.
+ *
+ * Returns the number of milliseconds from now (relative) to schedule the
+ * padding callback. If the padding timer is more than 1.1 seconds in the
+ * future, we return -1, to avoid scheduling excessive callbacks. If padding
+ * is disabled in the consensus, we return -2.
+ *
+ * Side-effects: Updates chan->next_padding_time_ms, storing an (absolute, not
+ * relative) millisecond representation of when we should send padding, unless
+ * other activity happens first. This side-effect allows us to avoid
+ * scheduling a libevent callback until we're within 1.1 seconds of the padding
+ * time.
+ */
+#define CHANNELPADDING_TIME_LATER -1
+#define CHANNELPADDING_TIME_DISABLED -2
+STATIC int64_t
+channelpadding_compute_time_until_pad_for_netflow(channel_t *chan)
+{
+  uint64_t long_now = monotime_coarse_absolute_msec();
+
+  if (!chan->next_padding_time_ms) {
+    /* If the below line or crypto_rand_int() shows up on a profile,
+     * we can avoid getting a timeout until we're at least nf_ito_lo
+     * from a timeout window. That will prevent us from setting timers
+     * on connections that were active up to 1.5 seconds ago.
+     * Idle connections should only call this once every 5.5s on average
+     * though, so that might be a micro-optimization for little gain. */
+    int64_t padding_timeout =
+        channelpadding_get_netflow_inactive_timeout_ms(chan);
+
+    if (!padding_timeout)
+      return CHANNELPADDING_TIME_DISABLED;
+
+    chan->next_padding_time_ms = padding_timeout
+                                 + chan->timestamp_xfer_ms;
+  }
+
+  /* If the next padding time is beyond the maximum possible consensus value,
+   * then this indicates a clock jump, so just send padding now. This is
+   * better than using monotonic time because we want to avoid the situation
+   * where we wait around forever for monotonic time to move forward after
+   * a clock jump far into the past.
+   */
+  if (chan->next_padding_time_ms > long_now +
+      DFLT_NETFLOW_INACTIVE_KEEPALIVE_MAX) {
+    tor_fragile_assert();
+    log_warn(LD_BUG,
+        "Channel padding timeout scheduled "I64_FORMAT"ms in the future. "
+        "Did the monotonic clock just jump?",
+        I64_PRINTF_ARG(chan->next_padding_time_ms - long_now));
+    return 0; /* Clock jumped: Send padding now */
+  }
+
+  /* If the timeout will expire before the next time we're called (1000ms
+     from now, plus some slack), then calculate the number of milliseconds
+     from now which we should send padding, so we can schedule a callback
+     then.
+   */
+  if (long_now +
+      (TOR_HOUSEKEEPING_CALLBACK_MSEC + TOR_HOUSEKEEPING_CALLBACK_SLACK_MSEC)
+      >= chan->next_padding_time_ms) {
+    int64_t ms_until_pad_for_netflow = chan->next_padding_time_ms -
+                                       long_now;
+    /* If the padding time is in the past, that means that libevent delayed
+     * calling the once-per-second callback due to other work taking too long.
+     * See https://bugs.torproject.org/22212 and
+     * https://bugs.torproject.org/16585. This is a systemic problem
+     * with being single-threaded, but let's emit a notice if this
+     * is long enough in the past that we might have missed a netflow window,
+     * and allowed a router to emit a netflow frame, just so we don't forget
+     * about it entirely.. */
+#define NETFLOW_MISSED_WINDOW (150000 - DFLT_NETFLOW_INACTIVE_KEEPALIVE_HIGH)
+    if (ms_until_pad_for_netflow < 0) {
+      int severity = (ms_until_pad_for_netflow < -NETFLOW_MISSED_WINDOW)
+                      ? LOG_NOTICE : LOG_INFO;
+      log_fn(severity, LD_OR,
+              "Channel padding timeout scheduled "I64_FORMAT"ms in the past. ",
+              I64_PRINTF_ARG(-ms_until_pad_for_netflow));
+      return 0; /* Clock jumped: Send padding now */
+    }
+
+    return ms_until_pad_for_netflow;
+  }
+  return CHANNELPADDING_TIME_LATER;
+}
+
+/**
+ * Returns a randomized value for channel idle timeout in seconds.
+ * The channel idle timeout governs how quickly we close a channel
+ * after its last circuit has disappeared.
+ *
+ * There are three classes of channels:
+ *  1. Client+non-canonical. These live for 3-4.5 minutes
+ *  2. relay to relay. These live for 45-75 min by default
+ *  3. Reduced padding clients. These live for 1.5-2.25 minutes.
+ *
+ * Also allows the default relay-to-relay value to be controlled by the
+ * consensus.
+ */
+unsigned int
+channelpadding_get_channel_idle_timeout(const channel_t *chan,
+                                        int is_canonical)
+{
+  const or_options_t *options = get_options();
+  unsigned int timeout;
+
+  /* Non-canonical and client channels only last for 3-4.5 min when idle */
+  if (!is_canonical || CHANNEL_IS_CLIENT(chan, options)) {
+#define CONNTIMEOUT_CLIENTS_BASE 180 // 3 to 4.5 min
+    timeout = CONNTIMEOUT_CLIENTS_BASE
+        + crypto_rand_int(CONNTIMEOUT_CLIENTS_BASE/2);
+  } else { // Canonical relay-to-relay channels
+    // 45..75min or consensus +/- 25%
+    timeout = consensus_nf_conntimeout_relays;
+    timeout = 3*timeout/4 + crypto_rand_int(timeout/2);
+  }
+
+  /* If ReducedConnectionPadding is set, we want to halve the duration of
+   * the channel idle timeout, since reducing the additional time that
+   * a channel stays open will reduce the total overhead for making
+   * new channels. This reduction in overhead/channel expense
+   * is important for mobile users. The option cannot be set by relays.
+   *
+   * We also don't reduce any values for timeout that the user explicitly
+   * set.
+   */
+  if (options->ReducedConnectionPadding
+      && !options->CircuitsAvailableTimeout) {
+    timeout /= 2;
+  }
+
+  return timeout;
+}
+
+/**
+ * This function controls how long we keep idle circuits open,
+ * and how long we build predicted circuits. This behavior is under
+ * the control of channelpadding because circuit availability is the
+ * dominant factor in channel lifespan, which influences total padding
+ * overhead.
+ *
+ * Returns a randomized number of seconds in a range from
+ * CircuitsAvailableTimeout to 2*CircuitsAvailableTimeout. This value is halved
+ * if ReducedConnectionPadding is set. The default value of
+ * CircuitsAvailableTimeout can be controlled by the consensus.
+ */
+int
+channelpadding_get_circuits_available_timeout(void)
+{
+  const or_options_t *options = get_options();
+  int timeout = options->CircuitsAvailableTimeout;
+
+  if (!timeout) {
+    timeout = consensus_nf_conntimeout_clients;
+
+    /* If ReducedConnectionPadding is set, we want to halve the duration of
+     * the channel idle timeout, since reducing the additional time that
+     * a channel stays open will reduce the total overhead for making
+     * new connections. This reduction in overhead/connection expense
+     * is important for mobile users. The option cannot be set by relays.
+     *
+     * We also don't reduce any values for timeout that the user explicitly
+     * set.
+     */
+    if (options->ReducedConnectionPadding) {
+      // half the value to 15..30min by default
+      timeout /= 2;
+    }
+  }
+
+  // 30..60min by default
+  timeout = timeout + crypto_rand_int(timeout);
+
+  return timeout;
+}
+
+/**
+ * Calling this function on a channel causes it to tell the other side
+ * not to send padding, and disables sending padding from this side as well.
+ */
+void
+channelpadding_disable_padding_on_channel(channel_t *chan)
+{
+  chan->padding_enabled = 0;
+
+  // Send cell to disable padding on the other end
+  channelpadding_send_disable_command(chan);
+}
+
+/**
+ * Calling this function on a channel causes it to tell the other side
+ * not to send padding, and reduces the rate that padding is sent from
+ * this side.
+ */
+void
+channelpadding_reduce_padding_on_channel(channel_t *chan)
+{
+  /* Padding can be forced and reduced by clients, regardless of if
+   * the channel supports it. So we check for support here before
+   * sending any commands. */
+  if (chan->padding_enabled) {
+    channelpadding_send_disable_command(chan);
+  }
+
+  chan->padding_timeout_low_ms = consensus_nf_ito_low_reduced;
+  chan->padding_timeout_high_ms = consensus_nf_ito_high_reduced;
+
+  log_fn(LOG_INFO,LD_OR,
+         "Reduced padding on channel "U64_FORMAT": lo=%d, hi=%d",
+         U64_PRINTF_ARG(chan->global_identifier),
+         chan->padding_timeout_low_ms, chan->padding_timeout_high_ms);
+}
+
+/**
+ * This function is called once per second by run_connection_housekeeping(),
+ * but only if the channel is still open, valid, and non-wedged.
+ *
+ * It decides if and when we should send a padding cell, and if needed,
+ * schedules a callback to send that cell at the appropriate time.
+ *
+ * Returns an enum that represents the current padding decision state.
+ * Return value is currently used only by unit tests.
+ */
+channelpadding_decision_t
+channelpadding_decide_to_pad_channel(channel_t *chan)
+{
+  const or_options_t *options = get_options();
+
+  /* Only pad open channels */
+  if (chan->state != CHANNEL_STATE_OPEN)
+    return CHANNELPADDING_WONTPAD;
+
+  if (chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS) {
+    if (!consensus_nf_pad_before_usage)
+      return CHANNELPADDING_WONTPAD;
+  } else if (chan->channel_usage != CHANNEL_USED_FOR_USER_TRAFFIC) {
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (chan->pending_padding_callback)
+    return CHANNELPADDING_PADDING_ALREADY_SCHEDULED;
+
+  /* Don't pad the channel if we didn't negotiate it, but still
+   * allow clients to force padding if options->ChannelPadding is
+   * explicitly set to 1.
+   */
+  if (!chan->padding_enabled && options->ConnectionPadding != 1) {
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (options->Tor2webMode && !consensus_nf_pad_tor2web) {
+    /* If the consensus just changed values, this channel may still
+     * think padding is enabled. Negotiate it off. */
+    if (chan->padding_enabled)
+      channelpadding_disable_padding_on_channel(chan);
+
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (rend_service_allow_non_anonymous_connection(options) &&
+      !consensus_nf_pad_single_onion) {
+    /* If the consensus just changed values, this channel may still
+     * think padding is enabled. Negotiate it off. */
+    if (chan->padding_enabled)
+      channelpadding_disable_padding_on_channel(chan);
+
+    return CHANNELPADDING_WONTPAD;
+  }
+
+  if (!chan->has_queued_writes(chan)) {
+    int is_client_channel = 0;
+
+    if (CHANNEL_IS_CLIENT(chan, options)) {
+       is_client_channel = 1;
+    }
+
+    /* If nf_pad_relays=1 is set in the consensus, we pad
+     * on *all* idle connections, relay-relay or relay-client.
+     * Otherwise pad only for client+bridge cons */
+    if (is_client_channel || consensus_nf_pad_relays) {
+      int64_t pad_time_ms =
+          channelpadding_compute_time_until_pad_for_netflow(chan);
+
+      if (pad_time_ms == CHANNELPADDING_TIME_DISABLED) {
+        return CHANNELPADDING_WONTPAD;
+      } else if (pad_time_ms == CHANNELPADDING_TIME_LATER) {
+        chan->currently_padding = 1;
+        return CHANNELPADDING_PADLATER;
+      } else {
+        if (BUG(pad_time_ms > INT_MAX)) {
+          pad_time_ms = INT_MAX;
+        }
+       /* We have to schedule a callback because we're called exactly once per
+        * second, but we don't want padding packets to go out exactly on an
+        * integer multiple of seconds. This callback will only be scheduled
+        * if we're within 1.1 seconds of the padding time.
+        */
+        chan->currently_padding = 1;
+        return channelpadding_schedule_padding(chan, (int)pad_time_ms);
+      }
+    } else {
+      chan->currently_padding = 0;
+      return CHANNELPADDING_WONTPAD;
+    }
+  } else {
+    return CHANNELPADDING_PADLATER;
+  }
+}
+
diff --git a/src/or/channelpadding.h b/src/or/channelpadding.h
new file mode 100644
index 0000000000..a227e27d5b
--- /dev/null
+++ b/src/or/channelpadding.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitbuild.h
+ * \brief Header file for circuitbuild.c.
+ **/
+#ifndef TOR_CHANNELPADDING_H
+#define TOR_CHANNELPADDING_H
+
+#include "channelpadding_negotiation.h"
+
+#define CHANNELPADDING_TOR2WEB_PARAM  "nf_pad_tor2web"
+#define CHANNELPADDING_TOR2WEB_DEFAULT 1
+#define CHANNELPADDING_SOS_PARAM  "nf_pad_single_onion"
+#define CHANNELPADDING_SOS_DEFAULT 1
+
+typedef enum {
+  CHANNELPADDING_WONTPAD,
+  CHANNELPADDING_PADLATER,
+  CHANNELPADDING_PADDING_SCHEDULED,
+  CHANNELPADDING_PADDING_ALREADY_SCHEDULED,
+  CHANNELPADDING_PADDING_SENT,
+} channelpadding_decision_t;
+
+channelpadding_decision_t channelpadding_decide_to_pad_channel(channel_t
+                                                               *chan);
+int channelpadding_update_padding_for_channel(channel_t *,
+                                              const channelpadding_negotiate_t
+                                              *chan);
+
+void channelpadding_disable_padding_on_channel(channel_t *chan);
+void channelpadding_reduce_padding_on_channel(channel_t *chan);
+int channelpadding_send_enable_command(channel_t *chan, uint16_t low_timeout,
+                                       uint16_t high_timeout);
+
+int channelpadding_get_circuits_available_timeout(void);
+unsigned int channelpadding_get_channel_idle_timeout(const channel_t *, int);
+void channelpadding_new_consensus_params(networkstatus_t *ns);
+
+#endif
+
diff --git a/src/or/channeltls.c b/src/or/channeltls.c
index 3a352d47fe..8fb91d412f 100644
--- a/src/or/channeltls.c
+++ b/src/or/channeltls.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,12 +49,17 @@
 #include "connection.h"
 #include "connection_or.h"
 #include "control.h"
+#include "entrynodes.h"
 #include "link_handshake.h"
 #include "relay.h"
 #include "rephist.h"
 #include "router.h"
 #include "routerlist.h"
 #include "scheduler.h"
+#include "torcert.h"
+#include "networkstatus.h"
+#include "channelpadding_negotiation.h"
+#include "channelpadding.h"
 
 /** How many CELL_PADDING cells have we received, ever? */
 uint64_t stats_n_padding_cells_processed = 0;
@@ -120,6 +125,8 @@ static void channel_tls_process_netinfo_cell(cell_t *cell,
 static int command_allowed_before_handshake(uint8_t command);
 static int enter_v3_handshake_with_cell(var_cell_t *cell,
                                         channel_tls_t *tlschan);
+static void channel_tls_process_padding_negotiate_cell(cell_t *cell,
+                                                       channel_tls_t *chan);
 
 /**
  * Do parts of channel_tls_t initialization common to channel_tls_connect()
@@ -170,7 +177,8 @@ channel_tls_common_init(channel_tls_t *tlschan)
 
 channel_t *
 channel_tls_connect(const tor_addr_t *addr, uint16_t port,
-                    const char *id_digest)
+                    const char *id_digest,
+                    const ed25519_public_key_t *ed_id)
 {
   channel_tls_t *tlschan = tor_malloc_zero(sizeof(*tlschan));
   channel_t *chan = &(tlschan->base_);
@@ -198,7 +206,7 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port,
   channel_mark_outgoing(chan);
 
   /* Set up or_connection stuff */
-  tlschan->conn = connection_or_connect(addr, port, id_digest, tlschan);
+  tlschan->conn = connection_or_connect(addr, port, id_digest, ed_id, tlschan);
   /* connection_or_connect() will fill in tlschan->conn */
   if (!(tlschan->conn)) {
     chan->reason_for_closing = CHANNEL_CLOSE_FOR_ERROR;
@@ -598,7 +606,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
         break;
       default:
         /* Something's broken in channel.c */
-        tor_assert(1);
+        tor_assert_nonfatal_unreached_once();
     }
   } else {
     strlcpy(buf, "(No connection)", sizeof(buf));
@@ -667,7 +675,7 @@ channel_tls_is_canonical_method(channel_t *chan, int req)
         break;
       default:
         /* This shouldn't happen; channel.c is broken if it does */
-        tor_assert(1);
+        tor_assert_nonfatal_unreached_once();
     }
   }
   /* else return 0 for tlschan->conn == NULL */
@@ -731,6 +739,15 @@ channel_tls_matches_target_method(channel_t *chan,
     return 0;
   }
 
+  /* real_addr is the address this connection came from.
+   * base_.addr is updated by connection_or_init_conn_from_address()
+   * to be the address in the descriptor. It may be tempting to
+   * allow either address to be allowed, but if we did so, it would
+   * enable someone who steals a relay's keys to impersonate/MITM it
+   * from anywhere on the Internet! (Because they could make long-lived
+   * TLS connections from anywhere to all relays, and wait for them to
+   * be used for extends).
+   */
   return tor_addr_eq(&(tlschan->conn->real_addr), target);
 }
 
@@ -1092,8 +1109,19 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn, conn->handshake_state, cell, 1);
 
+  /* We note that we're on the internet whenever we read a cell. This is
+   * a fast operation. */
+  entry_guards_note_internet_connectivity(get_guard_selection_info());
+  rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
+
+  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+    rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
+
   switch (cell->command) {
     case CELL_PADDING:
+      rep_hist_padding_count_read(PADDING_TYPE_CELL);
+      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+        rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
       break;
@@ -1104,6 +1132,10 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
       ++stats_n_netinfo_cells_processed;
       PROCESS_CELL(netinfo, cell, chan);
       break;
+    case CELL_PADDING_NEGOTIATE:
+      ++stats_n_netinfo_cells_processed;
+      PROCESS_CELL(padding_negotiate, cell, chan);
+      break;
     case CELL_CREATE:
     case CELL_CREATE_FAST:
     case CELL_CREATED:
@@ -1270,6 +1302,10 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
       return;
   }
 
+  /* We note that we're on the internet whenever we read a cell. This is
+   * a fast operation. */
+  entry_guards_note_internet_connectivity(get_guard_selection_info());
+
   /* Now handle the cell */
 
   switch (var_cell->command) {
@@ -1555,9 +1591,12 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
 
     /* We set this after sending the verions cell. */
     /*XXXXX symbolic const.*/
-    chan->base_.wide_circ_ids =
+    TLS_CHAN_TO_BASE(chan)->wide_circ_ids =
       chan->conn->link_proto >= MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS;
-    chan->conn->wide_circ_ids = chan->base_.wide_circ_ids;
+    chan->conn->wide_circ_ids = TLS_CHAN_TO_BASE(chan)->wide_circ_ids;
+
+    TLS_CHAN_TO_BASE(chan)->padding_enabled =
+      chan->conn->link_proto >= MIN_LINK_PROTO_FOR_CHANNEL_PADDING;
 
     if (send_certs) {
       if (connection_or_send_certs_cell(chan->conn) < 0) {
@@ -1584,6 +1623,43 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan)
 }
 
 /**
+ * Process a 'padding_negotiate' cell
+ *
+ * This function is called to handle an incoming PADDING_NEGOTIATE cell;
+ * enable or disable padding accordingly, and read and act on its timeout
+ * value contents.
+ */
+static void
+channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
+{
+  channelpadding_negotiate_t *negotiation;
+  tor_assert(cell);
+  tor_assert(chan);
+  tor_assert(chan->conn);
+
+  if (chan->conn->link_proto < MIN_LINK_PROTO_FOR_CHANNEL_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Received a PADDING_NEGOTIATE cell on v%d connection; dropping.",
+           chan->conn->link_proto);
+    return;
+  }
+
+  if (channelpadding_negotiate_parse(&negotiation, cell->payload,
+                                     CELL_PAYLOAD_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+          "Received malformed PADDING_NEGOTIATE cell on v%d connection; "
+          "dropping.", chan->conn->link_proto);
+
+    return;
+  }
+
+  channelpadding_update_padding_for_channel(TLS_CHAN_TO_BASE(chan),
+                                            negotiation);
+
+  channelpadding_negotiate_free(negotiation);
+}
+
+/**
  * Process a 'netinfo' cell
  *
  * This function is called to handle an incoming NETINFO cell; read and act
@@ -1600,6 +1676,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
+  const routerinfo_t *me = router_get_my_routerinfo();
 
   long apparent_skew = 0;
   tor_addr_t my_apparent_addr = TOR_ADDR_NULL;
@@ -1639,7 +1716,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       if (!(chan->conn->handshake_state->authenticated)) {
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
-                      authenticated_peer_id)));
+                      authenticated_rsa_peer_id)));
+        tor_assert(tor_mem_is_zero(
+                  (const char*)(chan->conn->handshake_state->
+                                authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
          * relay, and we must not use it for EXTEND requests (nor could we, as
          * there are no authenticated peer IDs) */
@@ -1650,8 +1730,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         connection_or_init_conn_from_address(chan->conn,
                   &(chan->conn->base_.addr),
                   chan->conn->base_.port,
+                  /* zero, checked above */
                   (const char*)(chan->conn->handshake_state->
-                   authenticated_peer_id),
+                                authenticated_rsa_peer_id),
+                  NULL, /* Ed25519 ID: Also checked as zero */
                   0);
       }
     }
@@ -1677,8 +1759,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
 
   if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
     tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+
+    if (!get_options()->BridgeRelay && me &&
+        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+      TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
+    }
+
   } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
     tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
+
+    if (!get_options()->BridgeRelay && me &&
+        !tor_addr_is_null(&me->ipv6_addr) &&
+        tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
+      TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
+    }
   }
 
   n_other_addrs = (uint8_t) *cp++;
@@ -1694,6 +1788,14 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_close_for_error(chan->conn, 0);
       return;
     }
+    /* A relay can connect from anywhere and be canonical, so
+     * long as it tells you from where it came. This may be a bit
+     * concerning.. Luckily we have another check in
+     * channel_tls_matches_target_method() to ensure that extends
+     * only go to the IP they ask for.
+     *
+     * XXX: Bleh. That check is not used if the connection is canonical.
+     */
     if (tor_addr_eq(&addr, &(chan->conn->real_addr))) {
       connection_or_set_canonical(chan->conn, 1);
       break;
@@ -1702,11 +1804,26 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     --n_other_addrs;
   }
 
+  if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
+      channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
+    const char *descr =
+      TLS_CHAN_TO_BASE(chan)->get_remote_descr(TLS_CHAN_TO_BASE(chan), 0);
+    log_info(LD_OR,
+             "We made a connection to a relay at %s (fp=%s) but we think "
+             "they will not consider this connection canonical. They "
+             "think we are at %s, but we think its %s.",
+             safe_str(descr),
+             safe_str(hex_str(chan->conn->identity_digest, DIGEST_LEN)),
+             safe_str(tor_addr_is_null(&my_apparent_addr) ?
+             "<none>" : fmt_and_decorate_addr(&my_apparent_addr)),
+             safe_str(fmt_addr32(me->addr)));
+  }
+
   /* Act on apparent skew. */
   /** Warn when we get a netinfo skew with at least this value. */
 #define NETINFO_NOTICE_SKEW 3600
   if (labs(apparent_skew) > NETINFO_NOTICE_SKEW &&
-      router_get_by_id_digest(chan->conn->identity_digest)) {
+      connection_or_digest_is_known_relay(chan->conn->identity_digest)) {
     int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest);
     clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL,
                        "NETINFO cell", "OR");
@@ -1748,6 +1865,41 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   assert_connection_ok(TO_CONN(chan->conn),time(NULL));
 }
 
+/** Types of certificates that we know how to parse from CERTS cells.  Each
+ * type corresponds to a different encoding format. */
+typedef enum cert_encoding_t {
+  CERT_ENCODING_UNKNOWN, /**< We don't recognize this. */
+  CERT_ENCODING_X509, /**< It's an RSA key, signed with RSA, encoded in x509.
+                   * (Actually, it might not be RSA. We test that later.) */
+  CERT_ENCODING_ED25519, /**< It's something signed with an Ed25519 key,
+                      * encoded asa a tor_cert_t.*/
+  CERT_ENCODING_RSA_CROSSCERT, /**< It's an Ed key signed with an RSA key. */
+} cert_encoding_t;
+
+/**
+ * Given one of the certificate type codes used in a CERTS cell,
+ * return the corresponding cert_encoding_t that we should use to parse
+ * the certificate.
+ */
+static cert_encoding_t
+certs_cell_typenum_to_cert_type(int typenum)
+{
+  switch (typenum) {
+  case CERTTYPE_RSA1024_ID_LINK:
+  case CERTTYPE_RSA1024_ID_ID:
+  case CERTTYPE_RSA1024_ID_AUTH:
+    return CERT_ENCODING_X509;
+  case CERTTYPE_ED_ID_SIGN:
+  case CERTTYPE_ED_SIGN_LINK:
+  case CERTTYPE_ED_SIGN_AUTH:
+    return CERT_ENCODING_ED25519;
+  case CERTTYPE_RSA1024_ID_EDID:
+    return CERT_ENCODING_RSA_CROSSCERT;
+  default:
+    return CERT_ENCODING_UNKNOWN;
+  }
+}
+
 /**
  * Process a CERTS cell from a channel.
  *
@@ -1767,14 +1919,21 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
 STATIC void
 channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
 {
-#define MAX_CERT_TYPE_WANTED OR_CERT_TYPE_AUTH_1024
-  tor_x509_cert_t *certs[MAX_CERT_TYPE_WANTED + 1];
+#define MAX_CERT_TYPE_WANTED CERTTYPE_RSA1024_ID_EDID
+  /* These arrays will be sparse, since a cert type can be at most one
+   * of ed/x509 */
+  tor_x509_cert_t *x509_certs[MAX_CERT_TYPE_WANTED + 1];
+  tor_cert_t *ed_certs[MAX_CERT_TYPE_WANTED + 1];
+  uint8_t *rsa_ed_cc_cert = NULL;
+  size_t rsa_ed_cc_cert_len = 0;
+
   int n_certs, i;
   certs_cell_t *cc = NULL;
 
   int send_netinfo = 0;
 
-  memset(certs, 0, sizeof(certs));
+  memset(x509_certs, 0, sizeof(x509_certs));
+  memset(ed_certs, 0, sizeof(ed_certs));
   tor_assert(cell);
   tor_assert(chan);
   tor_assert(chan->conn);
@@ -1818,77 +1977,149 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
 
     if (cert_type > MAX_CERT_TYPE_WANTED)
       continue;
+    const cert_encoding_t ct = certs_cell_typenum_to_cert_type(cert_type);
+    switch (ct) {
+      default:
+      case CERT_ENCODING_UNKNOWN:
+        break;
+      case CERT_ENCODING_X509: {
+        tor_x509_cert_t *x509_cert = tor_x509_cert_decode(cert_body, cert_len);
+        if (!x509_cert) {
+          log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                 "Received undecodable certificate in CERTS cell from %s:%d",
+                 safe_str(chan->conn->base_.address),
+               chan->conn->base_.port);
+        } else {
+          if (x509_certs[cert_type]) {
+            tor_x509_cert_free(x509_cert);
+            ERR("Duplicate x509 certificate");
+          } else {
+            x509_certs[cert_type] = x509_cert;
+          }
+        }
+        break;
+      }
+      case CERT_ENCODING_ED25519: {
+        tor_cert_t *ed_cert = tor_cert_parse(cert_body, cert_len);
+        if (!ed_cert) {
+          log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                 "Received undecodable Ed certificate "
+                 "in CERTS cell from %s:%d",
+                 safe_str(chan->conn->base_.address),
+               chan->conn->base_.port);
+        } else {
+          if (ed_certs[cert_type]) {
+            tor_cert_free(ed_cert);
+            ERR("Duplicate Ed25519 certificate");
+          } else {
+            ed_certs[cert_type] = ed_cert;
+          }
+        }
+        break;
+      }
 
-    tor_x509_cert_t *cert = tor_x509_cert_decode(cert_body, cert_len);
-    if (!cert) {
-      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-             "Received undecodable certificate in CERTS cell from %s:%d",
-             safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
-    } else {
-      if (certs[cert_type]) {
-        tor_x509_cert_free(cert);
-        ERR("Duplicate x509 certificate");
-      } else {
-        certs[cert_type] = cert;
+     case CERT_ENCODING_RSA_CROSSCERT: {
+        if (rsa_ed_cc_cert) {
+          ERR("Duplicate RSA->Ed25519 crosscert");
+        } else {
+          rsa_ed_cc_cert = tor_memdup(cert_body, cert_len);
+          rsa_ed_cc_cert_len = cert_len;
+        }
+        break;
       }
     }
   }
 
-  tor_x509_cert_t *id_cert = certs[OR_CERT_TYPE_ID_1024];
-  tor_x509_cert_t *auth_cert = certs[OR_CERT_TYPE_AUTH_1024];
-  tor_x509_cert_t *link_cert = certs[OR_CERT_TYPE_TLS_LINK];
+  /* Move the certificates we (might) want into the handshake_state->certs
+   * structure. */
+  tor_x509_cert_t *id_cert = x509_certs[CERTTYPE_RSA1024_ID_ID];
+  tor_x509_cert_t *auth_cert = x509_certs[CERTTYPE_RSA1024_ID_AUTH];
+  tor_x509_cert_t *link_cert = x509_certs[CERTTYPE_RSA1024_ID_LINK];
+  chan->conn->handshake_state->certs->auth_cert = auth_cert;
+  chan->conn->handshake_state->certs->link_cert = link_cert;
+  chan->conn->handshake_state->certs->id_cert = id_cert;
+  x509_certs[CERTTYPE_RSA1024_ID_ID] =
+    x509_certs[CERTTYPE_RSA1024_ID_AUTH] =
+    x509_certs[CERTTYPE_RSA1024_ID_LINK] = NULL;
+
+  tor_cert_t *ed_id_sign = ed_certs[CERTTYPE_ED_ID_SIGN];
+  tor_cert_t *ed_sign_link = ed_certs[CERTTYPE_ED_SIGN_LINK];
+  tor_cert_t *ed_sign_auth = ed_certs[CERTTYPE_ED_SIGN_AUTH];
+  chan->conn->handshake_state->certs->ed_id_sign = ed_id_sign;
+  chan->conn->handshake_state->certs->ed_sign_link = ed_sign_link;
+  chan->conn->handshake_state->certs->ed_sign_auth = ed_sign_auth;
+  ed_certs[CERTTYPE_ED_ID_SIGN] =
+    ed_certs[CERTTYPE_ED_SIGN_LINK] =
+    ed_certs[CERTTYPE_ED_SIGN_AUTH] = NULL;
+
+  chan->conn->handshake_state->certs->ed_rsa_crosscert = rsa_ed_cc_cert;
+  chan->conn->handshake_state->certs->ed_rsa_crosscert_len =
+    rsa_ed_cc_cert_len;
+  rsa_ed_cc_cert = NULL;
+
+  int severity;
+  /* Note that this warns more loudly about time and validity if we were
+   * _trying_ to connect to an authority, not necessarily if we _did_ connect
+   * to one. */
+  if (chan->conn->handshake_state->started_here &&
+      router_digest_is_trusted_dir(TLS_CHAN_TO_BASE(chan)->identity_digest))
+    severity = LOG_WARN;
+  else
+    severity = LOG_PROTOCOL_WARN;
+
+  const ed25519_public_key_t *checked_ed_id = NULL;
+  const common_digests_t *checked_rsa_id = NULL;
+  or_handshake_certs_check_both(severity,
+                                chan->conn->handshake_state->certs,
+                                chan->conn->tls,
+                                time(NULL),
+                                &checked_ed_id,
+                                &checked_rsa_id);
+
+  if (!checked_rsa_id)
+    ERR("Invalid certificate chain!");
 
   if (chan->conn->handshake_state->started_here) {
-    int severity;
-    if (! (id_cert && link_cert))
-      ERR("The certs we wanted were missing");
-    /* Okay. We should be able to check the certificates now. */
-    if (! tor_tls_cert_matches_key(chan->conn->tls, link_cert)) {
-      ERR("The link certificate didn't match the TLS public key");
-    }
-    /* Note that this warns more loudly about time and validity if we were
-    * _trying_ to connect to an authority, not necessarily if we _did_ connect
-    * to one. */
-    if (router_digest_is_trusted_dir(
-          TLS_CHAN_TO_BASE(chan)->identity_digest))
-      severity = LOG_WARN;
-    else
-      severity = LOG_PROTOCOL_WARN;
-
-    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, 0))
-      ERR("The link certificate was not valid");
-    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, 1))
-      ERR("The ID certificate was not valid");
+    /* No more information is needed. */
 
     chan->conn->handshake_state->authenticated = 1;
+    chan->conn->handshake_state->authenticated_rsa = 1;
     {
-      const common_digests_t *id_digests =
-        tor_x509_cert_get_id_digests(id_cert);
+      const common_digests_t *id_digests = checked_rsa_id;
       crypto_pk_t *identity_rcvd;
       if (!id_digests)
         ERR("Couldn't compute digests for key in ID cert");
 
       identity_rcvd = tor_tls_cert_get_key(id_cert);
-      if (!identity_rcvd)
-        ERR("Internal error: Couldn't get RSA key from ID cert.");
-      memcpy(chan->conn->handshake_state->authenticated_peer_id,
+      if (!identity_rcvd) {
+        ERR("Couldn't get RSA key from ID cert.");
+      }
+      memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id,
              id_digests->d[DIGEST_SHA1], DIGEST_LEN);
       channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd,
                 chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS);
       crypto_pk_free(identity_rcvd);
     }
 
+    if (checked_ed_id) {
+      chan->conn->handshake_state->authenticated_ed25519 = 1;
+      memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id,
+             checked_ed_id, sizeof(ed25519_public_key_t));
+    }
+
+    log_debug(LD_HANDSHAKE, "calling client_learned_peer_id from "
+              "process_certs_cell");
+
     if (connection_or_client_learned_peer_id(chan->conn,
-            chan->conn->handshake_state->authenticated_peer_id) < 0)
+                  chan->conn->handshake_state->authenticated_rsa_peer_id,
+                  checked_ed_id) < 0)
       ERR("Problem setting or checking peer id");
 
-    log_info(LD_OR,
-             "Got some good certificates from %s:%d: Authenticated it.",
-             safe_str(chan->conn->base_.address), chan->conn->base_.port);
-
-    chan->conn->handshake_state->id_cert = id_cert;
-    certs[OR_CERT_TYPE_ID_1024] = NULL;
+    log_info(LD_HANDSHAKE,
+             "Got some good certificates from %s:%d: Authenticated it with "
+             "RSA%s",
+             safe_str(chan->conn->base_.address), chan->conn->base_.port,
+             checked_ed_id ? " and Ed25519" : "");
 
     if (!public_server_mode(get_options())) {
       /* If we initiated the connection and we are not a public server, we
@@ -1897,25 +2128,14 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
       send_netinfo = 1;
     }
   } else {
-    if (! (id_cert && auth_cert))
-      ERR("The certs we wanted were missing");
-
-    /* Remember these certificates so we can check an AUTHENTICATE cell */
-    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, 1))
-      ERR("The authentication certificate was not valid");
-    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, 1))
-      ERR("The ID certificate was not valid");
-
+    /* We can't call it authenticated till we see an AUTHENTICATE cell. */
     log_info(LD_OR,
-             "Got some good certificates from %s:%d: "
+             "Got some good RSA%s certificates from %s:%d. "
              "Waiting for AUTHENTICATE.",
+             checked_ed_id ? " and Ed25519" : "",
              safe_str(chan->conn->base_.address),
              chan->conn->base_.port);
     /* XXXX check more stuff? */
-
-    chan->conn->handshake_state->id_cert = id_cert;
-    chan->conn->handshake_state->auth_cert = auth_cert;
-    certs[OR_CERT_TYPE_ID_1024] = certs[OR_CERT_TYPE_AUTH_1024] = NULL;
   }
 
   chan->conn->handshake_state->received_certs_cell = 1;
@@ -1929,9 +2149,13 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan)
   }
 
  err:
-  for (unsigned u = 0; u < ARRAY_LENGTH(certs); ++u) {
-    tor_x509_cert_free(certs[u]);
+  for (unsigned u = 0; u < ARRAY_LENGTH(x509_certs); ++u) {
+    tor_x509_cert_free(x509_certs[u]);
+  }
+  for (unsigned u = 0; u < ARRAY_LENGTH(ed_certs); ++u) {
+    tor_cert_free(ed_certs[u]);
   }
+  tor_free(rsa_ed_cc_cert);
   certs_cell_free(cc);
 #undef ERR
 }
@@ -1988,8 +2212,12 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
   /* Now see if there is an authentication type we can use */
   for (i = 0; i < n_types; ++i) {
     uint16_t authtype = auth_challenge_cell_get_methods(ac, i);
-    if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET)
-      use_type = authtype;
+    if (authchallenge_type_is_supported(authtype)) {
+      if (use_type == -1 ||
+          authchallenge_type_is_better(authtype, use_type)) {
+        use_type = authtype;
+      }
+    }
   }
 
   chan->conn->handshake_state->received_auth_challenge = 1;
@@ -2004,9 +2232,10 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
   if (use_type >= 0) {
     log_info(LD_OR,
              "Got an AUTH_CHALLENGE cell from %s:%d: Sending "
-             "authentication",
+             "authentication type %d",
              safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
+             chan->conn->base_.port,
+             use_type);
 
     if (connection_or_send_authenticate_cell(chan->conn, use_type) < 0) {
       log_warn(LD_OR,
@@ -2047,9 +2276,11 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan)
 STATIC void
 channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
 {
-  uint8_t expected[V3_AUTH_FIXED_PART_LEN+256];
+  var_cell_t *expected_cell = NULL;
   const uint8_t *auth;
   int authlen;
+  int authtype;
+  int bodylen;
 
   tor_assert(cell);
   tor_assert(chan);
@@ -2062,6 +2293,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
            safe_str(chan->conn->base_.address),                 \
            chan->conn->base_.port, (s));                        \
     connection_or_close_for_error(chan->conn, 0);               \
+    var_cell_free(expected_cell);                               \
     return;                                                     \
   } while (0)
 
@@ -2079,9 +2311,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
   }
   if (!(chan->conn->handshake_state->received_certs_cell))
     ERR("We never got a certs cell");
-  if (chan->conn->handshake_state->auth_cert == NULL)
-    ERR("We never got an authentication certificate");
-  if (chan->conn->handshake_state->id_cert == NULL)
+  if (chan->conn->handshake_state->certs->id_cert == NULL)
     ERR("We never got an identity certificate");
   if (cell->payload_len < 4)
     ERR("Cell was way too short");
@@ -2093,8 +2323,9 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
     if (4 + len > cell->payload_len)
       ERR("Authenticator was truncated");
 
-    if (type != AUTHTYPE_RSA_SHA256_TLSSECRET)
+    if (! authchallenge_type_is_supported(type))
       ERR("Authenticator type was not recognized");
+    authtype = type;
 
     auth += 4;
     authlen = len;
@@ -2103,25 +2334,55 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
   if (authlen < V3_AUTH_BODY_LEN + 1)
     ERR("Authenticator was too short");
 
-  ssize_t bodylen =
-    connection_or_compute_authenticate_cell_body(
-                        chan->conn, expected, sizeof(expected), NULL, 1);
-  if (bodylen < 0 || bodylen != V3_AUTH_FIXED_PART_LEN)
+  expected_cell = connection_or_compute_authenticate_cell_body(
+                chan->conn, authtype, NULL, NULL, 1);
+  if (! expected_cell)
     ERR("Couldn't compute expected AUTHENTICATE cell body");
 
-  if (tor_memneq(expected, auth, bodylen))
+  int sig_is_rsa;
+  if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET ||
+      authtype == AUTHTYPE_RSA_SHA256_RFC5705) {
+    bodylen = V3_AUTH_BODY_LEN;
+    sig_is_rsa = 1;
+  } else {
+    tor_assert(authtype == AUTHTYPE_ED25519_SHA256_RFC5705);
+    /* Our earlier check had better have made sure we had room
+     * for an ed25519 sig (inadvertently) */
+    tor_assert(V3_AUTH_BODY_LEN > ED25519_SIG_LEN);
+    bodylen = authlen - ED25519_SIG_LEN;
+    sig_is_rsa = 0;
+  }
+  if (expected_cell->payload_len != bodylen+4) {
+    ERR("Expected AUTHENTICATE cell body len not as expected.");
+  }
+
+  /* Length of random part. */
+  if (BUG(bodylen < 24)) {
+    // LCOV_EXCL_START
+    ERR("Bodylen is somehow less than 24, which should really be impossible");
+    // LCOV_EXCL_STOP
+  }
+
+  if (tor_memneq(expected_cell->payload+4, auth, bodylen-24))
     ERR("Some field in the AUTHENTICATE cell body was not as expected");
 
-  {
+  if (sig_is_rsa) {
+    if (chan->conn->handshake_state->certs->ed_id_sign != NULL)
+      ERR("RSA-signed AUTHENTICATE response provided with an ED25519 cert");
+
+    if (chan->conn->handshake_state->certs->auth_cert == NULL)
+      ERR("We never got an RSA authentication certificate");
+
     crypto_pk_t *pk = tor_tls_cert_get_key(
-                                   chan->conn->handshake_state->auth_cert);
+                             chan->conn->handshake_state->certs->auth_cert);
     char d[DIGEST256_LEN];
     char *signed_data;
     size_t keysize;
     int signed_len;
 
-    if (!pk)
-      ERR("Internal error: couldn't get RSA key from AUTH cert.");
+    if (! pk) {
+      ERR("Couldn't get RSA key from AUTH cert.");
+    }
     crypto_digest256(d, (char*)auth, V3_AUTH_BODY_LEN, DIGEST_SHA256);
 
     keysize = crypto_pk_keysize(pk);
@@ -2132,7 +2393,7 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
     crypto_pk_free(pk);
     if (signed_len < 0) {
       tor_free(signed_data);
-      ERR("Signature wasn't valid");
+      ERR("RSA signature wasn't valid");
     }
     if (signed_len < DIGEST256_LEN) {
       tor_free(signed_data);
@@ -2145,41 +2406,75 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan)
       ERR("Signature did not match data to be signed.");
     }
     tor_free(signed_data);
+  } else {
+    if (chan->conn->handshake_state->certs->ed_id_sign == NULL)
+      ERR("We never got an Ed25519 identity certificate.");
+    if (chan->conn->handshake_state->certs->ed_sign_auth == NULL)
+      ERR("We never got an Ed25519 authentication certificate.");
+
+    const ed25519_public_key_t *authkey =
+      &chan->conn->handshake_state->certs->ed_sign_auth->signed_key;
+    ed25519_signature_t sig;
+    tor_assert(authlen > ED25519_SIG_LEN);
+    memcpy(&sig.sig, auth + authlen - ED25519_SIG_LEN, ED25519_SIG_LEN);
+    if (ed25519_checksig(&sig, auth, authlen - ED25519_SIG_LEN, authkey)<0) {
+      ERR("Ed25519 signature wasn't valid.");
+    }
   }
 
   /* Okay, we are authenticated. */
   chan->conn->handshake_state->received_authenticate = 1;
   chan->conn->handshake_state->authenticated = 1;
+  chan->conn->handshake_state->authenticated_rsa = 1;
   chan->conn->handshake_state->digest_received_data = 0;
   {
-    crypto_pk_t *identity_rcvd =
-      tor_tls_cert_get_key(chan->conn->handshake_state->id_cert);
-    const common_digests_t *id_digests =
-      tor_x509_cert_get_id_digests(chan->conn->handshake_state->id_cert);
+    tor_x509_cert_t *id_cert = chan->conn->handshake_state->certs->id_cert;
+    crypto_pk_t *identity_rcvd = tor_tls_cert_get_key(id_cert);
+    const common_digests_t *id_digests = tor_x509_cert_get_id_digests(id_cert);
+    const ed25519_public_key_t *ed_identity_received = NULL;
+
+    if (! sig_is_rsa) {
+      chan->conn->handshake_state->authenticated_ed25519 = 1;
+      ed_identity_received =
+        &chan->conn->handshake_state->certs->ed_id_sign->signing_key;
+      memcpy(&chan->conn->handshake_state->authenticated_ed25519_peer_id,
+             ed_identity_received, sizeof(ed25519_public_key_t));
+    }
 
     /* This must exist; we checked key type when reading the cert. */
     tor_assert(id_digests);
 
-    memcpy(chan->conn->handshake_state->authenticated_peer_id,
+    memcpy(chan->conn->handshake_state->authenticated_rsa_peer_id,
            id_digests->d[DIGEST_SHA1], DIGEST_LEN);
 
     channel_set_circid_type(TLS_CHAN_TO_BASE(chan), identity_rcvd,
                chan->conn->link_proto < MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS);
     crypto_pk_free(identity_rcvd);
 
+    log_debug(LD_HANDSHAKE,
+              "Calling connection_or_init_conn_from_address for %s "
+              " from %s, with%s ed25519 id.",
+              safe_str(chan->conn->base_.address),
+              __func__,
+              ed_identity_received ? "" : "out");
+
     connection_or_init_conn_from_address(chan->conn,
                   &(chan->conn->base_.addr),
                   chan->conn->base_.port,
                   (const char*)(chan->conn->handshake_state->
-                    authenticated_peer_id),
+                    authenticated_rsa_peer_id),
+                  ed_identity_received,
                   0);
 
-    log_info(LD_OR,
-             "Got an AUTHENTICATE cell from %s:%d: Looks good.",
+    log_debug(LD_HANDSHAKE,
+             "Got an AUTHENTICATE cell from %s:%d, type %d: Looks good.",
              safe_str(chan->conn->base_.address),
-             chan->conn->base_.port);
+             chan->conn->base_.port,
+             authtype);
   }
 
+  var_cell_free(expected_cell);
+
 #undef ERR
 }
 
diff --git a/src/or/channeltls.h b/src/or/channeltls.h
index 8b5863a461..1f9a39d8a4 100644
--- a/src/or/channeltls.h
+++ b/src/or/channeltls.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,8 @@ struct channel_tls_s {
 #endif /* TOR_CHANNEL_INTERNAL_ */
 
 channel_t * channel_tls_connect(const tor_addr_t *addr, uint16_t port,
-                                const char *id_digest);
+                                const char *id_digest,
+                                const ed25519_public_key_t *ed_id);
 channel_listener_t * channel_tls_get_listener(void);
 channel_listener_t * channel_tls_start_listener(void);
 channel_t * channel_tls_handle_incoming(or_connection_t *orconn);
diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c
index 9f93e737f7..4c0bd9e455 100644
--- a/src/or/circpathbias.c
+++ b/src/or/circpathbias.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,14 @@
  * different tor nodes, in an attempt to detect attacks where
  * an attacker deliberately causes circuits to fail until the client
  * choses a path they like.
+ *
+ * This code is currently configured in a warning-only mode, though false
+ * positives appear to be rare in practice.  There is also support for
+ * disabling really bad guards, but it's quite experimental and may have bad
+ * anonymity effects.
+ *
+ * The information here is associated with the entry_guard_t object for
+ * each guard, and stored persistently in the state file.
  */
 
 #include "or.h"
@@ -43,19 +51,21 @@ static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard);
 static int
 entry_guard_inc_circ_attempt_count(entry_guard_t *guard)
 {
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
   entry_guards_changed();
 
   pathbias_measure_close_rate(guard);
 
-  if (guard->path_bias_disabled)
+  if (pb->path_bias_disabled)
     return -1;
 
   pathbias_scale_close_rates(guard);
-  guard->circ_attempts++;
+  pb->circ_attempts++;
 
-  log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)",
-           guard->circ_successes, guard->circ_attempts, guard->nickname,
-           hex_str(guard->identity, DIGEST_LEN));
+  log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+           pb->circ_successes, pb->circ_attempts,
+           entry_guard_describe(guard));
   return 0;
 }
 
@@ -505,14 +515,16 @@ pathbias_count_build_success(origin_circuit_t *circ)
     }
 
     if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
       if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) {
         circ->path_state = PATH_STATE_BUILD_SUCCEEDED;
-        guard->circ_successes++;
+        pb->circ_successes++;
         entry_guards_changed();
 
-        log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)",
-                 guard->circ_successes, guard->circ_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+        log_info(LD_CIRC, "Got success count %f/%f for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
       } else {
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
@@ -527,11 +539,11 @@ pathbias_count_build_success(origin_circuit_t *circ)
         }
       }
 
-      if (guard->circ_attempts < guard->circ_successes) {
+      if (pb->circ_attempts < pb->circ_successes) {
         log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) "
-                 "for guard %s ($%s)",
-                 guard->circ_successes, guard->circ_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+                 "for guard %s",
+                 pb->circ_successes, pb->circ_attempts,
+                 entry_guard_describe(guard));
       }
     /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
      * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here.
@@ -574,8 +586,6 @@ pathbias_count_build_success(origin_circuit_t *circ)
 void
 pathbias_count_use_attempt(origin_circuit_t *circ)
 {
-  entry_guard_t *guard;
-
   if (!pathbias_should_count(circ)) {
     return;
   }
@@ -588,19 +598,21 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
   } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) {
-    guard = entry_guard_get_by_id_digest(
+    entry_guard_t *guard = entry_guard_get_by_id_digest(
                 circ->cpath->extend_info->identity_digest);
     if (guard) {
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
       pathbias_measure_use_rate(guard);
       pathbias_scale_use_rates(guard);
-      guard->use_attempts++;
+      pb->use_attempts++;
       entry_guards_changed();
 
       log_debug(LD_CIRC,
-               "Marked circuit %d (%f/%f) as used for guard %s ($%s).",
+               "Marked circuit %d (%f/%f) as used for guard %s.",
                circ->global_identifier,
-               guard->use_successes, guard->use_attempts,
-               guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+               pb->use_successes, pb->use_attempts,
+               entry_guard_describe(guard));
     }
 
     circ->path_state = PATH_STATE_USE_ATTEMPTED;
@@ -702,22 +714,23 @@ pathbias_count_use_success(origin_circuit_t *circ)
     guard = entry_guard_get_by_id_digest(
                 circ->cpath->extend_info->identity_digest);
     if (guard) {
-      guard->use_successes++;
+      guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+      pb->use_successes++;
       entry_guards_changed();
 
-      if (guard->use_attempts < guard->use_successes) {
+      if (pb->use_attempts < pb->use_successes) {
         log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) "
-                 "for guard %s=%s",
-                 guard->use_successes, guard->use_attempts,
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+                 "for guard %s",
+                 pb->use_successes, pb->use_attempts,
+                 entry_guard_describe(guard));
       }
 
       log_debug(LD_CIRC,
-                "Marked circuit %d (%f/%f) as used successfully for guard "
-                "%s ($%s).",
-                circ->global_identifier, guard->use_successes,
-                guard->use_attempts, guard->nickname,
-                hex_str(guard->identity, DIGEST_LEN));
+                "Marked circuit %d (%f/%f) as used successfully for guard %s",
+                circ->global_identifier, pb->use_successes,
+                pb->use_attempts,
+                entry_guard_describe(guard));
     }
   }
 
@@ -1018,9 +1031,11 @@ pathbias_count_successful_close(origin_circuit_t *circ)
   }
 
   if (guard) {
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
     /* In the long run: circuit_success ~= successful_circuit_close +
      *                                     circ_failure + stream_failure */
-    guard->successful_circuits_closed++;
+    pb->successful_circuits_closed++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1057,7 +1072,9 @@ pathbias_count_collapse(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->collapsed_circuits++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->collapsed_circuits++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1090,7 +1107,9 @@ pathbias_count_use_failed(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->unusable_circuits++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->unusable_circuits++;
     entry_guards_changed();
   } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
    /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to
@@ -1133,7 +1152,9 @@ pathbias_count_timeout(origin_circuit_t *circ)
   }
 
   if (guard) {
-    guard->timeouts++;
+    guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+    pb->timeouts++;
     entry_guards_changed();
   }
 }
@@ -1165,7 +1186,7 @@ pathbias_count_circs_in_states(entry_guard_t *guard,
     if (ocirc->path_state >= from &&
         ocirc->path_state <= to &&
         pathbias_should_count(ocirc) &&
-        fast_memeq(guard->identity,
+        fast_memeq(entry_guard_get_rsa_id_digest(guard),
                    ocirc->cpath->extend_info->identity_digest,
                    DIGEST_LEN)) {
       log_debug(LD_CIRC, "Found opened circuit %d in path_state %s",
@@ -1189,7 +1210,9 @@ pathbias_count_circs_in_states(entry_guard_t *guard,
 double
 pathbias_get_close_success_count(entry_guard_t *guard)
 {
-  return guard->successful_circuits_closed +
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->successful_circuits_closed +
          pathbias_count_circs_in_states(guard,
                        PATH_STATE_BUILD_SUCCEEDED,
                        PATH_STATE_USE_SUCCEEDED);
@@ -1205,7 +1228,9 @@ pathbias_get_close_success_count(entry_guard_t *guard)
 double
 pathbias_get_use_success_count(entry_guard_t *guard)
 {
-  return guard->use_successes +
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
+
+  return pb->use_successes +
          pathbias_count_circs_in_states(guard,
                        PATH_STATE_USE_ATTEMPTED,
                        PATH_STATE_USE_SUCCEEDED);
@@ -1223,18 +1248,19 @@ static void
 pathbias_measure_use_rate(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
-  if (guard->use_attempts > pathbias_get_min_use(options)) {
+  if (pb->use_attempts > pathbias_get_min_use(options)) {
     /* Note: We rely on the < comparison here to allow us to set a 0
      * rate and disable the feature entirely. If refactoring, don't
      * change to <= */
-    if (pathbias_get_use_success_count(guard)/guard->use_attempts
+    if (pathbias_get_use_success_count(guard)/pb->use_attempts
         < pathbias_get_extreme_use_rate(options)) {
       /* Dropping is currently disabled by default. */
       if (pathbias_get_dropguards(options)) {
-        if (!guard->path_bias_disabled) {
+        if (!pb->path_bias_disabled) {
           log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry an extremely large "
+                 "Your Guard %s is failing to carry an extremely large "
                  "amount of stream on its circuits. "
                  "To avoid potential route manipulation attacks, Tor has "
                  "disabled use of this guard. "
@@ -1242,25 +1268,23 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
-          guard->path_bias_disabled = 1;
-          guard->bad_since = approx_time();
-          entry_guards_changed();
+          pb->path_bias_disabled = 1;
           return;
         }
-      } else if (!guard->path_bias_use_extreme) {
-        guard->path_bias_use_extreme = 1;
+      } else if (!pb->path_bias_use_extreme) {
+        pb->path_bias_use_extreme = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry an extremely large "
+                 "Your Guard %s is failing to carry an extremely large "
                  "amount of streams on its circuits. "
                  "This could indicate a route manipulation attack, network "
                  "overload, bad local network connectivity, or a bug. "
@@ -1268,23 +1292,23 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_use_success_count(guard)/guard->use_attempts
+    } else if (pathbias_get_use_success_count(guard)/pb->use_attempts
                < pathbias_get_notice_use_rate(options)) {
-      if (!guard->path_bias_use_noticed) {
-        guard->path_bias_use_noticed = 1;
+      if (!pb->path_bias_use_noticed) {
+        pb->path_bias_use_noticed = 1;
         log_notice(LD_CIRC,
-                 "Your Guard %s ($%s) is failing to carry more streams on its "
+                 "Your Guard %s is failing to carry more streams on its "
                  "circuits than usual. "
                  "Most likely this means the Tor network is overloaded "
                  "or your network connection is poor. "
@@ -1292,15 +1316,15 @@ pathbias_measure_use_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
+                 tor_lround(pb->use_attempts),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->circ_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
     }
@@ -1329,18 +1353,19 @@ static void
 pathbias_measure_close_rate(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
-  if (guard->circ_attempts > pathbias_get_min_circs(options)) {
+  if (pb->circ_attempts > pathbias_get_min_circs(options)) {
     /* Note: We rely on the < comparison here to allow us to set a 0
      * rate and disable the feature entirely. If refactoring, don't
      * change to <= */
-    if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    if (pathbias_get_close_success_count(guard)/pb->circ_attempts
         < pathbias_get_extreme_rate(options)) {
       /* Dropping is currently disabled by default. */
       if (pathbias_get_dropguards(options)) {
-        if (!guard->path_bias_disabled) {
+        if (!pb->path_bias_disabled) {
           log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing an extremely large "
+                 "Your Guard %s is failing an extremely large "
                  "amount of circuits. "
                  "To avoid potential route manipulation attacks, Tor has "
                  "disabled use of this guard. "
@@ -1348,25 +1373,23 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
-          guard->path_bias_disabled = 1;
-          guard->bad_since = approx_time();
-          entry_guards_changed();
+          pb->path_bias_disabled = 1;
           return;
         }
-      } else if (!guard->path_bias_extreme) {
-        guard->path_bias_extreme = 1;
+      } else if (!pb->path_bias_extreme) {
+        pb->path_bias_extreme = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing an extremely large "
+                 "Your Guard %s is failing an extremely large "
                  "amount of circuits. "
                  "This could indicate a route manipulation attack, "
                  "extreme network overload, or a bug. "
@@ -1374,23 +1397,23 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
                 < pathbias_get_warn_rate(options)) {
-      if (!guard->path_bias_warned) {
-        guard->path_bias_warned = 1;
+      if (!pb->path_bias_warned) {
+        pb->path_bias_warned = 1;
         log_warn(LD_CIRC,
-                 "Your Guard %s ($%s) is failing a very large "
+                 "Your Guard %s is failing a very large "
                  "amount of circuits. "
                  "Most likely this means the Tor network is "
                  "overloaded, but it could also mean an attack against "
@@ -1399,38 +1422,38 @@ pathbias_measure_close_rate(entry_guard_t *guard)
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
-    } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts
+    } else if (pathbias_get_close_success_count(guard)/pb->circ_attempts
                < pathbias_get_notice_rate(options)) {
-      if (!guard->path_bias_noticed) {
-        guard->path_bias_noticed = 1;
+      if (!pb->path_bias_noticed) {
+        pb->path_bias_noticed = 1;
         log_notice(LD_CIRC,
-                 "Your Guard %s ($%s) is failing more circuits than "
+                 "Your Guard %s is failing more circuits than "
                  "usual. "
                  "Most likely this means the Tor network is overloaded. "
                  "Success counts are %ld/%ld. Use counts are %ld/%ld. "
                  "%ld circuits completed, %ld were unusable, %ld collapsed, "
                  "and %ld timed out. "
                  "For reference, your timeout cutoff is %ld seconds.",
-                 guard->nickname, hex_str(guard->identity, DIGEST_LEN),
+                 entry_guard_describe(guard),
                  tor_lround(pathbias_get_close_success_count(guard)),
-                 tor_lround(guard->circ_attempts),
+                 tor_lround(pb->circ_attempts),
                  tor_lround(pathbias_get_use_success_count(guard)),
-                 tor_lround(guard->use_attempts),
-                 tor_lround(guard->circ_successes),
-                 tor_lround(guard->unusable_circuits),
-                 tor_lround(guard->collapsed_circuits),
-                 tor_lround(guard->timeouts),
+                 tor_lround(pb->use_attempts),
+                 tor_lround(pb->circ_successes),
+                 tor_lround(pb->unusable_circuits),
+                 tor_lround(pb->collapsed_circuits),
+                 tor_lround(pb->timeouts),
                  tor_lround(get_circuit_build_close_time_ms()/1000));
       }
     }
@@ -1450,9 +1473,10 @@ static void
 pathbias_scale_close_rates(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
   /* If we get a ton of circuits, just scale everything down */
-  if (guard->circ_attempts > pathbias_get_scale_threshold(options)) {
+  if (pb->circ_attempts > pathbias_get_scale_threshold(options)) {
     double scale_ratio = pathbias_get_scale_ratio(options);
     int opened_attempts = pathbias_count_circs_in_states(guard,
             PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED);
@@ -1460,38 +1484,38 @@ pathbias_scale_close_rates(entry_guard_t *guard)
                         PATH_STATE_BUILD_SUCCEEDED,
                         PATH_STATE_USE_FAILED);
     /* Verify that the counts are sane before and after scaling */
-    int counts_are_sane = (guard->circ_attempts >= guard->circ_successes);
+    int counts_are_sane = (pb->circ_attempts >= pb->circ_successes);
 
-    guard->circ_attempts -= (opened_attempts+opened_built);
-    guard->circ_successes -= opened_built;
+    pb->circ_attempts -= (opened_attempts+opened_built);
+    pb->circ_successes -= opened_built;
 
-    guard->circ_attempts *= scale_ratio;
-    guard->circ_successes *= scale_ratio;
-    guard->timeouts *= scale_ratio;
-    guard->successful_circuits_closed *= scale_ratio;
-    guard->collapsed_circuits *= scale_ratio;
-    guard->unusable_circuits *= scale_ratio;
+    pb->circ_attempts *= scale_ratio;
+    pb->circ_successes *= scale_ratio;
+    pb->timeouts *= scale_ratio;
+    pb->successful_circuits_closed *= scale_ratio;
+    pb->collapsed_circuits *= scale_ratio;
+    pb->unusable_circuits *= scale_ratio;
 
-    guard->circ_attempts += (opened_attempts+opened_built);
-    guard->circ_successes += opened_built;
+    pb->circ_attempts += (opened_attempts+opened_built);
+    pb->circ_successes += opened_built;
 
     entry_guards_changed();
 
     log_info(LD_CIRC,
              "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard "
-             "%s ($%s)",
-             guard->circ_successes, guard->successful_circuits_closed,
-             guard->circ_attempts, opened_built, opened_attempts,
-             guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+             "%s",
+             pb->circ_successes, pb->successful_circuits_closed,
+             pb->circ_attempts, opened_built, opened_attempts,
+             entry_guard_describe(guard));
 
     /* Have the counts just become invalid by this scaling attempt? */
-    if (counts_are_sane && guard->circ_attempts < guard->circ_successes) {
+    if (counts_are_sane && pb->circ_attempts < pb->circ_successes) {
       log_notice(LD_BUG,
                "Scaling has mangled pathbias counts to %f/%f (%d/%d open) "
-               "for guard %s ($%s)",
-               guard->circ_successes, guard->circ_attempts, opened_built,
-               opened_attempts, guard->nickname,
-               hex_str(guard->identity, DIGEST_LEN));
+               "for guard %s",
+               pb->circ_successes, pb->circ_attempts, opened_built,
+               opened_attempts,
+               entry_guard_describe(guard));
     }
   }
 }
@@ -1509,35 +1533,35 @@ void
 pathbias_scale_use_rates(entry_guard_t *guard)
 {
   const or_options_t *options = get_options();
+  guard_pathbias_t *pb = entry_guard_get_pathbias_state(guard);
 
   /* If we get a ton of circuits, just scale everything down */
-  if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) {
+  if (pb->use_attempts > pathbias_get_scale_use_threshold(options)) {
     double scale_ratio = pathbias_get_scale_ratio(options);
     int opened_attempts = pathbias_count_circs_in_states(guard,
             PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED);
     /* Verify that the counts are sane before and after scaling */
-    int counts_are_sane = (guard->use_attempts >= guard->use_successes);
+    int counts_are_sane = (pb->use_attempts >= pb->use_successes);
 
-    guard->use_attempts -= opened_attempts;
+    pb->use_attempts -= opened_attempts;
 
-    guard->use_attempts *= scale_ratio;
-    guard->use_successes *= scale_ratio;
+    pb->use_attempts *= scale_ratio;
+    pb->use_successes *= scale_ratio;
 
-    guard->use_attempts += opened_attempts;
+    pb->use_attempts += opened_attempts;
 
     log_info(LD_CIRC,
-           "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)",
-           guard->use_successes, guard->use_attempts, opened_attempts,
-           guard->nickname, hex_str(guard->identity, DIGEST_LEN));
+           "Scaled pathbias use counts to %f/%f (%d open) for guard %s",
+           pb->use_successes, pb->use_attempts, opened_attempts,
+           entry_guard_describe(guard));
 
     /* Have the counts just become invalid by this scaling attempt? */
-    if (counts_are_sane && guard->use_attempts < guard->use_successes) {
+    if (counts_are_sane && pb->use_attempts < pb->use_successes) {
       log_notice(LD_BUG,
                "Scaling has mangled pathbias usage counts to %f/%f "
-               "(%d open) for guard %s ($%s)",
-               guard->circ_successes, guard->circ_attempts,
-               opened_attempts, guard->nickname,
-               hex_str(guard->identity, DIGEST_LEN));
+               "(%d open) for guard %s",
+               pb->circ_successes, pb->circ_attempts,
+               opened_attempts, entry_guard_describe(guard));
     }
 
     entry_guards_changed();
diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h
index ce76689d5f..2a4c316807 100644
--- a/src/or/circpathbias.h
+++ b/src/or/circpathbias.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c
index cb9c146fb7..41ae51a3f2 100644
--- a/src/or/circuitbuild.c
+++ b/src/or/circuitbuild.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,11 +9,26 @@
  *
  * \brief Implements the details of building circuits (by chosing paths,
  * constructing/sending create/extend cells, and so on).
+ *
+ * On the client side, this module handles launching circuits. Circuit
+ * launches are srtarted from circuit_establish_circuit(), called from
+ * circuit_launch_by_extend_info()).  To choose the path the circuit will
+ * take, onion_extend_cpath() calls into a maze of node selection functions.
+ *
+ * Once the circuit is ready to be launched, the first hop is treated as a
+ * special case with circuit_handle_first_hop(), since it might need to open a
+ * channel.  As the channel opens, and later as CREATED and RELAY_EXTENDED
+ * cells arrive, the client will invoke circuit_send_next_onion_skin() to send
+ * CREATE or RELAY_EXTEND cells.
+ *
+ * On the server side, this module also handles the logic of responding to
+ * RELAY_EXTEND requests, using circuit_extend().
  **/
 
 #define CIRCUITBUILD_PRIVATE
 
 #include "or.h"
+#include "bridges.h"
 #include "channel.h"
 #include "circpathbias.h"
 #define CIRCUITBUILD_PRIVATE
@@ -49,15 +64,15 @@
 #include "transports.h"
 
 static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
-                                               uint16_t port,
-                                               const char *id_digest);
+                                            uint16_t port,
+                                            const char *id_digest,
+                                            const ed25519_public_key_t *ed_id);
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        const create_cell_t *create_cell,
                                        int relayed);
 static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit);
 static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
 static int onion_extend_cpath(origin_circuit_t *circ);
-static int count_acceptable_nodes(smartlist_t *routers);
 static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 
 /** This function tries to get a channel to the specified endpoint,
@@ -66,11 +81,12 @@ static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
  */
 static channel_t *
 channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest)
+                            const char *id_digest,
+                            const ed25519_public_key_t *ed_id)
 {
   channel_t *chan;
 
-  chan = channel_connect(addr, port, id_digest);
+  chan = channel_connect(addr, port, id_digest, ed_id);
   if (chan) command_setup_channel(chan);
 
   return chan;
@@ -502,6 +518,13 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
   return circ;
 }
 
+/** Return the guard state associated with <b>circ</b>, which may be NULL. */
+circuit_guard_state_t *
+origin_circuit_get_guard_state(origin_circuit_t *circ)
+{
+  return circ->guard_state;
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -540,6 +563,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
                          firsthop->extend_info->port));
 
   n_chan = channel_get_for_extend(firsthop->extend_info->identity_digest,
+                                  &firsthop->extend_info->ed_identity,
                                   &firsthop->extend_info->addr,
                                   &msg,
                                   &should_launch);
@@ -557,7 +581,8 @@ circuit_handle_first_hop(origin_circuit_t *circ)
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
-          firsthop->extend_info->identity_digest);
+          firsthop->extend_info->identity_digest,
+          &firsthop->extend_info->ed_identity);
       if (!n_chan) { /* connect failed, forget the whole thing */
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
@@ -791,12 +816,7 @@ should_use_create_fast_for_circuit(origin_circuit_t *circ)
      * creating on behalf of others. */
     return 0;
   }
-  if (options->FastFirstHopPK == -1) {
-    /* option is "auto", so look at the consensus. */
-    return networkstatus_get_param(NULL, "usecreatefast", 1, 0, 1);
-  }
-
-  return options->FastFirstHopPK;
+  return networkstatus_get_param(NULL, "usecreatefast", 0, 0, 1);
 }
 
 /** Return true if <b>circ</b> is the type of circuit we want to count
@@ -866,6 +886,27 @@ circuit_pick_extend_handshake(uint8_t *cell_type_out,
   }
 }
 
+/**
+ * Return true iff <b>purpose</b> is a purpose for a circuit which is
+ * allowed to have no guard configured, even if the circuit is multihop
+ * and guards are enabled.
+ */
+static int
+circuit_purpose_may_omit_guard(int purpose)
+{
+  switch (purpose) {
+    case CIRCUIT_PURPOSE_TESTING:
+    case CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT:
+      /* Testing circuits may omit guards because they're measuring
+       * liveness or performance, and don't want guards to interfere. */
+      return 1;
+    default:
+      /* All other multihop circuits should use guards if guards are
+       * enabled. */
+      return 0;
+  }
+}
+
 /** This is the backbone function for building circuits.
  *
  * If circ's first hop is closed, then we need to build a create
@@ -893,9 +934,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
     memset(&cc, 0, sizeof(cc));
     if (circ->build_state->onehop_tunnel)
       control_event_bootstrap(BOOTSTRAP_STATUS_ONEHOP_CREATE, 0);
-    else
+    else {
       control_event_bootstrap(BOOTSTRAP_STATUS_CIRCUIT_CREATE, 0);
 
+      /* If this is not a one-hop tunnel, the channel is being used
+       * for traffic that wants anonymity and protection from traffic
+       * analysis (such as netflow record retention). That means we want
+       * to pad it.
+       */
+      if (circ->base_.n_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS)
+        circ->base_.n_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+    }
+
     node = node_get_by_id(circ->base_.n_chan->identity_digest);
     fast = should_use_create_fast_for_circuit(circ);
     if (!fast) {
@@ -940,7 +990,37 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
     memset(&ec, 0, sizeof(ec));
     if (!hop) {
       /* done building the circuit. whew. */
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+      guard_usable_t r;
+      if (! circ->guard_state) {
+        if (circuit_get_cpath_len(circ) != 1 &&
+            ! circuit_purpose_may_omit_guard(circ->base_.purpose) &&
+            get_options()->UseEntryGuards) {
+          log_warn(LD_BUG, "%d-hop circuit %p with purpose %d has no "
+                   "guard state",
+                   circuit_get_cpath_len(circ), circ, circ->base_.purpose);
+        }
+        r = GUARD_USABLE_NOW;
+      } else {
+        r = entry_guard_succeeded(&circ->guard_state);
+      }
+      const int is_usable_for_streams = (r == GUARD_USABLE_NOW);
+      if (r == GUARD_USABLE_NOW) {
+        circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+      } else if (r == GUARD_MAYBE_USABLE_LATER) {
+        // Wait till either a better guard succeeds, or till
+        // all better guards fail.
+        circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_GUARD_WAIT);
+      } else {
+        tor_assert_nonfatal(r == GUARD_USABLE_NEVER);
+        return - END_CIRC_REASON_INTERNAL;
+      }
+
+      /* XXXX #21422 -- the rest of this branch needs careful thought!
+       * Some of the things here need to happen when a circuit becomes
+       * mechanically open; some need to happen when it is actually usable.
+       * I think I got them right, but more checking would be wise. -NM
+       */
+
       if (circuit_timeout_want_to_count_circ(circ)) {
         struct timeval end;
         long timediff;
@@ -958,7 +1038,7 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
                               "Assuming clock jump. Purpose %d (%s)", timediff,
                      circ->base_.purpose,
                      circuit_purpose_to_string(circ->base_.purpose));
-        } else if (!circuit_build_times_disabled()) {
+        } else if (!circuit_build_times_disabled(get_options())) {
           /* Only count circuit times if the network is live */
           if (circuit_build_times_network_check_live(
               get_circuit_build_times())) {
@@ -982,7 +1062,8 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
 
       pathbias_count_build_success(circ);
       circuit_rep_hist_note_result(circ);
-      circuit_has_opened(circ); /* do other actions as necessary */
+      if (is_usable_for_streams)
+        circuit_has_opened(circ); /* do other actions as necessary */
 
       if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) {
         const or_options_t *options = get_options();
@@ -1025,6 +1106,9 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
     ec.orport_ipv4.port = hop->extend_info->port;
     tor_addr_make_unspec(&ec.orport_ipv6.addr);
     memcpy(ec.node_id, hop->extend_info->identity_digest, DIGEST_LEN);
+    /* Set the ED25519 identity too -- it will only get included
+     * in the extend2 cell if we're configured to use it, though. */
+    ed25519_pubkey_copy(&ec.ed_pubkey, &hop->extend_info->ed_identity);
 
     len = onion_skin_create(ec.create_cell.handshake_type,
                             hop->extend_info,
@@ -1143,7 +1227,7 @@ circuit_extend(cell_t *cell, circuit_t *circ)
 
   /* Check if they asked us for 0000..0000. We support using
    * an empty fingerprint for the first hop (e.g. for a bridge relay),
-   * but we don't want to let people send us extend cells for empty
+   * but we don't want to let clients send us extend cells for empty
    * fingerprints -- a) because it opens the user up to a mitm attack,
    * and b) because it lets an attacker force the relay to hold open a
    * new TLS connection for each extend request. */
@@ -1153,6 +1237,18 @@ circuit_extend(cell_t *cell, circuit_t *circ)
     return -1;
   }
 
+  /* Fill in ed_pubkey if it was not provided and we can infer it from
+   * our networkstatus */
+  if (ed25519_public_key_is_zero(&ec.ed_pubkey)) {
+    const node_t *node = node_get_by_id((const char*)ec.node_id);
+    const ed25519_public_key_t *node_ed_id = NULL;
+    if (node &&
+        node_supports_ed25519_link_authentication(node) &&
+        (node_ed_id = node_get_ed25519_id(node))) {
+      ed25519_pubkey_copy(&ec.ed_pubkey, node_ed_id);
+    }
+  }
+
   /* Next, check if we're being asked to connect to the hop that the
    * extend cell came from. There isn't any reason for that, and it can
    * assist circular-path attacks. */
@@ -1164,7 +1260,18 @@ circuit_extend(cell_t *cell, circuit_t *circ)
     return -1;
   }
 
+  /* Check the previous hop Ed25519 ID too */
+  if (! ed25519_public_key_is_zero(&ec.ed_pubkey) &&
+      ed25519_pubkey_eq(&ec.ed_pubkey,
+                        &TO_OR_CIRCUIT(circ)->p_chan->ed25519_identity)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop "
+           "(by Ed25519 ID).");
+    return -1;
+  }
+
   n_chan = channel_get_for_extend((const char*)ec.node_id,
+                                  &ec.ed_pubkey,
                                   &ec.orport_ipv4.addr,
                                   &msg,
                                   &should_launch);
@@ -1176,8 +1283,9 @@ circuit_extend(cell_t *cell, circuit_t *circ)
 
     circ->n_hop = extend_info_new(NULL /*nickname*/,
                                   (const char*)ec.node_id,
-                                  NULL /*onion_key*/,
-                                  NULL /*curve25519_key*/,
+                                  &ec.ed_pubkey,
+                                  NULL, /*onion_key*/
+                                  NULL, /*curve25519_key*/
                                   &ec.orport_ipv4.addr,
                                   ec.orport_ipv4.port);
 
@@ -1190,7 +1298,8 @@ circuit_extend(cell_t *cell, circuit_t *circ)
       /* we should try to open a connection */
       n_chan = channel_connect_for_circuit(&ec.orport_ipv4.addr,
                                            ec.orport_ipv4.port,
-                                           (const char*)ec.node_id);
+                                           (const char*)ec.node_id,
+                                           &ec.ed_pubkey);
       if (!n_chan) {
         log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
         circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
@@ -1441,13 +1550,98 @@ onionskin_answer(or_circuit_t *circ,
   return 0;
 }
 
-/** Choose a length for a circuit of purpose <b>purpose</b>: three + the
- * number of endpoints that would give something away about our destination.
+/** Helper for new_route_len().  Choose a circuit length for purpose
+ * <b>purpose</b>: DEFAULT_ROUTE_LEN (+ 1 if someone else chose the
+ * exit).  If someone else chose the exit, they could be colluding
+ * with the exit, so add a randomly selected node to preserve
+ * anonymity.
+ *
+ * Here, "exit node" sometimes means an OR acting as an internal
+ * endpoint, rather than as a relay to an external endpoint.  This
+ * means there need to be at least DEFAULT_ROUTE_LEN routers between
+ * us and the internal endpoint to preserve the same anonymity
+ * properties that we would get when connecting to an external
+ * endpoint.  These internal endpoints can include:
+ *
+ *   - Connections to a directory of hidden services
+ *     (CIRCUIT_PURPOSE_C_GENERAL)
+ *
+ *   - A client connecting to an introduction point, which the hidden
+ *     service picked (CIRCUIT_PURPOSE_C_INTRODUCING, via
+ *     circuit_get_open_circ_or_launch() which rewrites it from
+ *     CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)
+ *
+ *   - A hidden service connecting to a rendezvous point, which the
+ *     client picked (CIRCUIT_PURPOSE_S_CONNECT_REND, via
+ *     rend_service_receive_introduction() and
+ *     rend_service_relaunch_rendezvous)
+ *
+ * There are currently two situations where we picked the exit node
+ * ourselves, making DEFAULT_ROUTE_LEN a safe circuit length:
+ *
+ *   - We are a hidden service connecting to an introduction point
+ *     (CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, via
+ *     rend_service_launch_establish_intro())
+ *
+ *   - We are a router testing its own reachabiity
+ *     (CIRCUIT_PURPOSE_TESTING, via consider_testing_reachability())
+ *
+ * onion_pick_cpath_exit() bypasses us (by not calling
+ * new_route_len()) in the one-hop tunnel case, so we don't need to
+ * handle that.
+ */
+static int
+route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
+{
+  int routelen = DEFAULT_ROUTE_LEN;
+  int known_purpose = 0;
+
+  if (!exit_ei)
+    return routelen;
+
+  switch (purpose) {
+    /* These two purposes connect to a router that we chose, so
+     * DEFAULT_ROUTE_LEN is safe. */
+  case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO:
+    /* hidden service connecting to introduction point */
+  case CIRCUIT_PURPOSE_TESTING:
+    /* router reachability testing */
+    known_purpose = 1;
+    break;
+
+    /* These three purposes connect to a router that someone else
+     * might have chosen, so add an extra hop to protect anonymity. */
+  case CIRCUIT_PURPOSE_C_GENERAL:
+    /* connecting to hidden service directory */
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    /* client connecting to introduction point */
+  case CIRCUIT_PURPOSE_S_CONNECT_REND:
+    /* hidden service connecting to rendezvous point */
+    known_purpose = 1;
+    routelen++;
+    break;
+
+  default:
+    /* Got a purpose not listed above along with a chosen exit.
+     * Increase the circuit length by one anyway for safety. */
+    routelen++;
+    break;
+  }
+
+  if (BUG(exit_ei && !known_purpose)) {
+    log_warn(LD_BUG, "Unhandled purpose %d with a chosen exit; "
+             "assuming routelen %d.", purpose, routelen);
+  }
+  return routelen;
+}
+
+/** Choose a length for a circuit of purpose <b>purpose</b> and check
+ * if enough routers are available.
  *
  * If the routerlist <b>nodes</b> doesn't have enough routers
  * to handle the desired path length, return -1.
  */
-static int
+STATIC int
 new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
 {
   int num_acceptable_routers;
@@ -1455,11 +1649,7 @@ new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
 
   tor_assert(nodes);
 
-  routelen = DEFAULT_ROUTE_LEN;
-  if (exit_ei &&
-      purpose != CIRCUIT_PURPOSE_TESTING &&
-      purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO)
-    routelen++;
+  routelen = route_len_for_purpose(purpose, exit_ei);
 
   num_acceptable_routers = count_acceptable_nodes(nodes);
 
@@ -1492,9 +1682,9 @@ circuit_get_unhandled_ports(time_t now)
  * If we're returning 0, set need_uptime and need_capacity to
  * indicate any requirements that the unhandled ports have.
  */
-int
-circuit_all_predicted_ports_handled(time_t now, int *need_uptime,
-                                    int *need_capacity)
+MOCK_IMPL(int,
+circuit_all_predicted_ports_handled, (time_t now, int *need_uptime,
+                                      int *need_capacity))
 {
   int i, enough;
   uint16_t *port;
@@ -1643,15 +1833,16 @@ choose_good_exit_server_general(int need_uptime, int need_capacity)
                  * we'll retry later in this function with need_update and
                  * need_capacity set to 0. */
     }
-    if (!(node->is_valid || options->AllowInvalid_ & ALLOW_INVALID_EXIT)) {
+    if (!(node->is_valid)) {
       /* if it's invalid and we don't want it */
       n_supported[i] = -1;
 //      log_fn(LOG_DEBUG,"Skipping node %s (index %d) -- invalid router.",
 //             router->nickname, i);
       continue; /* skip invalid routers */
     }
-    if (options->ExcludeSingleHopRelays &&
-        node_allows_single_hop_exits(node)) {
+    /* We do not allow relays that allow single hop exits by default. Option
+     * was deprecated in 0.2.9.2-alpha and removed in 0.3.1.0-alpha. */
+    if (node_allows_single_hop_exits(node)) {
       n_supported[i] = -1;
       continue;
     }
@@ -1783,7 +1974,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags,
                              const or_options_t *options)
 {
   const node_t *rp_node = NULL;
-  const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0;
   const int need_desc = (flags & CRN_NEED_DESC) != 0;
   const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
@@ -1795,7 +1985,6 @@ pick_tor2web_rendezvous_node(router_crn_flags_t flags,
 
   /* Add all running nodes to all_live_nodes */
   router_add_running_nodes_to_smartlist(all_live_nodes,
-                                        allow_invalid,
                                         0, 0, 0,
                                         need_desc,
                                         pref_addr,
@@ -1837,9 +2026,6 @@ pick_rendezvous_node(router_crn_flags_t flags)
 {
   const or_options_t *options = get_options();
 
-  if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS)
-    flags |= CRN_ALLOW_INVALID;
-
 #ifdef ENABLE_TOR2WEB_MODE
   /* We want to connect directly to the node if we can */
   router_crn_flags_t direct_flags = flags;
@@ -1896,8 +2082,6 @@ choose_good_exit_server(uint8_t purpose,
 
   switch (purpose) {
     case CIRCUIT_PURPOSE_C_GENERAL:
-      if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE)
-        flags |= CRN_ALLOW_INVALID;
       if (is_internal) /* pick it like a middle hop */
         return router_choose_random_node(NULL, options->ExcludeNodes, flags);
       else
@@ -2026,7 +2210,8 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
       return -1;
     }
     exit_ei = extend_info_from_node(node, 0);
-    tor_assert(exit_ei);
+    if (BUG(exit_ei == NULL))
+      return -1;
   }
   state->chosen_exit = exit_ei;
   return 0;
@@ -2082,8 +2267,8 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
 /** Return the number of routers in <b>routers</b> that are currently up
  * and available for building circuits through.
  */
-static int
-count_acceptable_nodes(smartlist_t *nodes)
+MOCK_IMPL(STATIC int,
+count_acceptable_nodes, (smartlist_t *nodes))
 {
   int num=0;
 
@@ -2094,10 +2279,6 @@ count_acceptable_nodes(smartlist_t *nodes)
     if (! node->is_running)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not running.",i);
       continue;
-    /* XXX This clause makes us count incorrectly: if AllowInvalidRouters
-     * allows this node in some places, then we're getting an inaccurate
-     * count. For now, be conservative and don't count it. But later we
-     * should try to be smarter. */
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
@@ -2168,8 +2349,6 @@ choose_good_middle_server(uint8_t purpose,
     flags |= CRN_NEED_UPTIME;
   if (state->need_capacity)
     flags |= CRN_NEED_CAPACITY;
-  if (options->AllowInvalid_ & ALLOW_INVALID_MIDDLE)
-    flags |= CRN_ALLOW_INVALID;
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
   smartlist_free(excluded);
   return choice;
@@ -2182,9 +2361,14 @@ choose_good_middle_server(uint8_t purpose,
  *
  * If <b>state</b> is NULL, we're choosing a router to serve as an entry
  * guard, not for any particular circuit.
+ *
+ * Set *<b>guard_state_out</b> to information about the guard that
+ * we're selecting, which we'll use later to remember whether the
+ * guard worked or not.
  */
 const node_t *
-choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
+choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
+                         circuit_guard_state_t **guard_state_out)
 {
   const node_t *choice;
   smartlist_t *excluded;
@@ -2199,7 +2383,8 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
       (purpose != CIRCUIT_PURPOSE_TESTING || options->BridgeRelay)) {
     /* This request is for an entry server to use for a regular circuit,
      * and we use entry guard nodes.  Just return one of the guard nodes.  */
-    return choose_random_entry(state);
+    tor_assert(guard_state_out);
+    return guards_choose_guard(state, guard_state_out);
   }
 
   excluded = smartlist_new();
@@ -2209,25 +2394,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
      * family. */
     nodelist_add_node_and_family(excluded, node);
   }
-  /* and exclude current entry guards and their families,
-   * unless we're in a test network, and excluding guards
-   * would exclude all nodes (i.e. we're in an incredibly small tor network,
-   * or we're using TestingAuthVoteGuard *).
-   * This is an incomplete fix, but is no worse than the previous behaviour,
-   * and only applies to minimal, testing tor networks
-   * (so it's no less secure) */
-  /*XXXX++ use the using_as_guard flag to accomplish this.*/
-  if (options->UseEntryGuards
-      && (!options->TestingTorNetwork ||
-         smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards())
-     )) {
-    SMARTLIST_FOREACH(get_entry_guards(), const entry_guard_t *, entry,
-      {
-        if ((node = node_get_by_id(entry->identity))) {
-          nodelist_add_node_and_family(excluded, node);
-        }
-      });
-  }
 
   if (state) {
     if (state->need_uptime)
@@ -2235,8 +2401,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state)
     if (state->need_capacity)
       flags |= CRN_NEED_CAPACITY;
   }
-  if (options->AllowInvalid_ & ALLOW_INVALID_ENTRY)
-    flags |= CRN_ALLOW_INVALID;
 
   choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
   smartlist_free(excluded);
@@ -2283,7 +2447,8 @@ onion_extend_cpath(origin_circuit_t *circ)
   if (cur_len == state->desired_path_len - 1) { /* Picking last node */
     info = extend_info_dup(state->chosen_exit);
   } else if (cur_len == 0) { /* picking first node */
-    const node_t *r = choose_good_entry_server(purpose, state);
+    const node_t *r = choose_good_entry_server(purpose, state,
+                                               &circ->guard_state);
     if (r) {
       /* If we're a client, use the preferred address rather than the
          primary address, for potentially connecting to an IPv6 OR
@@ -2291,14 +2456,14 @@ onion_extend_cpath(origin_circuit_t *circ)
       int client = (server_mode(get_options()) == 0);
       info = extend_info_from_node(r, client);
       /* Clients can fail to find an allowed address */
-      tor_assert(info || client);
+      tor_assert_nonfatal(info || client);
     }
   } else {
     const node_t *r =
       choose_good_middle_server(purpose, state, circ->cpath, cur_len);
     if (r) {
       info = extend_info_from_node(r, 0);
-      tor_assert(info);
+      tor_assert_nonfatal(info);
     }
   }
 
@@ -2341,19 +2506,23 @@ onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
 
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
-extend_info_new(const char *nickname, const char *digest,
+extend_info_new(const char *nickname,
+                const char *rsa_id_digest,
+                const ed25519_public_key_t *ed_id,
                 crypto_pk_t *onion_key,
-                const curve25519_public_key_t *curve25519_key,
+                const curve25519_public_key_t *ntor_key,
                 const tor_addr_t *addr, uint16_t port)
 {
   extend_info_t *info = tor_malloc_zero(sizeof(extend_info_t));
-  memcpy(info->identity_digest, digest, DIGEST_LEN);
+  memcpy(info->identity_digest, rsa_id_digest, DIGEST_LEN);
+  if (ed_id && !ed25519_public_key_is_zero(ed_id))
+    memcpy(&info->ed_identity, ed_id, sizeof(ed25519_public_key_t));
   if (nickname)
     strlcpy(info->nickname, nickname, sizeof(info->nickname));
   if (onion_key)
     info->onion_key = crypto_pk_dup_key(onion_key);
-  if (curve25519_key)
-    memcpy(&info->curve25519_onion_key, curve25519_key,
+  if (ntor_key)
+    memcpy(&info->curve25519_onion_key, ntor_key,
            sizeof(curve25519_public_key_t));
   tor_addr_copy(&info->addr, addr);
   info->port = port;
@@ -2403,20 +2572,35 @@ extend_info_from_node(const node_t *node, int for_direct_connect)
     return NULL;
   }
 
+  const ed25519_public_key_t *ed_pubkey = NULL;
+
+  /* Don't send the ed25519 pubkey unless the target node actually supports
+   * authenticating with it. */
+  if (node_supports_ed25519_link_authentication(node)) {
+    log_info(LD_CIRC, "Including Ed25519 ID for %s", node_describe(node));
+    ed_pubkey = node_get_ed25519_id(node);
+  } else if (node_get_ed25519_id(node)) {
+    log_info(LD_CIRC, "Not including the ed25519 ID for %s, since it won't "
+             " be able to authenticate it.",
+             node_describe(node));
+  }
+
   if (valid_addr && node->ri)
     return extend_info_new(node->ri->nickname,
-                             node->identity,
-                             node->ri->onion_pkey,
-                             node->ri->onion_curve25519_pkey,
-                             &ap.addr,
-                             ap.port);
+                           node->identity,
+                           ed_pubkey,
+                           node->ri->onion_pkey,
+                           node->ri->onion_curve25519_pkey,
+                           &ap.addr,
+                           ap.port);
   else if (valid_addr && node->rs && node->md)
     return extend_info_new(node->rs->nickname,
-                             node->identity,
-                             node->md->onion_pkey,
-                             node->md->onion_curve25519_pkey,
-                             &ap.addr,
-                             ap.port);
+                           node->identity,
+                           ed_pubkey,
+                           node->md->onion_pkey,
+                           node->md->onion_curve25519_pkey,
+                           &ap.addr,
+                           ap.port);
   else
     return NULL;
 }
@@ -2447,8 +2631,8 @@ extend_info_dup(extend_info_t *info)
   return newinfo;
 }
 
-/** Return the routerinfo_t for the chosen exit router in <b>state</b>.
- * If there is no chosen exit, or if we don't know the routerinfo_t for
+/** Return the node_t for the chosen exit router in <b>state</b>.
+ * If there is no chosen exit, or if we don't know the node_t for
  * the chosen exit, return NULL.
  */
 const node_t *
@@ -2459,6 +2643,17 @@ build_state_get_exit_node(cpath_build_state_t *state)
   return node_get_by_id(state->chosen_exit->identity_digest);
 }
 
+/** Return the RSA ID digest for the chosen exit router in <b>state</b>.
+ * If there is no chosen exit, return NULL.
+ */
+const uint8_t *
+build_state_get_exit_rsa_id(cpath_build_state_t *state)
+{
+  if (!state || !state->chosen_exit)
+    return NULL;
+  return (const uint8_t *) state->chosen_exit->identity_digest;
+}
+
 /** Return the nickname for the chosen exit router in <b>state</b>. If
  * there is no chosen exit, or if we don't know the routerinfo_t for the
  * chosen exit, return NULL.
@@ -2551,3 +2746,26 @@ extend_info_has_preferred_onion_key(const extend_info_t* ei)
   return extend_info_supports_ntor(ei);
 }
 
+/** Find the circuits that are waiting to find out whether their guards are
+ * usable, and if any are ready to become usable, mark them open and try
+ * attaching streams as appropriate. */
+void
+circuit_upgrade_circuits_from_guard_wait(void)
+{
+  smartlist_t *to_upgrade =
+    circuit_find_circuits_to_upgrade_from_guard_wait();
+
+  if (to_upgrade == NULL)
+    return;
+
+  log_info(LD_GUARD, "Upgrading %d circuits from 'waiting for better guard' "
+           "to 'open'.", smartlist_len(to_upgrade));
+
+  SMARTLIST_FOREACH_BEGIN(to_upgrade, origin_circuit_t *, circ) {
+    circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+    circuit_has_opened(circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(to_upgrade);
+}
+
diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h
index 1244601f71..45d9b2fb75 100644
--- a/src/or/circuitbuild.h
+++ b/src/or/circuitbuild.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,6 +21,8 @@ origin_circuit_t *origin_circuit_init(uint8_t purpose, int flags);
 origin_circuit_t *circuit_establish_circuit(uint8_t purpose,
                                             extend_info_t *exit,
                                             int flags);
+struct circuit_guard_state_t *origin_circuit_get_guard_state(
+                                            origin_circuit_t *circ);
 int circuit_handle_first_hop(origin_circuit_t *circ);
 void circuit_n_chan_done(channel_t *chan, int status,
                          int close_origin_circuits);
@@ -40,15 +42,18 @@ int onionskin_answer(or_circuit_t *circ,
                      const struct created_cell_t *created_cell,
                      const char *keys,
                      const uint8_t *rend_circ_nonce);
-int circuit_all_predicted_ports_handled(time_t now, int *need_uptime,
-                                        int *need_capacity);
+MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
+                                                     int *need_uptime,
+                                                     int *need_capacity));
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
 void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
-extend_info_t *extend_info_new(const char *nickname, const char *digest,
+extend_info_t *extend_info_new(const char *nickname,
+                               const char *rsa_id_digest,
+                               const ed25519_public_key_t *ed_id,
                                crypto_pk_t *onion_key,
-                               const curve25519_public_key_t *curve25519_key,
+                               const curve25519_public_key_t *ntor_key,
                                const tor_addr_t *addr, uint16_t port);
 extend_info_t *extend_info_from_node(const node_t *r, int for_direct_connect);
 extend_info_t *extend_info_dup(extend_info_t *info);
@@ -59,14 +64,22 @@ int extend_info_supports_ntor(const extend_info_t* ei);
 int circuit_can_use_tap(const origin_circuit_t *circ);
 int circuit_has_usable_onion_key(const origin_circuit_t *circ);
 int extend_info_has_preferred_onion_key(const extend_info_t* ei);
+const uint8_t *build_state_get_exit_rsa_id(cpath_build_state_t *state);
 const node_t *build_state_get_exit_node(cpath_build_state_t *state);
 const char *build_state_get_exit_nickname(cpath_build_state_t *state);
 
+struct circuit_guard_state_t;
+
 const node_t *choose_good_entry_server(uint8_t purpose,
-                                       cpath_build_state_t *state);
+                           cpath_build_state_t *state,
+                           struct circuit_guard_state_t **guard_state_out);
+void circuit_upgrade_circuits_from_guard_wait(void);
 
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
+STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+                         smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
 #if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS)
 STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags,
                                                   const or_options_t *options);
diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c
index 977afca18d..6ffaabc16f 100644
--- a/src/or/circuitlist.c
+++ b/src/or/circuitlist.c
@@ -1,13 +1,54 @@
 /* Copyright 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file circuitlist.c
  *
- * \brief Manage the global circuit list, and looking up circuits within it.
+ * \brief Manage global structures that list and index circuits, and
+ *   look up circuits within them.
+ *
+ * One of the most frequent operations in Tor occurs every time that
+ * a relay cell arrives on a channel.  When that happens, we need to
+ * find which circuit it is associated with, based on the channel and the
+ * circuit ID in the relay cell.
+ *
+ * To handle that, we maintain a global list of circuits, and a hashtable
+ * mapping [channel,circID] pairs to circuits.  Circuits are added to and
+ * removed from this mapping using circuit_set_p_circid_chan() and
+ * circuit_set_n_circid_chan().  To look up a circuit from this map, most
+ * callers should use circuit_get_by_circid_channel(), though
+ * circuit_get_by_circid_channel_even_if_marked() is appropriate under some
+ * circumstances.
+ *
+ * We also need to allow for the possibility that we have blocked use of a
+ * circuit ID (because we are waiting to send a DESTROY cell), but the
+ * circuit is not there any more.  For that case, we allow placeholder
+ * entries in the table, using channel_mark_circid_unusable().
+ *
+ * To efficiently handle a channel that has just opened, we also maintain a
+ * list of the circuits waiting for channels, so we can attach them as
+ * needed without iterating through the whole list of circuits, using
+ * circuit_get_all_pending_on_channel().
+ *
+ * In this module, we also handle the list of circuits that have been
+ * marked for close elsewhere, and close them as needed.  (We use this
+ * "mark now, close later" pattern here and elsewhere to avoid
+ * unpredictable recursion if we closed every circuit immediately upon
+ * realizing it needed to close.)  See circuit_mark_for_close() for the
+ * mark function, and circuit_close_all_marked() for the close function.
+ *
+ * For hidden services, we need to be able to look up introduction point
+ * circuits and rendezvous circuits by cookie, key, etc.  These are
+ * currently handled with linear searches in
+ * circuit_get_ready_rend_circuit_by_rend_data(),
+ * circuit_get_next_by_pk_and_purpose(), and with hash lookups in
+ * circuit_get_rendezvous() and circuit_get_intro_point().
+ *
+ * This module is also the entry point for our out-of-memory handler
+ * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
 #include "or.h"
@@ -22,7 +63,10 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "entrynodes.h"
 #include "main.h"
+#include "hs_circuitmap.h"
+#include "hs_common.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "onion.h"
@@ -34,6 +78,7 @@
 #include "rephist.h"
 #include "routerlist.h"
 #include "routerset.h"
+#include "channelpadding.h"
 
 #include "ht.h"
 
@@ -42,18 +87,23 @@
 /** A global list of all circuits at this hop. */
 static smartlist_t *global_circuitlist = NULL;
 
+/** A global list of all origin circuits. Every element of this is also
+ * an element of global_circuitlist. */
+static smartlist_t *global_origin_circuit_list = NULL;
+
 /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */
 static smartlist_t *circuits_pending_chans = NULL;
 
+/** List of all the (origin) circuits whose state is
+ * CIRCUIT_STATE_GUARD_WAIT. */
+static smartlist_t *circuits_pending_other_guards = NULL;
+
 /** A list of all the circuits that have been marked with
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
 static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
-//static void circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ,
-//                                   const uint8_t *token);
-static void circuit_clear_rend_token(or_circuit_t *circ);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
 
@@ -309,8 +359,8 @@ channel_note_destroy_pending(channel_t *chan, circid_t id)
 
 /** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with
  * circuit ID <b>id</b> -- typically, because it has been sent. */
-MOCK_IMPL(void, channel_note_destroy_not_pending,
-          (channel_t *chan, circid_t id))
+MOCK_IMPL(void,
+channel_note_destroy_not_pending,(channel_t *chan, circid_t id))
 {
   circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan);
   if (circ) {
@@ -386,8 +436,10 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   tor_assert(circ);
   if (state == circ->state)
     return;
-  if (!circuits_pending_chans)
+  if (PREDICT_UNLIKELY(!circuits_pending_chans))
     circuits_pending_chans = smartlist_new();
+  if (PREDICT_UNLIKELY(!circuits_pending_other_guards))
+    circuits_pending_other_guards = smartlist_new();
   if (circ->state == CIRCUIT_STATE_CHAN_WAIT) {
     /* remove from waiting-circuit list. */
     smartlist_remove(circuits_pending_chans, circ);
@@ -396,7 +448,13 @@ circuit_set_state(circuit_t *circ, uint8_t state)
     /* add to waiting-circuit list. */
     smartlist_add(circuits_pending_chans, circ);
   }
-  if (state == CIRCUIT_STATE_OPEN)
+  if (circ->state == CIRCUIT_STATE_GUARD_WAIT) {
+    smartlist_remove(circuits_pending_other_guards, circ);
+  }
+  if (state == CIRCUIT_STATE_GUARD_WAIT) {
+    smartlist_add(circuits_pending_other_guards, circ);
+  }
+  if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
 }
@@ -452,6 +510,39 @@ circuit_count_pending_on_channel(channel_t *chan)
   return cnt;
 }
 
+/** Remove <b>origin_circ</b> from the global list of origin circuits.
+ * Called when we are freeing a circuit.
+ */
+static void
+circuit_remove_from_origin_circuit_list(origin_circuit_t *origin_circ)
+{
+  int origin_idx = origin_circ->global_origin_circuit_list_idx;
+  if (origin_idx < 0)
+    return;
+  origin_circuit_t *c2;
+  tor_assert(origin_idx <= smartlist_len(global_origin_circuit_list));
+  c2 = smartlist_get(global_origin_circuit_list, origin_idx);
+  tor_assert(origin_circ == c2);
+  smartlist_del(global_origin_circuit_list, origin_idx);
+  if (origin_idx < smartlist_len(global_origin_circuit_list)) {
+    origin_circuit_t *replacement =
+      smartlist_get(global_origin_circuit_list, origin_idx);
+    replacement->global_origin_circuit_list_idx = origin_idx;
+  }
+  origin_circ->global_origin_circuit_list_idx = -1;
+}
+
+/** Add <b>origin_circ</b> to the global list of origin circuits. Called
+ * when creating the circuit. */
+static void
+circuit_add_to_origin_circuit_list(origin_circuit_t *origin_circ)
+{
+  tor_assert(origin_circ->global_origin_circuit_list_idx == -1);
+  smartlist_t *lst = circuit_get_global_origin_circuit_list();
+  smartlist_add(lst, origin_circ);
+  origin_circ->global_origin_circuit_list_idx = smartlist_len(lst) - 1;
+}
+
 /** Detach from the global circuit list, and deallocate, all
  * circuits that have been marked for close.
  */
@@ -474,6 +565,11 @@ circuit_close_all_marked(void)
     }
     circ->global_circuitlist_idx = -1;
 
+    /* Remove it from the origin circuit list, if appropriate. */
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      circuit_remove_from_origin_circuit_list(TO_ORIGIN_CIRCUIT(circ));
+    }
+
     circuit_about_to_free(circ);
     circuit_free(circ);
   } SMARTLIST_FOREACH_END(circ);
@@ -481,7 +577,7 @@ circuit_close_all_marked(void)
   smartlist_clear(circuits_pending_close);
 }
 
-/** Return the head of the global linked list of circuits. */
+/** Return a pointer to the global list of circuits. */
 MOCK_IMPL(smartlist_t *,
 circuit_get_global_list,(void))
 {
@@ -490,6 +586,15 @@ circuit_get_global_list,(void))
   return global_circuitlist;
 }
 
+/** Return a pointer to the global list of origin circuits. */
+smartlist_t *
+circuit_get_global_origin_circuit_list(void)
+{
+  if (NULL == global_origin_circuit_list)
+    global_origin_circuit_list = smartlist_new();
+  return global_origin_circuit_list;
+}
+
 /** Function to make circ-\>state human-readable */
 const char *
 circuit_state_to_string(int state)
@@ -499,6 +604,8 @@ circuit_state_to_string(int state)
     case CIRCUIT_STATE_BUILDING: return "doing handshakes";
     case CIRCUIT_STATE_ONIONSKIN_PENDING: return "processing the onion";
     case CIRCUIT_STATE_CHAN_WAIT: return "connecting to server";
+    case CIRCUIT_STATE_GUARD_WAIT: return "waiting to see how other "
+      "guards perform";
     case CIRCUIT_STATE_OPEN: return "open";
     default:
       log_warn(LD_BUG, "Unknown circuit state %d", state);
@@ -708,6 +815,11 @@ init_circuit_base(circuit_t *circ)
   circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1;
 }
 
+/** If we haven't yet decided on a good timeout value for circuit
+ * building, we close idle circuits aggressively so we can get more
+ * data points. */
+#define IDLE_TIMEOUT_WHILE_LEARNING (1*60)
+
 /** Allocate space for a new circuit, initializing with <b>p_circ_id</b>
  * and <b>p_conn</b>. Add it to the global circuit list.
  */
@@ -729,8 +841,47 @@ origin_circuit_new(void)
 
   init_circuit_base(TO_CIRCUIT(circ));
 
+  /* Add to origin-list. */
+  circ->global_origin_circuit_list_idx = -1;
+  circuit_add_to_origin_circuit_list(circ);
+
   circuit_build_times_update_last_circ(get_circuit_build_times_mutable());
 
+  if (! circuit_build_times_disabled(get_options()) &&
+      circuit_build_times_needs_circuits(get_circuit_build_times())) {
+    /* Circuits should be shorter lived if we need more of them
+     * for learning a good build timeout */
+    circ->circuit_idle_timeout = IDLE_TIMEOUT_WHILE_LEARNING;
+  } else {
+    // This should always be larger than the current port prediction time
+    // remaining, or else we'll end up with the case where a circuit times out
+    // and another one is built, effectively doubling the timeout window.
+    //
+    // We also randomize it by up to 5% more (ie 5% of 0 to 3600 seconds,
+    // depending on how much circuit prediction time is remaining) so that
+    // we don't close a bunch of unused circuits all at the same time.
+    int prediction_time_remaining =
+      predicted_ports_prediction_time_remaining(time(NULL));
+    circ->circuit_idle_timeout = prediction_time_remaining+1+
+        crypto_rand_int(1+prediction_time_remaining/20);
+
+    if (circ->circuit_idle_timeout <= 0) {
+      log_warn(LD_BUG,
+               "Circuit chose a negative idle timeout of %d based on "
+               "%d seconds of predictive building remaining.",
+               circ->circuit_idle_timeout,
+               prediction_time_remaining);
+      circ->circuit_idle_timeout = IDLE_TIMEOUT_WHILE_LEARNING;
+    }
+
+    log_info(LD_CIRC,
+              "Circuit " U64_FORMAT " chose an idle timeout of %d based on "
+              "%d seconds of predictive building remaining.",
+              U64_PRINTF_ARG(circ->global_identifier),
+              circ->circuit_idle_timeout,
+              prediction_time_remaining);
+  }
+
   return circ;
 }
 
@@ -786,6 +937,9 @@ circuit_free(circuit_t *circ)
     mem = ocirc;
     memlen = sizeof(origin_circuit_t);
     tor_assert(circ->magic == ORIGIN_CIRCUIT_MAGIC);
+
+    circuit_remove_from_origin_circuit_list(ocirc);
+
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
         circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
@@ -793,6 +947,12 @@ circuit_free(circuit_t *circ)
     }
     tor_free(ocirc->build_state);
 
+    /* Cancel before freeing, if we haven't already succeeded or failed. */
+    if (ocirc->guard_state) {
+      entry_guard_cancel(&ocirc->guard_state);
+    }
+    circuit_guard_state_free(ocirc->guard_state);
+
     circuit_clear_cpath(ocirc);
 
     crypto_pk_free(ocirc->intro_key);
@@ -824,8 +984,6 @@ circuit_free(circuit_t *circ)
     crypto_cipher_free(ocirc->n_crypto);
     crypto_digest_free(ocirc->n_digest);
 
-    circuit_clear_rend_token(ocirc);
-
     if (ocirc->rend_splice) {
       or_circuit_t *other = ocirc->rend_splice;
       tor_assert(other->base_.magic == OR_CIRCUIT_MAGIC);
@@ -857,6 +1015,11 @@ circuit_free(circuit_t *circ)
   /* Remove from map. */
   circuit_set_n_circid_chan(circ, 0, NULL);
 
+  /* Clear HS circuitmap token from this circ (if any) */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+
   /* Clear cell queue _after_ removing it from the map.  Otherwise our
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
@@ -925,12 +1088,18 @@ circuit_free_all(void)
   smartlist_free(lst);
   global_circuitlist = NULL;
 
+  smartlist_free(global_origin_circuit_list);
+  global_origin_circuit_list = NULL;
+
   smartlist_free(circuits_pending_chans);
   circuits_pending_chans = NULL;
 
   smartlist_free(circuits_pending_close);
   circuits_pending_close = NULL;
 
+  smartlist_free(circuits_pending_other_guards);
+  circuits_pending_other_guards = NULL;
+
   {
     chan_circid_circuit_map_t **elt, **next, *c;
     for (elt = HT_START(chan_circid_map, &chan_circid_map);
@@ -1311,9 +1480,11 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data)
     if (!circ->marked_for_close &&
         circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) {
       origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-      if (ocirc->rend_data &&
-          !rend_cmp_service_ids(rend_data->onion_address,
-                                ocirc->rend_data->onion_address) &&
+      if (ocirc->rend_data == NULL) {
+        continue;
+      }
+      if (!rend_cmp_service_ids(rend_data_get_address(rend_data),
+                                rend_data_get_address(ocirc->rend_data)) &&
           tor_memeq(ocirc->rend_data->rend_cookie,
                     rend_data->rend_cookie,
                     REND_COOKIE_LEN))
@@ -1324,14 +1495,50 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data)
   return NULL;
 }
 
+/** Return the first service introduction circuit originating from the global
+ * circuit list after <b>start</b> or at the start of the list if <b>start</b>
+ * is NULL. Return NULL if no circuit is found.
+ *
+ * A service introduction point circuit has a purpose of either
+ * CIRCUIT_PURPOSE_S_ESTABLISH_INTRO or CIRCUIT_PURPOSE_S_INTRO. This does not
+ * return a circuit marked for close and its state must be open. */
+origin_circuit_t *
+circuit_get_next_service_intro_circ(origin_circuit_t *start)
+{
+  int idx = 0;
+  smartlist_t *lst = circuit_get_global_list();
+
+  if (start) {
+    idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1;
+  }
+
+  for ( ; idx < smartlist_len(lst); ++idx) {
+    circuit_t *circ = smartlist_get(lst, idx);
+
+    /* Ignore a marked for close circuit or purpose not matching a service
+     * intro point or if the state is not open. */
+    if (circ->marked_for_close || circ->state != CIRCUIT_STATE_OPEN ||
+        (circ->purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO &&
+         circ->purpose != CIRCUIT_PURPOSE_S_INTRO)) {
+      continue;
+    }
+    /* The purposes we are looking for are only for origin circuits so the
+     * following is valid. */
+    return TO_ORIGIN_CIRCUIT(circ);
+  }
+  /* Not found. */
+  return NULL;
+}
+
 /** Return the first circuit originating here in global_circuitlist after
- * <b>start</b> whose purpose is <b>purpose</b>, and where
- * <b>digest</b> (if set) matches the rend_pk_digest field. Return NULL if no
- * circuit is found.  If <b>start</b> is NULL, begin at the start of the list.
+ * <b>start</b> whose purpose is <b>purpose</b>, and where <b>digest</b> (if
+ * set) matches the private key digest of the rend data associated with the
+ * circuit. Return NULL if no circuit is found. If <b>start</b> is NULL,
+ * begin at the start of the list.
  */
 origin_circuit_t *
 circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
-                                   const char *digest, uint8_t purpose)
+                                   const uint8_t *digest, uint8_t purpose)
 {
   int idx;
   smartlist_t *lst = circuit_get_global_list();
@@ -1343,190 +1550,25 @@ circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
 
   for ( ; idx < smartlist_len(lst); ++idx) {
     circuit_t *circ = smartlist_get(lst, idx);
+    origin_circuit_t *ocirc;
 
     if (circ->marked_for_close)
       continue;
     if (circ->purpose != purpose)
       continue;
+    /* At this point we should be able to get a valid origin circuit because
+     * the origin purpose we are looking for matches this circuit. */
+    if (BUG(!CIRCUIT_PURPOSE_IS_ORIGIN(circ->purpose))) {
+      break;
+    }
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
     if (!digest)
-      return TO_ORIGIN_CIRCUIT(circ);
-    else if (TO_ORIGIN_CIRCUIT(circ)->rend_data &&
-             tor_memeq(TO_ORIGIN_CIRCUIT(circ)->rend_data->rend_pk_digest,
-                     digest, DIGEST_LEN))
-      return TO_ORIGIN_CIRCUIT(circ);
-  }
-  return NULL;
-}
-
-/** Map from rendezvous cookie to or_circuit_t */
-static digestmap_t *rend_cookie_map = NULL;
-
-/** Map from introduction point digest to or_circuit_t */
-static digestmap_t *intro_digest_map = NULL;
-
-/** Return the OR circuit whose purpose is <b>purpose</b>, and whose
- * rend_token is the REND_TOKEN_LEN-byte <b>token</b>. If <b>is_rend_circ</b>,
- * look for rendezvous point circuits; otherwise look for introduction point
- * circuits. */
-static or_circuit_t *
-circuit_get_by_rend_token_and_purpose(uint8_t purpose, int is_rend_circ,
-                                      const char *token)
-{
-  or_circuit_t *circ;
-  digestmap_t *map = is_rend_circ ? rend_cookie_map : intro_digest_map;
-
-  if (!map)
-    return NULL;
-
-  circ = digestmap_get(map, token);
-  if (!circ ||
-      circ->base_.purpose != purpose ||
-      circ->base_.marked_for_close)
-    return NULL;
-
-  if (!circ->rendinfo) {
-    char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN));
-    log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned a "
-             "circuit with no rendinfo set.",
-             safe_str(t), is_rend_circ);
-    tor_free(t);
-    return NULL;
-  }
-
-  if (! bool_eq(circ->rendinfo->is_rend_circ, is_rend_circ) ||
-      tor_memneq(circ->rendinfo->rend_token, token, REND_TOKEN_LEN)) {
-    char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN));
-    log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned %s:%d",
-             safe_str(t), is_rend_circ,
-             safe_str(hex_str(circ->rendinfo->rend_token, REND_TOKEN_LEN)),
-             (int)circ->rendinfo->is_rend_circ);
-    tor_free(t);
-    return NULL;
-  }
-
-  return circ;
-}
-
-/** Clear the rendezvous cookie or introduction point key digest that's
- * configured on <b>circ</b>, if any, and remove it from any such maps. */
-static void
-circuit_clear_rend_token(or_circuit_t *circ)
-{
-  or_circuit_t *found_circ;
-  digestmap_t *map;
-
-  if (!circ || !circ->rendinfo)
-    return;
-
-  map = circ->rendinfo->is_rend_circ ? rend_cookie_map : intro_digest_map;
-
-  if (!map) {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but found no map");
-    return;
-  }
-
-  found_circ = digestmap_get(map, circ->rendinfo->rend_token);
-  if (found_circ == circ) {
-    /* Great, this is the right one. */
-    digestmap_remove(map, circ->rendinfo->rend_token);
-  } else if (found_circ) {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but "
-             "it was already replaced in the map.");
-  } else {
-    log_warn(LD_BUG, "Tried to clear rend token on circuit, but "
-             "it not in the map at all.");
-  }
-
-  tor_free(circ->rendinfo); /* Sets it to NULL too */
-}
-
-/** Set the rendezvous cookie (if is_rend_circ), or the introduction point
- * digest (if ! is_rend_circ) of <b>circ</b> to the REND_TOKEN_LEN-byte value
- * in <b>token</b>, and add it to the appropriate map.  If it previously had a
- * token, clear it.  If another circuit previously had the same
- * cookie/intro-digest, mark that circuit and remove it from the map. */
-static void
-circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ,
-                       const uint8_t *token)
-{
-  digestmap_t **map_p, *map;
-  or_circuit_t *found_circ;
-
-  /* Find the right map, creating it as needed */
-  map_p = is_rend_circ ? &rend_cookie_map : &intro_digest_map;
-
-  if (!*map_p)
-    *map_p = digestmap_new();
-
-  map = *map_p;
-
-  /* If this circuit already has a token, we need to remove that. */
-  if (circ->rendinfo)
-    circuit_clear_rend_token(circ);
-
-  if (token == NULL) {
-    /* We were only trying to remove this token, not set a new one. */
-    return;
-  }
-
-  found_circ = digestmap_get(map, (const char *)token);
-  if (found_circ) {
-    tor_assert(found_circ != circ);
-    circuit_clear_rend_token(found_circ);
-    if (! found_circ->base_.marked_for_close) {
-      circuit_mark_for_close(TO_CIRCUIT(found_circ), END_CIRC_REASON_FINISHED);
-      if (is_rend_circ) {
-        log_fn(LOG_PROTOCOL_WARN, LD_REND,
-               "Duplicate rendezvous cookie (%s...) used on two circuits",
-               hex_str((const char*)token, 4)); /* only log first 4 chars */
-      }
+      return ocirc;
+    if (rend_circuit_pk_digest_eq(ocirc, digest)) {
+      return ocirc;
     }
   }
-
-  /* Now set up the rendinfo */
-  circ->rendinfo = tor_malloc(sizeof(*circ->rendinfo));
-  memcpy(circ->rendinfo->rend_token, token, REND_TOKEN_LEN);
-  circ->rendinfo->is_rend_circ = is_rend_circ ? 1 : 0;
-
-  digestmap_set(map, (const char *)token, circ);
-}
-
-/** Return the circuit waiting for a rendezvous with the provided cookie.
- * Return NULL if no such circuit is found.
- */
-or_circuit_t *
-circuit_get_rendezvous(const uint8_t *cookie)
-{
-  return circuit_get_by_rend_token_and_purpose(
-                                     CIRCUIT_PURPOSE_REND_POINT_WAITING,
-                                     1, (const char*)cookie);
-}
-
-/** Return the circuit waiting for intro cells of the given digest.
- * Return NULL if no such circuit is found.
- */
-or_circuit_t *
-circuit_get_intro_point(const uint8_t *digest)
-{
-  return circuit_get_by_rend_token_and_purpose(
-                                     CIRCUIT_PURPOSE_INTRO_POINT, 0,
-                                     (const char *)digest);
-}
-
-/** Set the rendezvous cookie of <b>circ</b> to <b>cookie</b>.  If another
- * circuit previously had that cookie, mark it. */
-void
-circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie)
-{
-  circuit_set_rend_token(circ, 1, cookie);
-}
-
-/** Set the intro point key digest of <b>circ</b> to <b>cookie</b>.  If another
- * circuit previously had that intro point digest, mark it. */
-void
-circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest)
-{
-  circuit_set_rend_token(circ, 0, digest);
+  return NULL;
 }
 
 /** Return a circuit that is open, is CIRCUIT_PURPOSE_C_GENERAL,
@@ -1539,6 +1581,14 @@ circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest)
  * cannibalize.
  *
  * If !CIRCLAUNCH_NEED_UPTIME, prefer returning non-uptime circuits.
+ *
+ * To "cannibalize" a circuit means to extend it an extra hop, and use it
+ * for some other purpose than we had originally intended.  We do this when
+ * we want to perform some low-bandwidth task at a specific relay, and we
+ * would like the circuit to complete as soon as possible.  (If we were going
+ * to use a lot of bandwidth, we wouldn't want a circuit with an extra hop.
+ * If we didn't care about circuit completion latency, we would just build
+ * a new circuit.)
  */
 origin_circuit_t *
 circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
@@ -1613,6 +1663,37 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info,
   return best;
 }
 
+/**
+ * Check whether any of the origin circuits that are waiting to see if
+ * their guard is good enough to use can be upgraded to "ready". If so,
+ * return a new smartlist containing them. Otherwise return NULL.
+ */
+smartlist_t *
+circuit_find_circuits_to_upgrade_from_guard_wait(void)
+{
+  /* Only if some circuit is actually waiting on an upgrade should we
+   * run the algorithm. */
+  if (! circuits_pending_other_guards ||
+      smartlist_len(circuits_pending_other_guards)==0)
+    return NULL;
+  /* Only if we have some origin circuits should we run the algorithm. */
+  if (!global_origin_circuit_list)
+    return NULL;
+
+  /* Okay; we can pass our circuit list to entrynodes.c.*/
+  smartlist_t *result = smartlist_new();
+  int circuits_upgraded  = entry_guards_upgrade_waiting_circuits(
+                                                 get_guard_selection_info(),
+                                                 global_origin_circuit_list,
+                                                 result);
+  if (circuits_upgraded && smartlist_len(result)) {
+    return result;
+  } else {
+    smartlist_free(result);
+    return NULL;
+  }
+}
+
 /** Return the number of hops in circuit's path. If circ has no entries,
  * or is NULL, returns 0. */
 int
@@ -1807,7 +1888,8 @@ circuit_about_to_free(circuit_t *circ)
    * module then.  If it isn't OPEN, we send it there now to remember which
    * links worked and which didn't.
    */
-  if (circ->state != CIRCUIT_STATE_OPEN) {
+  if (circ->state != CIRCUIT_STATE_OPEN &&
+      circ->state != CIRCUIT_STATE_GUARD_WAIT) {
     if (CIRCUIT_IS_ORIGIN(circ)) {
       origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
       circuit_build_failed(ocirc); /* take actions if necessary */
@@ -1818,9 +1900,14 @@ circuit_about_to_free(circuit_t *circ)
     if (circuits_pending_chans)
       smartlist_remove(circuits_pending_chans, circ);
   }
+  if (circuits_pending_other_guards) {
+    smartlist_remove(circuits_pending_other_guards, circ);
+  }
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
-     (circ->state == CIRCUIT_STATE_OPEN)?CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
+     (circ->state == CIRCUIT_STATE_OPEN ||
+      circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
+                                 CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
      orig_reason);
   }
 
@@ -1833,7 +1920,7 @@ circuit_about_to_free(circuit_t *circ)
     if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) {
       /* treat this like getting a nack from it */
       log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
-          safe_str_client(ocirc->rend_data->onion_address),
+          safe_str_client(rend_data_get_address(ocirc->rend_data)),
           safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
           timed_out ? "Recording timeout." : "Removing from descriptor.");
       rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
@@ -1850,7 +1937,7 @@ circuit_about_to_free(circuit_t *circ)
         log_info(LD_REND, "Failed intro circ %s to %s "
             "(building circuit to intro point). "
             "Marking intro point as possibly unreachable.",
-            safe_str_client(ocirc->rend_data->onion_address),
+            safe_str_client(rend_data_get_address(ocirc->rend_data)),
             safe_str_client(build_state_get_exit_nickname(
                                               ocirc->build_state)));
         rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
@@ -1945,10 +2032,10 @@ single_conn_free_bytes(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_DIR) {
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
-    if (dir_conn->zlib_state) {
-      result += tor_zlib_state_size(dir_conn->zlib_state);
-      tor_zlib_free(dir_conn->zlib_state);
-      dir_conn->zlib_state = NULL;
+    if (dir_conn->compress_state) {
+      result += tor_compress_state_size(dir_conn->compress_state);
+      tor_compress_free(dir_conn->compress_state);
+      dir_conn->compress_state = NULL;
     }
   }
   return result;
@@ -2343,7 +2430,8 @@ assert_circuit_ok(const circuit_t *c)
 
   tor_assert(c->deliver_window >= 0);
   tor_assert(c->package_window >= 0);
-  if (c->state == CIRCUIT_STATE_OPEN) {
+  if (c->state == CIRCUIT_STATE_OPEN ||
+      c->state == CIRCUIT_STATE_GUARD_WAIT) {
     tor_assert(!c->n_chan_create_cell);
     if (or_circ) {
       tor_assert(or_circ->n_crypto);
diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h
index 2707b426ab..d647062f46 100644
--- a/src/or/circuitlist.h
+++ b/src/or/circuitlist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,6 +15,7 @@
 #include "testsupport.h"
 
 MOCK_DECL(smartlist_t *, circuit_get_global_list, (void));
+smartlist_t *circuit_get_global_origin_circuit_list(void);
 const char *circuit_state_to_string(int state);
 const char *circuit_purpose_to_controller_string(uint8_t purpose);
 const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose);
@@ -45,11 +46,8 @@ origin_circuit_t *circuit_get_by_global_id(uint32_t id);
 origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data(
   const rend_data_t *rend_data);
 origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start,
-                                         const char *digest, uint8_t purpose);
-or_circuit_t *circuit_get_rendezvous(const uint8_t *cookie);
-or_circuit_t *circuit_get_intro_point(const uint8_t *digest);
-void circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie);
-void circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest);
+                                     const uint8_t *digest, uint8_t purpose);
+origin_circuit_t *circuit_get_next_service_intro_circ(origin_circuit_t *start);
 origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose,
                                               extend_info_t *info, int flags);
 void circuit_mark_all_unused_circs(void);
@@ -77,6 +75,8 @@ void channel_note_destroy_pending(channel_t *chan, circid_t id);
 MOCK_DECL(void, channel_note_destroy_not_pending,
           (channel_t *chan, circid_t id));
 
+smartlist_t *circuit_find_circuits_to_upgrade_from_guard_wait(void);
+
 #ifdef CIRCUITLIST_PRIVATE
 STATIC void circuit_free(circuit_t *circ);
 STATIC size_t n_cells_in_circ_queues(const circuit_t *c);
diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c
index 036fb9570d..6c825011b6 100644
--- a/src/or/circuitmux.c
+++ b/src/or/circuitmux.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h
index f53abdd8c8..bf93bb8cbf 100644
--- a/src/or/circuitmux.h
+++ b/src/or/circuitmux.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c
index 5c2ebde73b..c2440b13f0 100644
--- a/src/or/circuitmux_ewma.c
+++ b/src/or/circuitmux_ewma.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -500,7 +500,7 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
   tor_assert(pol_data_2);
 
   p1 = TO_EWMA_POL_DATA(pol_data_1);
-  p2 = TO_EWMA_POL_DATA(pol_data_1);
+  p2 = TO_EWMA_POL_DATA(pol_data_2);
 
   if (p1 != p2) {
     /* Get the head cell_ewma_t from each queue */
diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h
index a7b8961ac6..1f04408789 100644
--- a/src/or/circuitmux_ewma.h
+++ b/src/or/circuitmux_ewma.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c
index 418acc0024..51d580a1a4 100644
--- a/src/or/circuitstats.c
+++ b/src/or/circuitstats.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -105,13 +105,21 @@ get_circuit_build_timeout_ms(void)
  *  6. If we are configured in Single Onion mode
  */
 int
-circuit_build_times_disabled(void)
+circuit_build_times_disabled(const or_options_t *options)
+{
+  return circuit_build_times_disabled_(options, 0);
+}
+
+/** As circuit_build_times_disabled, but take options as an argument. */
+int
+circuit_build_times_disabled_(const or_options_t *options,
+                              int ignore_consensus)
 {
   if (unit_tests) {
     return 0;
   } else {
-    const or_options_t *options = get_options();
-    int consensus_disabled = networkstatus_get_param(NULL, "cbtdisabled",
+    int consensus_disabled =
+      ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled",
                                                      0, 0, 1);
     int config_disabled = !options->LearnCircuitBuildTimeout;
     int dirauth_disabled = options->AuthoritativeDir;
@@ -417,7 +425,7 @@ circuit_build_times_new_consensus_params(circuit_build_times_t *cbt,
    * update if we aren't.
    */
 
-  if (!circuit_build_times_disabled()) {
+  if (!circuit_build_times_disabled(get_options())) {
     num = circuit_build_times_recent_circuit_count(ns);
 
     if (num > 0) {
@@ -493,14 +501,15 @@ static double
 circuit_build_times_get_initial_timeout(void)
 {
   double timeout;
+  const or_options_t *options = get_options();
 
   /*
    * Check if we have LearnCircuitBuildTimeout, and if we don't,
    * always use CircuitBuildTimeout, no questions asked.
    */
-  if (!unit_tests && get_options()->CircuitBuildTimeout) {
-    timeout = get_options()->CircuitBuildTimeout*1000;
-    if (!circuit_build_times_disabled() &&
+  if (!unit_tests && options->CircuitBuildTimeout) {
+    timeout = options->CircuitBuildTimeout*1000;
+    if (!circuit_build_times_disabled(options) &&
         timeout < circuit_build_times_min_timeout()) {
       log_warn(LD_CIRC, "Config CircuitBuildTimeout too low. Setting to %ds",
                circuit_build_times_min_timeout()/1000);
@@ -542,7 +551,7 @@ circuit_build_times_init(circuit_build_times_t *cbt)
    * Check if we really are using adaptive timeouts, and don't keep
    * track of this stuff if not.
    */
-  if (!circuit_build_times_disabled()) {
+  if (!circuit_build_times_disabled(get_options())) {
     cbt->liveness.num_recent_circs =
       circuit_build_times_recent_circuit_count(NULL);
     cbt->liveness.timeouts_after_firsthop =
@@ -906,7 +915,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
   int err = 0;
   circuit_build_times_init(cbt);
 
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     return 0;
   }
 
@@ -1431,7 +1440,7 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt)
 
 #define MAX_TIMEOUT ((int32_t) (INT32_MAX/2))
   /* Check to see if this has happened before. If so, double the timeout
-   * to give people on abysmally bad network connections a shot at access */
+   * to give clients on abysmally bad network connections a shot at access */
   if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) {
     if (cbt->timeout_ms > MAX_TIMEOUT || cbt->close_ms > MAX_TIMEOUT) {
       log_warn(LD_CIRC, "Insanely large circuit build timeout value. "
@@ -1507,7 +1516,7 @@ circuit_build_times_count_close(circuit_build_times_t *cbt,
                                 int did_onehop,
                                 time_t start_time)
 {
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     cbt->close_ms = cbt->timeout_ms
                   = circuit_build_times_get_initial_timeout();
     return 0;
@@ -1538,7 +1547,7 @@ void
 circuit_build_times_count_timeout(circuit_build_times_t *cbt,
                                   int did_onehop)
 {
-  if (circuit_build_times_disabled()) {
+  if (circuit_build_times_disabled(get_options())) {
     cbt->close_ms = cbt->timeout_ms
                   = circuit_build_times_get_initial_timeout();
     return;
@@ -1612,7 +1621,7 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt)
   /*
    * Just return if we aren't using adaptive timeouts
    */
-  if (circuit_build_times_disabled())
+  if (circuit_build_times_disabled(get_options()))
     return;
 
   if (!circuit_build_times_set_timeout_worker(cbt))
diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h
index 72b160983f..8a1dec4bfd 100644
--- a/src/or/circuitstats.h
+++ b/src/or/circuitstats.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,7 +17,10 @@ circuit_build_times_t *get_circuit_build_times_mutable(void);
 double get_circuit_build_close_time_ms(void);
 double get_circuit_build_timeout_ms(void);
 
-int circuit_build_times_disabled(void);
+int circuit_build_times_disabled(const or_options_t *options);
+int circuit_build_times_disabled_(const or_options_t *options,
+                                  int ignore_consensus);
+
 int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt);
 void circuit_build_times_update_state(const circuit_build_times_t *cbt,
                                       or_state_t *state);
diff --git a/src/or/circuituse.c b/src/or/circuituse.c
index 84574cd5b9..9f9d3abf7c 100644
--- a/src/or/circuituse.c
+++ b/src/or/circuituse.c
@@ -1,16 +1,35 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file circuituse.c
- * \brief Launch the right sort of circuits and attach streams to them.
+ * \brief Launch the right sort of circuits and attach the right streams to
+ * them.
+ *
+ * As distinct from circuitlist.c, which manages lookups to find circuits, and
+ * circuitbuild.c, which handles the logistics of circuit construction, this
+ * module keeps track of which streams can be attached to which circuits (in
+ * circuit_get_best()), and attaches streams to circuits (with
+ * circuit_try_attaching_streams(), connection_ap_handshake_attach_circuit(),
+ * and connection_ap_handshake_attach_chosen_circuit() ).
+ *
+ * This module also makes sure that we are building circuits for all of the
+ * predicted ports, using circuit_remove_handled_ports(),
+ * circuit_stream_is_being_handled(), and circuit_build_needed_cirs().  It
+ * handles launching circuits for specific targets using
+ * circuit_launch_by_extend_info().
+ *
+ * This is also where we handle expiring circuits that have been around for
+ * too long without actually completing, along with the circuit_build_timeout
+ * logic in circuitstats.c.
  **/
 
 #include "or.h"
 #include "addressmap.h"
+#include "bridges.h"
 #include "channel.h"
 #include "circpathbias.h"
 #include "circuitbuild.h"
@@ -22,6 +41,7 @@
 #include "connection_edge.h"
 #include "control.h"
 #include "entrynodes.h"
+#include "hs_common.h"
 #include "nodelist.h"
 #include "networkstatus.h"
 #include "policies.h"
@@ -154,8 +174,8 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
     if ((edge_conn->rend_data && !origin_circ->rend_data) ||
         (!edge_conn->rend_data && origin_circ->rend_data) ||
         (edge_conn->rend_data && origin_circ->rend_data &&
-         rend_cmp_service_ids(edge_conn->rend_data->onion_address,
-                              origin_circ->rend_data->onion_address))) {
+         rend_cmp_service_ids(rend_data_get_address(edge_conn->rend_data),
+                            rend_data_get_address(origin_circ->rend_data)))) {
       /* this circ is not for this conn */
       return 0;
     }
@@ -530,16 +550,14 @@ circuit_expire_building(void)
                                       == CPATH_STATE_OPEN;
           log_info(LD_CIRC,
                  "No circuits are opened. Relaxing timeout for circuit %d "
-                 "(a %s %d-hop circuit in state %s with channel state %s). "
-                 "%d guards are live.",
+                 "(a %s %d-hop circuit in state %s with channel state %s).",
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier,
                  circuit_purpose_to_string(victim->purpose),
                  TO_ORIGIN_CIRCUIT(victim)->build_state ?
                    TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len :
                    -1,
                  circuit_state_to_string(victim->state),
-                 channel_state_to_string(victim->n_chan->state),
-                 num_live_entry_guards(0));
+                 channel_state_to_string(victim->n_chan->state));
 
           /* We count the timeout here for CBT, because technically this
            * was a timeout, and the timeout value needs to reset if we
@@ -557,7 +575,7 @@ circuit_expire_building(void)
                  "No circuits are opened. Relaxed timeout for circuit %d "
                  "(a %s %d-hop circuit in state %s with channel state %s) to "
                  "%ldms. However, it appears the circuit has timed out "
-                 "anyway. %d guards are live.",
+                 "anyway.",
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier,
                  circuit_purpose_to_string(victim->purpose),
                  TO_ORIGIN_CIRCUIT(victim)->build_state ?
@@ -565,8 +583,7 @@ circuit_expire_building(void)
                    -1,
                  circuit_state_to_string(victim->state),
                  channel_state_to_string(victim->n_chan->state),
-                 (long)build_close_ms,
-                 num_live_entry_guards(0));
+                 (long)build_close_ms);
       }
     }
 
@@ -688,18 +705,15 @@ circuit_expire_building(void)
       }
     }
 
-    /* If this is a hidden service client circuit which is far enough
-     * along in connecting to its destination, and we haven't already
-     * flagged it as 'timed out', and the user has not told us to
-     * close such circs immediately on timeout, flag it as 'timed out'
-     * so we'll launch another intro or rend circ, but don't mark it
-     * for close yet.
+    /* If this is a hidden service client circuit which is far enough along in
+     * connecting to its destination, and we haven't already flagged it as
+     * 'timed out', flag it so we'll launch another intro or rend circ, but
+     * don't mark it for close yet.
      *
      * (Circs flagged as 'timed out' are given a much longer timeout
      * period above, so we won't close them in the next call to
      * circuit_expire_building.) */
-    if (!(options->CloseHSClientCircuitsImmediatelyOnTimeout) &&
-        !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
+    if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
       switch (victim->purpose) {
       case CIRCUIT_PURPOSE_C_REND_READY:
         /* We only want to spare a rend circ if it has been specified in
@@ -733,8 +747,7 @@ circuit_expire_building(void)
     /* If this is a service-side rendezvous circuit which is far
      * enough along in connecting to its destination, consider sparing
      * it. */
-    if (!(options->CloseHSServiceRendCircuitsImmediatelyOnTimeout) &&
-        !(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) &&
+    if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) &&
         victim->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
       log_info(LD_CIRC,"Marking circ %u (state %d:%s, purpose %d) "
                "as timed-out HS circ; relaunching rendezvous attempt.",
@@ -780,6 +793,25 @@ circuit_expire_building(void)
   } SMARTLIST_FOREACH_END(victim);
 }
 
+/**
+ * Mark for close all circuits that start here, that were built through a
+ * guard we weren't sure if we wanted to use, and that have been waiting
+ * around for way too long.
+ */
+void
+circuit_expire_waiting_for_better_guard(void)
+{
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_origin_circuit_list(),
+                          origin_circuit_t *, circ) {
+    if (TO_CIRCUIT(circ)->marked_for_close)
+      continue;
+    if (circ->guard_state == NULL)
+      continue;
+    if (entry_guard_state_should_expire(circ->guard_state))
+      circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_NONE);
+  } SMARTLIST_FOREACH_END(circ);
+}
+
 /** For debugging #8387: track when we last called
  * circuit_expire_old_circuits_clientside. */
 static time_t last_expired_clientside_circuits = 0;
@@ -1003,8 +1035,117 @@ circuit_stream_is_being_handled(entry_connection_t *conn,
 /** Don't keep more than this many unused open circuits around. */
 #define MAX_UNUSED_OPEN_CIRCUITS 14
 
-/** Figure out how many circuits we have open that are clean. Make
- * sure it's enough for all the upcoming behaviors we predict we'll have.
+/* Return true if a circuit is available for use, meaning that it is open,
+ * clean, usable for new multi-hop connections, and a general purpose origin
+ * circuit.
+ * Accept any kind of circuit, return false if the above conditions are not
+ * met. */
+STATIC int
+circuit_is_available_for_use(const circuit_t *circ)
+{
+  const origin_circuit_t *origin_circ;
+  cpath_build_state_t *build_state;
+
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0; /* We first filter out only origin circuits before doing the
+                 following checks. */
+  if (circ->marked_for_close)
+    return 0; /* Don't mess with marked circs */
+  if (circ->timestamp_dirty)
+    return 0; /* Only count clean circs */
+  if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL)
+    return 0; /* We only pay attention to general purpose circuits.
+                 General purpose circuits are always origin circuits. */
+
+  origin_circ = CONST_TO_ORIGIN_CIRCUIT(circ);
+  if (origin_circ->unusable_for_new_conns)
+    return 0;
+
+  build_state = origin_circ->build_state;
+  if (build_state->onehop_tunnel)
+    return 0;
+
+  return 1;
+}
+
+/* Return true if we need any more exit circuits.
+ * needs_uptime and needs_capacity are set only if we need more exit circuits.
+ * Check if we know of a port that's been requested recently and no circuit
+ * is currently available that can handle it. */
+STATIC int
+needs_exit_circuits(time_t now, int *needs_uptime, int *needs_capacity)
+{
+  return (!circuit_all_predicted_ports_handled(now, needs_uptime,
+                                               needs_capacity) &&
+          router_have_consensus_path() == CONSENSUS_PATH_EXIT);
+}
+
+/* Hidden services need at least this many internal circuits */
+#define SUFFICIENT_UPTIME_INTERNAL_HS_SERVERS 3
+
+/* Return true if we need any more hidden service server circuits.
+ * HS servers only need an internal circuit. */
+STATIC int
+needs_hs_server_circuits(int num_uptime_internal)
+{
+  return (num_rend_services() &&
+          num_uptime_internal < SUFFICIENT_UPTIME_INTERNAL_HS_SERVERS &&
+          router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN);
+}
+
+/* We need at least this many internal circuits for hidden service clients */
+#define SUFFICIENT_INTERNAL_HS_CLIENTS 3
+
+/* We need at least this much uptime for internal circuits for hidden service
+ * clients */
+#define SUFFICIENT_UPTIME_INTERNAL_HS_CLIENTS 2
+
+/* Return true if we need any more hidden service client circuits.
+ * HS clients only need an internal circuit. */
+STATIC int
+needs_hs_client_circuits(time_t now, int *needs_uptime, int *needs_capacity,
+    int num_internal, int num_uptime_internal)
+{
+  int used_internal_recently = rep_hist_get_predicted_internal(now,
+                                                               needs_uptime,
+                                                               needs_capacity);
+  int requires_uptime = num_uptime_internal <
+                        SUFFICIENT_UPTIME_INTERNAL_HS_CLIENTS &&
+                        needs_uptime;
+
+  return (used_internal_recently &&
+         (requires_uptime || num_internal < SUFFICIENT_INTERNAL_HS_CLIENTS) &&
+          router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN);
+}
+
+/* The minimum number of open slots we should keep in order to preemptively
+ * build circuits. */
+#define CBT_MIN_REMAINING_PREEMPTIVE_CIRCUITS 2
+
+/* Check to see if we need more circuits to have a good build timeout. However,
+ * leave a couple slots open so that we can still build circuits preemptively
+ * as needed. */
+#define CBT_MAX_UNUSED_OPEN_CIRCUITS (MAX_UNUSED_OPEN_CIRCUITS - \
+                                      CBT_MIN_REMAINING_PREEMPTIVE_CIRCUITS)
+
+/* Return true if we need more circuits for a good build timeout.
+ * XXXX make the assumption that build timeout streams should be
+ * created whenever we can build internal circuits. */
+STATIC int
+needs_circuits_for_build(int num)
+{
+  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+    if (num < CBT_MAX_UNUSED_OPEN_CIRCUITS &&
+        !circuit_build_times_disabled(get_options()) &&
+        circuit_build_times_needs_circuits_now(get_circuit_build_times())) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/** Determine how many circuits we have open that are clean,
+ * Make sure it's enough for all the upcoming behaviors we predict we'll have.
  * But put an upper bound on the total number of circuits.
  */
 static void
@@ -1016,25 +1157,14 @@ circuit_predict_and_launch_new(void)
   time_t now = time(NULL);
   int flags = 0;
 
-  /* First, count how many of each type of circuit we have already. */
+  /* Count how many of each type of circuit we currently have. */
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    cpath_build_state_t *build_state;
-    origin_circuit_t *origin_circ;
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    if (circ->marked_for_close)
-      continue; /* don't mess with marked circs */
-    if (circ->timestamp_dirty)
-      continue; /* only count clean circs */
-    if (circ->purpose != CIRCUIT_PURPOSE_C_GENERAL)
-      continue; /* only pay attention to general-purpose circs */
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-    if (origin_circ->unusable_for_new_conns)
-      continue;
-    build_state = origin_circ->build_state;
-    if (build_state->onehop_tunnel)
+    if (!circuit_is_available_for_use(circ))
       continue;
+
     num++;
+
+    cpath_build_state_t *build_state = TO_ORIGIN_CIRCUIT(circ)->build_state;
     if (build_state->is_internal)
       num_internal++;
     if (build_state->need_uptime && build_state->is_internal)
@@ -1044,19 +1174,14 @@ circuit_predict_and_launch_new(void)
 
   /* If that's enough, then stop now. */
   if (num >= MAX_UNUSED_OPEN_CIRCUITS)
-    return; /* we already have many, making more probably will hurt */
-
-  /* Second, see if we need any more exit circuits. */
-  /* check if we know of a port that's been requested recently
-   * and no circuit is currently available that can handle it.
-   * Exits (obviously) require an exit circuit. */
-  if (!circuit_all_predicted_ports_handled(now, &port_needs_uptime,
-                                           &port_needs_capacity)
-      && router_have_consensus_path() == CONSENSUS_PATH_EXIT) {
+    return;
+
+  if (needs_exit_circuits(now, &port_needs_uptime, &port_needs_capacity)) {
     if (port_needs_uptime)
       flags |= CIRCLAUNCH_NEED_UPTIME;
     if (port_needs_capacity)
       flags |= CIRCLAUNCH_NEED_CAPACITY;
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d internal), need another exit circ.",
              num, num_internal);
@@ -1064,12 +1189,10 @@ circuit_predict_and_launch_new(void)
     return;
   }
 
-  /* Third, see if we need any more hidden service (server) circuits.
-   * HS servers only need an internal circuit. */
-  if (num_rend_services() && num_uptime_internal < 3
-      && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+  if (needs_hs_server_circuits(num_uptime_internal)) {
     flags = (CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_NEED_UPTIME |
              CIRCLAUNCH_IS_INTERNAL);
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d internal), need another internal "
              "circ for my hidden service.",
@@ -1078,18 +1201,16 @@ circuit_predict_and_launch_new(void)
     return;
   }
 
-  /* Fourth, see if we need any more hidden service (client) circuits.
-   * HS clients only need an internal circuit. */
-  if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime,
-                                      &hidserv_needs_capacity) &&
-      ((num_uptime_internal<2 && hidserv_needs_uptime) ||
-        num_internal<3)
-        && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
+  if (needs_hs_client_circuits(now, &hidserv_needs_uptime,
+                               &hidserv_needs_capacity,
+                               num_internal, num_uptime_internal))
+  {
     if (hidserv_needs_uptime)
       flags |= CIRCLAUNCH_NEED_UPTIME;
     if (hidserv_needs_capacity)
       flags |= CIRCLAUNCH_NEED_CAPACITY;
     flags |= CIRCLAUNCH_IS_INTERNAL;
+
     log_info(LD_CIRC,
              "Have %d clean circs (%d uptime-internal, %d internal), need"
              " another hidden service circ.",
@@ -1098,34 +1219,25 @@ circuit_predict_and_launch_new(void)
     return;
   }
 
-  /* Finally, check to see if we still need more circuits to learn
-   * a good build timeout. But if we're close to our max number we
-   * want, don't do another -- we want to leave a few slots open so
-   * we can still build circuits preemptively as needed.
-   * XXXX make the assumption that build timeout streams should be
-   * created whenever we can build internal circuits. */
-  if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) {
-    if (num < MAX_UNUSED_OPEN_CIRCUITS-2 &&
-        ! circuit_build_times_disabled() &&
-        circuit_build_times_needs_circuits_now(get_circuit_build_times())) {
-      flags = CIRCLAUNCH_NEED_CAPACITY;
-      /* if there are no exits in the consensus, make timeout
-       * circuits internal */
-      if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL)
-        flags |= CIRCLAUNCH_IS_INTERNAL;
+  if (needs_circuits_for_build(num)) {
+    flags = CIRCLAUNCH_NEED_CAPACITY;
+    /* if there are no exits in the consensus, make timeout
+     * circuits internal */
+    if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL)
+      flags |= CIRCLAUNCH_IS_INTERNAL;
+
       log_info(LD_CIRC,
                "Have %d clean circs need another buildtime test circ.", num);
       circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags);
       return;
-    }
   }
 }
 
 /** Build a new test circuit every 5 minutes */
 #define TESTING_CIRCUIT_INTERVAL 300
 
-/** This function is called once a second, if router_have_min_dir_info() is
- * true. Its job is to make sure all services we offer have enough circuits
+/** This function is called once a second, if router_have_minimum_dir_info()
+ * is true. Its job is to make sure all services we offer have enough circuits
  * available. Some services just want enough circuits for current tasks,
  * whereas others want a minimum set of idle circuits hanging around.
  */
@@ -1267,11 +1379,6 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
   tor_fragile_assert();
 }
 
-/** If we haven't yet decided on a good timeout value for circuit
- * building, we close idles circuits aggressively so we can get more
- * data points. */
-#define IDLE_TIMEOUT_WHILE_LEARNING (10*60)
-
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
@@ -1281,21 +1388,15 @@ circuit_expire_old_circuits_clientside(void)
   struct timeval cutoff, now;
 
   tor_gettimeofday(&now);
-  cutoff = now;
   last_expired_clientside_circuits = now.tv_sec;
 
-  if (! circuit_build_times_disabled() &&
-      circuit_build_times_needs_circuits(get_circuit_build_times())) {
-    /* Circuits should be shorter lived if we need more of them
-     * for learning a good build timeout */
-    cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING;
-  } else {
-    cutoff.tv_sec -= get_options()->CircuitIdleTimeout;
-  }
-
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
     if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ))
       continue;
+
+    cutoff = now;
+    cutoff.tv_sec -= TO_ORIGIN_CIRCUIT(circ)->circuit_idle_timeout;
+
     /* If the circuit has been dirty for too long, and there are no streams
      * on it, mark it for close.
      */
@@ -1321,8 +1422,10 @@ circuit_expire_old_circuits_clientside(void)
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
-          log_debug(LD_CIRC,
-                    "Closing circuit that has been unused for %ld msec.",
+          log_info(LD_CIRC,
+                    "Closing circuit "U64_FORMAT
+                    " that has been unused for %ld msec.",
+                    U64_PRINTF_ARG(TO_ORIGIN_CIRCUIT(circ)->global_identifier),
                     tv_mdiff(&circ->timestamp_began, &now));
           circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
         } else if (!TO_ORIGIN_CIRCUIT(circ)->is_ancient) {
@@ -1613,7 +1716,9 @@ circuit_build_failed(origin_circuit_t *circ)
                "Our circuit died before the first hop with no connection");
     }
     if (n_chan_id && !already_marked) {
-      entry_guard_register_connect_status(n_chan_id, 0, 1, time(NULL));
+      /* New guard API: we failed. */
+      if (circ->guard_state)
+        entry_guard_failed(&circ->guard_state);
       /* if there are any one-hop streams waiting on this circuit, fail
        * them now so they can retry elsewhere. */
       connection_ap_fail_onehop(n_chan_id, circ->build_state);
@@ -1874,16 +1979,22 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
             c->state, conn_state_to_string(c->type, c->state));
   }
   tor_assert(ENTRY_TO_CONN(conn)->state == AP_CONN_STATE_CIRCUIT_WAIT);
+
+  /* Will the exit policy of the exit node apply to this stream? */
   check_exit_policy =
       conn->socks_request->command == SOCKS_COMMAND_CONNECT &&
       !conn->use_begindir &&
       !connection_edge_is_rendezvous_stream(ENTRY_TO_EDGE_CONN(conn));
+
+  /* Does this connection want a one-hop circuit? */
   want_onehop = conn->want_onehop;
 
+  /* Do we need a high-uptime circuit? */
   need_uptime = !conn->want_onehop && !conn->use_begindir &&
                 smartlist_contains_int_as_string(options->LongLivedPorts,
                                           conn->socks_request->port);
 
+  /* Do we need an "internal" circuit? */
   if (desired_circuit_purpose != CIRCUIT_PURPOSE_C_GENERAL)
     need_internal = 1;
   else if (conn->use_begindir || conn->want_onehop)
@@ -1891,23 +2002,33 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
   else
     need_internal = 0;
 
-  circ = circuit_get_best(conn, 1, desired_circuit_purpose,
+  /* We now know what kind of circuit we need.  See if there is an
+   * open circuit that we can use for this stream */
+  circ = circuit_get_best(conn, 1 /* Insist on open circuits */,
+                          desired_circuit_purpose,
                           need_uptime, need_internal);
 
   if (circ) {
+    /* We got a circuit that will work for this stream!  We can return it. */
     *circp = circ;
     return 1; /* we're happy */
   }
 
+  /* Okay, there's no circuit open that will work for this stream. Let's
+   * see if there's an in-progress circuit or if we have to launch one */
+
+  /* Do we know enough directory info to build circuits at all? */
   int have_path = have_enough_path_info(!need_internal);
 
   if (!want_onehop && (!router_have_minimum_dir_info() || !have_path)) {
+    /* If we don't have enough directory information, we can't build
+     * multihop circuits.
+     */
     if (!connection_get_by_type(CONN_TYPE_DIR)) {
       int severity = LOG_NOTICE;
-      /* FFFF if this is a tunneled directory fetch, don't yell
-       * as loudly. the user doesn't even know it's happening. */
+      /* Retry some stuff that might help the connection work. */
       if (entry_list_is_constrained(options) &&
-          entries_known_but_down(options)) {
+          guards_retry_optimistic(options)) {
         log_fn(severity, LD_APP|LD_DIR,
                "Application request when we haven't %s. "
                "Optimistically trying known %s again.",
@@ -1915,7 +2036,6 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                "used client functionality lately" :
                "received a consensus with exits",
                options->UseBridges ? "bridges" : "entrynodes");
-        entries_retry_all(options);
       } else if (!options->UseBridges || any_bridge_descriptors_known()) {
         log_fn(severity, LD_APP|LD_DIR,
                "Application request when we haven't %s. "
@@ -1926,14 +2046,16 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
         routerlist_retry_directory_downloads(time(NULL));
       }
     }
-    /* the stream will be dealt with when router_have_minimum_dir_info becomes
-     * 1, or when all directory attempts fail and directory_all_unreachable()
+    /* Since we didn't have enough directory info, we can't attach now.  The
+     * stream will be dealt with when router_have_minimum_dir_info becomes 1,
+     * or when all directory attempts fail and directory_all_unreachable()
      * kills it.
      */
     return 0;
   }
 
-  /* Do we need to check exit policy? */
+  /* Check whether the exit policy of the chosen exit, or the exit policies
+   * of _all_ nodes, would forbid this node. */
   if (check_exit_policy) {
     if (!conn->chosen_exit_name) {
       struct in_addr in;
@@ -1974,16 +2096,25 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     }
   }
 
-  /* is one already on the way? */
-  circ = circuit_get_best(conn, 0, desired_circuit_purpose,
+  /* Now, check whether there already a circuit on the way that could handle
+   * this stream. This check matches the one above, but this time we
+   * do not require that the circuit will work. */
+  circ = circuit_get_best(conn, 0 /* don't insist on open circuits */,
+                          desired_circuit_purpose,
                           need_uptime, need_internal);
   if (circ)
     log_debug(LD_CIRC, "one on the way!");
+
   if (!circ) {
+    /* No open or in-progress circuit could handle this stream!  We
+     * will have to launch one!
+     */
+
+    /* The chosen exit node, if there is one. */
     extend_info_t *extend_info=NULL;
-    uint8_t new_circ_purpose;
     const int n_pending = count_pending_general_client_circuits();
 
+    /* Do we have too many pending circuits? */
     if (n_pending >= options->MaxClientCircuitsPending) {
       static ratelim_t delay_limit = RATELIM_INIT(10*60);
       char *m;
@@ -1997,6 +2128,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       return 0;
     }
 
+    /* If this is a hidden service trying to start an introduction point,
+     * handle that case. */
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
       /* need to pick an intro point */
       rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
@@ -2005,7 +2138,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       if (!extend_info) {
         log_info(LD_REND,
                  "No intro points for '%s': re-fetching service descriptor.",
-                 safe_str_client(rend_data->onion_address));
+                 safe_str_client(rend_data_get_address(rend_data)));
         rend_client_refetch_v2_renddesc(rend_data);
         connection_ap_mark_as_non_pending_circuit(conn);
         ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_RENDDESC_WAIT;
@@ -2013,7 +2146,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       }
       log_info(LD_REND,"Chose %s as intro point for '%s'.",
                extend_info_describe(extend_info),
-               safe_str_client(rend_data->onion_address));
+               safe_str_client(rend_data_get_address(rend_data)));
     }
 
     /* If we have specified a particular exit node for our
@@ -2034,12 +2167,16 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                      "Discarding this circuit.", conn->chosen_exit_name);
             return -1;
           }
-        } else {
+        } else  { /* ! (r && node_has_descriptor(r)) */
           log_debug(LD_DIR, "considering %d, %s",
                     want_onehop, conn->chosen_exit_name);
           if (want_onehop && conn->chosen_exit_name[0] == '$') {
             /* We're asking for a one-hop circuit to a router that
              * we don't have a routerinfo about. Make up an extend_info. */
+            /* XXX prop220: we need to make chosen_exit_name able to
+             * encode both key formats. This is not absolutely critical
+             * since this is just for one-hop circuits, but we should
+             * still get it done */
             char digest[DIGEST_LEN];
             char *hexdigest = conn->chosen_exit_name+1;
             tor_addr_t addr;
@@ -2054,10 +2191,13 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
                        escaped_safe_str_client(conn->socks_request->address));
               return -1;
             }
+            /* XXXX prop220 add a workaround for ed25519 ID below*/
             extend_info = extend_info_new(conn->chosen_exit_name+1,
-                                          digest, NULL, NULL, &addr,
-                                          conn->socks_request->port);
-          } else {
+                                          digest,
+                                          NULL, /* Ed25519 ID */
+                                          NULL, NULL, /* onion keys */
+                                          &addr, conn->socks_request->port);
+          } else { /* ! (want_onehop && conn->chosen_exit_name[0] == '$') */
             /* We will need an onion key for the router, and we
              * don't have one. Refuse or relax requirements. */
             log_fn(opt ? LOG_INFO : LOG_WARN, LD_APP,
@@ -2075,8 +2215,10 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           }
         }
       }
-    }
+    } /* Done checking for general circutis with chosen exits. */
 
+    /* What purpose do we need to launch this circuit with? */
+    uint8_t new_circ_purpose;
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_REND_JOINED)
       new_circ_purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND;
     else if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)
@@ -2085,6 +2227,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       new_circ_purpose = desired_circuit_purpose;
 
 #ifdef ENABLE_TOR2WEB_MODE
+    /* If tor2Web is on, then hidden service requests should be one-hop.
+     */
     if (options->Tor2webMode &&
         (new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND ||
          new_circ_purpose == CIRCUIT_PURPOSE_C_INTRODUCING)) {
@@ -2092,6 +2236,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
     }
 #endif
 
+    /* Determine what kind of a circuit to launch, and actually launch it. */
     {
       int flags = CIRCLAUNCH_NEED_CAPACITY;
       if (want_onehop) flags |= CIRCLAUNCH_ONEHOP_TUNNEL;
@@ -2103,6 +2248,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
 
     extend_info_free(extend_info);
 
+    /* Now trigger things that need to happen when we launch circuits */
+
     if (desired_circuit_purpose == CIRCUIT_PURPOSE_C_GENERAL) {
       /* We just caused a circuit to get built because of this stream.
        * If this stream has caused a _lot_ of circuits to be built, that's
@@ -2126,6 +2273,10 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
       }
     }
   } /* endif (!circ) */
+
+  /* We either found a good circuit, or launched a new circuit, or failed to
+   * do so. Report success, and delay. */
+
   if (circ) {
     /* Mark the circuit with the isolation fields for this connection.
      * When the circuit arrives, we'll clear these flags: this is
@@ -2325,7 +2476,9 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn,
 
   pathbias_count_use_attempt(circ);
 
+  /* Now, actually link the connection. */
   link_apconn_to_circ(conn, circ, cpath);
+
   tor_assert(conn->socks_request);
   if (conn->socks_request->command == SOCKS_COMMAND_CONNECT) {
     if (!conn->use_begindir)
@@ -2340,12 +2493,11 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn,
   return 1;
 }
 
-/** Try to find a safe live circuit for CONN_TYPE_AP connection conn. If
- * we don't find one: if conn cannot be handled by any known nodes,
- * warn and return -1 (conn needs to die, and is maybe already marked);
- * else launch new circuit (if necessary) and return 0.
- * Otherwise, associate conn with a safe live circuit, do the
- * right next step, and return 1.
+/** Try to find a safe live circuit for stream <b>conn</b>.  If we find one,
+ * attach the stream, send appropriate cells, and return 1.  Otherwise,
+ * try to launch new circuit(s) for the stream.  If we can launch
+ * circuits, return 0.  Otherwise, if we simply can't proceed with
+ * this stream, return -1. (conn needs to die, and is maybe already marked).
  */
 /* XXXX this function should mark for close whenever it returns -1;
  * its callers shouldn't have to worry about that. */
@@ -2364,6 +2516,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
 
   conn_age = (int)(time(NULL) - base_conn->timestamp_created);
 
+  /* Is this connection so old that we should give up on it? */
   if (conn_age >= get_options()->SocksTimeout) {
     int severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port) ?
       LOG_INFO : LOG_NOTICE;
@@ -2374,12 +2527,14 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
     return -1;
   }
 
+  /* We handle "general" (non-onion) connections much more straightforwardly.
+   */
   if (!connection_edge_is_rendezvous_stream(ENTRY_TO_EDGE_CONN(conn))) {
     /* we're a general conn */
     origin_circuit_t *circ=NULL;
 
     /* Are we linked to a dir conn that aims to fetch a consensus?
-     * We check here because this conn might no longer be needed. */
+     * We check here because the conn might no longer be needed. */
     if (base_conn->linked_conn &&
         base_conn->linked_conn->type == CONN_TYPE_DIR &&
         base_conn->linked_conn->purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
@@ -2397,6 +2552,9 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
+    /* If we have a chosen exit, we need to use a circuit that's
+     * open to that exit. See what exit we meant, and whether we can use it.
+     */
     if (conn->chosen_exit_name) {
       const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1);
       int opt = conn->chosen_exit_optional;
@@ -2410,6 +2568,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
                "Requested exit point '%s' is not known. %s.",
                conn->chosen_exit_name, opt ? "Trying others" : "Closing");
         if (opt) {
+          /* If we are allowed to ignore the .exit request, do so */
           conn->chosen_exit_optional = 0;
           tor_free(conn->chosen_exit_name);
           return 0;
@@ -2422,6 +2581,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
                "would refuse request. %s.",
                conn->chosen_exit_name, opt ? "Trying others" : "Closing");
         if (opt) {
+          /* If we are allowed to ignore the .exit request, do so */
           conn->chosen_exit_optional = 0;
           tor_free(conn->chosen_exit_name);
           return 0;
@@ -2430,20 +2590,25 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn)
       }
     }
 
-    /* find the circuit that we should use, if there is one. */
+    /* Find the circuit that we should use, if there is one. Otherwise
+     * launch it. */
     retval = circuit_get_open_circ_or_launch(
         conn, CIRCUIT_PURPOSE_C_GENERAL, &circ);
-    if (retval < 1) // XXXX++ if we totally fail, this still returns 0 -RD
+    if (retval < 1) {
+      /* We were either told "-1" (complete failure) or 0 (circuit in
+       * progress); we can't attach this stream yet. */
       return retval;
+    }
 
     log_debug(LD_APP|LD_CIRC,
               "Attaching apconn to circ %u (stream %d sec old).",
               (unsigned)circ->base_.n_circ_id, conn_age);
-    /* print the circ's path, so people can figure out which circs are
+    /* print the circ's path, so clients can figure out which circs are
      * sucking. */
     circuit_log_path(LOG_INFO,LD_APP|LD_CIRC,circ);
 
-    /* We have found a suitable circuit for our conn. Hurray. */
+    /* We have found a suitable circuit for our conn. Hurray.  Do
+     * the attachment. */
     return connection_ap_handshake_attach_chosen_circuit(conn, circ, NULL);
 
   } else { /* we're a rendezvous conn */
diff --git a/src/or/circuituse.h b/src/or/circuituse.h
index 5973978c45..ad4c214a3b 100644
--- a/src/or/circuituse.h
+++ b/src/or/circuituse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_CIRCUITUSE_H
 
 void circuit_expire_building(void);
+void circuit_expire_waiting_for_better_guard(void);
 void circuit_remove_handled_ports(smartlist_t *needed_ports);
 int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port,
                                     int min);
@@ -59,5 +60,25 @@ int hostname_in_track_host_exits(const or_options_t *options,
                                  const char *address);
 void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ);
 
+#ifdef TOR_UNIT_TESTS
+/* Used only by circuituse.c and test_circuituse.c */
+
+STATIC int circuit_is_available_for_use(const circuit_t *circ);
+
+STATIC int needs_exit_circuits(time_t now,
+                               int *port_needs_uptime,
+                               int *port_needs_capacity);
+STATIC int needs_hs_server_circuits(int num_uptime_internal);
+
+STATIC int needs_hs_client_circuits(time_t now,
+                                    int *needs_uptime,
+                                    int *needs_capacity,
+                                    int num_internal,
+                                    int num_uptime_internal);
+
+STATIC int needs_circuits_for_build(int num);
+
+#endif
+
 #endif
 
diff --git a/src/or/command.c b/src/or/command.c
index 3f5386c5a3..be912dffa7 100644
--- a/src/or/command.c
+++ b/src/or/command.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -339,10 +339,13 @@ command_process_create_cell(cell_t *cell, channel_t *chan)
     return;
   }
 
+  if (connection_or_digest_is_known_relay(chan->identity_digest)) {
+    rep_hist_note_circuit_handshake_requested(create_cell->handshake_type);
+  }
+
   if (create_cell->handshake_type != ONION_HANDSHAKE_TYPE_FAST) {
     /* hand it off to the cpuworkers, and then return. */
-    if (connection_or_digest_is_known_relay(chan->identity_digest))
-      rep_hist_note_circuit_handshake_requested(create_cell->handshake_type);
+
     if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) {
       log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing.");
       circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT);
diff --git a/src/or/command.h b/src/or/command.h
index 12cda6a463..5079d42e75 100644
--- a/src/or/command.h
+++ b/src/or/command.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/config.c b/src/or/config.c
index 75e4065859..cac4ce8125 100644
--- a/src/or/config.c
+++ b/src/or/config.c
@@ -1,16 +1,66 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file config.c
- * \brief Code to parse and interpret configuration files.
+ * \brief Code to interpret the user's configuration of Tor.
+ *
+ * This module handles torrc configuration file, including parsing it,
+ * combining it with torrc.defaults and the command line, allowing
+ * user changes to it (via editing and SIGHUP or via the control port),
+ * writing it back to disk (because of SAVECONF from the control port),
+ * and -- most importantly, acting on it.
+ *
+ * The module additionally has some tools for manipulating and
+ * inspecting values that are calculated as a result of the
+ * configured options.
+ *
+ * <h3>How to add  new options</h3>
+ *
+ * To add new items to the torrc, there are a minimum of three places to edit:
+ * <ul>
+ *   <li>The or_options_t structure in or.h, where the options are stored.
+ *   <li>The option_vars_ array below in this module, which configures
+ *       the names of the torrc options, their types, their multiplicities,
+ *       and their mappings to fields in or_options_t.
+ *   <li>The manual in doc/tor.1.txt, to document what the new option
+ *       is, and how it works.
+ * </ul>
+ *
+ * Additionally, you might need to edit these places too:
+ * <ul>
+ *   <li>options_validate() below, in case you want to reject some possible
+ *       values of the new configuration option.
+ *   <li>options_transition_allowed() below, in case you need to
+ *       forbid some or all changes in the option while Tor is
+ *       running.
+ *   <li>options_transition_affects_workers(), in case changes in the option
+ *       might require Tor to relaunch or reconfigure its worker threads.
+ *   <li>options_transition_affects_descriptor(), in case changes in the
+ *       option might require a Tor relay to build and publish a new server
+ *       descriptor.
+ *   <li>options_act() and/or options_act_reversible(), in case there's some
+ *       action that needs to be taken immediately based on the option's
+ *       value.
+ * </ul>
+ *
+ * <h3>Changing the value of an option</h3>
+ *
+ * Because of the SAVECONF command from the control port, it's a bad
+ * idea to change the value of any user-configured option in the
+ * or_options_t.  If you want to sometimes do this anyway, we recommend
+ * that you create a secondary field in or_options_t; that you have the
+ * user option linked only to the secondary field; that you use the
+ * secondary field to initialize the one that Tor actually looks at; and that
+ * you use the one Tor looks as the one that you modify.
  **/
 
 #define CONFIG_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "compat.h"
 #include "addressmap.h"
 #include "channel.h"
@@ -19,10 +69,12 @@
 #include "circuitmux.h"
 #include "circuitmux_ewma.h"
 #include "circuitstats.h"
+#include "compress.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
 #include "connection_or.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "confparse.h"
 #include "cpuworker.h"
@@ -50,7 +102,6 @@
 #include "statefile.h"
 #include "transports.h"
 #include "ext_orport.h"
-#include "torgzip.h"
 #ifdef _WIN32
 #include <shlobj.h>
 #endif
@@ -134,8 +185,17 @@ static config_abbrev_t option_abbrevs_[] = {
 /** An entry for config_vars: "The option <b>name</b> is obsolete." */
 #define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
 
-#define VPORT(member,conftype,initvalue)                                    \
-  VAR(#member, conftype, member ## _lines, initvalue)
+/**
+ * Macro to declare *Port options.  Each one comes in three entries.
+ * For example, most users should use "SocksPort" to configure the
+ * socks port, but TorBrowser wants to use __SocksPort so that it
+ * isn't stored by SAVECONF.  The SocksPortLines virtual option is
+ * used to query both options from the controller.
+ */
+#define VPORT(member)                                           \
+  VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
+  VAR(#member, LINELIST_S, member ## _lines, NULL),             \
+  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
 
 /** Array of configuration options.  Until we disallow nonstandard
  * abbreviations, order is significant, since the first matching option will
@@ -147,10 +207,10 @@ static config_var_t option_vars_[] = {
   V(AccountingStart,             STRING,   NULL),
   V(Address,                     STRING,   NULL),
   V(AllowDotExit,                BOOL,     "0"),
-  V(AllowInvalidNodes,           CSV,      "middle,rendezvous"),
+  OBSOLETE("AllowInvalidNodes"),
   V(AllowNonRFC953Hostnames,     BOOL,     "0"),
-  V(AllowSingleHopCircuits,      BOOL,     "0"),
-  V(AllowSingleHopExits,         BOOL,     "0"),
+  OBSOLETE("AllowSingleHopCircuits"),
+  OBSOLETE("AllowSingleHopExits"),
   V(AlternateBridgeAuthority,    LINELIST, NULL),
   V(AlternateDirAuthority,       LINELIST, NULL),
   OBSOLETE("AlternateHSAuthority"),
@@ -163,14 +223,14 @@ static config_var_t option_vars_[] = {
   V(AuthDirInvalidCCs,           CSV,      ""),
   V(AuthDirFastGuarantee,        MEMUNIT,  "100 KB"),
   V(AuthDirGuardBWGuarantee,     MEMUNIT,  "2 MB"),
-  V(AuthDirPinKeys,              BOOL,     "0"),
+  V(AuthDirPinKeys,              BOOL,     "1"),
   V(AuthDirReject,               LINELIST, NULL),
   V(AuthDirRejectCCs,            CSV,      ""),
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
   V(AuthDirListBadExits,         BOOL,     "0"),
   V(AuthDirMaxServersPerAddr,    UINT,     "2"),
-  V(AuthDirMaxServersPerAuthAddr,UINT,     "5"),
+  OBSOLETE("AuthDirMaxServersPerAuthAddr"),
   V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
   V(AutomapHostsOnResolve,       BOOL,     "0"),
@@ -184,9 +244,11 @@ static config_var_t option_vars_[] = {
   V(BridgeRecordUsageByCountry,  BOOL,     "1"),
   V(BridgeRelay,                 BOOL,     "0"),
   V(CellStatistics,              BOOL,     "0"),
+  V(PaddingStatistics,           BOOL,     "1"),
   V(LearnCircuitBuildTimeout,    BOOL,     "1"),
   V(CircuitBuildTimeout,         INTERVAL, "0"),
-  V(CircuitIdleTimeout,          INTERVAL, "1 hour"),
+  OBSOLETE("CircuitIdleTimeout"),
+  V(CircuitsAvailableTimeout,    INTERVAL, "0"),
   V(CircuitStreamTimeout,        INTERVAL, "0"),
   V(CircuitPriorityHalflife,     DOUBLE,  "-100.0"), /*negative:'Use default'*/
   V(ClientDNSRejectInternalAddresses, BOOL,"1"),
@@ -203,8 +265,8 @@ static config_var_t option_vars_[] = {
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
   V(ContactInfo,                 STRING,   NULL),
-  V(ControlListenAddress,        LINELIST, NULL),
-  VPORT(ControlPort,                 LINELIST, NULL),
+  OBSOLETE("ControlListenAddress"),
+  VPORT(ControlPort),
   V(ControlPortFileGroupReadable,BOOL,     "0"),
   V(ControlPortWriteToFile,      FILENAME, NULL),
   V(ControlSocket,               LINELIST, NULL),
@@ -220,9 +282,9 @@ static config_var_t option_vars_[] = {
   V(DisableNetwork,              BOOL,     "0"),
   V(DirAllowPrivateAddresses,    BOOL,     "0"),
   V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"),
-  V(DirListenAddress,            LINELIST, NULL),
+  OBSOLETE("DirListenAddress"),
   V(DirPolicy,                   LINELIST, NULL),
-  VPORT(DirPort,                     LINELIST, NULL),
+  VPORT(DirPort),
   V(DirPortFrontPage,            FILENAME, NULL),
   VAR("DirReqStatistics",        BOOL,     DirReqStatistics_option, "1"),
   VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
@@ -240,8 +302,8 @@ static config_var_t option_vars_[] = {
   OBSOLETE("DisableIOCP"),
   OBSOLETE("DisableV2DirectoryInfo_"),
   OBSOLETE("DynamicDHGroups"),
-  VPORT(DNSPort,                     LINELIST, NULL),
-  V(DNSListenAddress,            LINELIST, NULL),
+  VPORT(DNSPort),
+  OBSOLETE("DNSListenAddress"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
   V(DoSCircuitCreationMinConnections,      UINT, "0"),
@@ -265,7 +327,7 @@ static config_var_t option_vars_[] = {
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
   V(ExcludeNodes,                ROUTERSET, NULL),
   V(ExcludeExitNodes,            ROUTERSET, NULL),
-  V(ExcludeSingleHopRelays,      BOOL,     "1"),
+  OBSOLETE("ExcludeSingleHopRelays"),
   V(ExitNodes,                   ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
@@ -273,17 +335,19 @@ static config_var_t option_vars_[] = {
   V(ExitPortStatistics,          BOOL,     "0"),
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExitRelay,                   AUTOBOOL, "auto"),
-  VPORT(ExtORPort,               LINELIST, NULL),
+  VPORT(ExtORPort),
   V(ExtORPortCookieAuthFile,     STRING,   NULL),
   V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
+  V(ExtendByEd25519ID,           AUTOBOOL, "auto"),
   V(FallbackDir,                 LINELIST, NULL),
+
   V(UseDefaultFallbackDirs,      BOOL,     "1"),
 
   OBSOLETE("FallbackNetworkstatusFile"),
   V(FascistFirewall,             BOOL,     "0"),
   V(FirewallPorts,               CSV,      ""),
-  V(FastFirstHopPK,              AUTOBOOL, "auto"),
+  OBSOLETE("FastFirstHopPK"),
   V(FetchDirInfoEarly,           BOOL,     "0"),
   V(FetchDirInfoExtraEarly,      BOOL,     "0"),
   V(FetchServerDescriptors,      BOOL,     "1"),
@@ -318,10 +382,10 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreams",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
-  V(HiddenServiceStatistics,     BOOL,     "1"),
+  VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
-  V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"),
-  V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"),
+  OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"),
+  OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"),
   V(HiddenServiceSingleHopMode,  BOOL,     "0"),
   V(HiddenServiceNonAnonymousMode,BOOL,    "0"),
   V(HTTPProxy,                   STRING,   NULL),
@@ -350,27 +414,30 @@ static config_var_t option_vars_[] = {
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
   V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
   V(MaxOnionQueueDelay,          MSEC_INTERVAL, "1750 msec"),
   V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"),
   V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"),
-  V(MyFamily,                    STRING,   NULL),
+  VAR("MyFamily",                LINELIST, MyFamily_lines,       NULL),
   V(NewCircuitPeriod,            INTERVAL, "30 seconds"),
   OBSOLETE("NamingAuthoritativeDirectory"),
-  V(NATDListenAddress,           LINELIST, NULL),
-  VPORT(NATDPort,                    LINELIST, NULL),
+  OBSOLETE("NATDListenAddress"),
+  VPORT(NATDPort),
   V(Nickname,                    STRING,   NULL),
-  V(PredictedPortsRelevanceTime,  INTERVAL, "1 hour"),
-  V(WarnUnsafeSocks,              BOOL,     "1"),
+  OBSOLETE("PredictedPortsRelevanceTime"),
+  OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
   V(NumCPUs,                     UINT,     "0"),
   V(NumDirectoryGuards,          UINT,     "0"),
   V(NumEntryGuards,              UINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
-  V(ORListenAddress,             LINELIST, NULL),
-  VPORT(ORPort,                      LINELIST, NULL),
+  OBSOLETE("ORListenAddress"),
+  VPORT(ORPort),
   V(OutboundBindAddress,         LINELIST,   NULL),
+  V(OutboundBindAddressOR,       LINELIST,   NULL),
+  V(OutboundBindAddressExit,     LINELIST,   NULL),
 
   OBSOLETE("PathBiasDisableRate"),
   V(PathBiasCircThreshold,       INT,      "-1"),
@@ -416,6 +483,8 @@ static config_var_t option_vars_[] = {
   V(RecommendedClientVersions,   LINELIST, NULL),
   V(RecommendedServerVersions,   LINELIST, NULL),
   V(RecommendedPackages,         LINELIST, NULL),
+  V(ReducedConnectionPadding,    BOOL,     "0"),
+  V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
@@ -439,9 +508,9 @@ static config_var_t option_vars_[] = {
   V(SchedulerHighWaterMark__,    MEMUNIT,  "101 MB"),
   V(SchedulerMaxFlushCells__,    UINT,     "1000"),
   V(ShutdownWaitLength,          INTERVAL, "30 seconds"),
-  V(SocksListenAddress,          LINELIST, NULL),
+  OBSOLETE("SocksListenAddress"),
   V(SocksPolicy,                 LINELIST, NULL),
-  VPORT(SocksPort,                   LINELIST, NULL),
+  VPORT(SocksPort),
   V(SocksTimeout,                INTERVAL, "2 minutes"),
   V(SSLKeyLifetime,              INTERVAL, "0"),
   OBSOLETE("StrictEntryNodes"),
@@ -452,23 +521,24 @@ static config_var_t option_vars_[] = {
   V(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
   V(Tor2webMode,                 BOOL,     "0"),
   V(Tor2webRendezvousPoints,      ROUTERSET, NULL),
-  V(TLSECGroup,                  STRING,   NULL),
+  OBSOLETE("TLSECGroup"),
   V(TrackHostExits,              CSV,      NULL),
   V(TrackHostExitsExpire,        INTERVAL, "30 minutes"),
-  V(TransListenAddress,          LINELIST, NULL),
-  VPORT(TransPort,                   LINELIST, NULL),
+  OBSOLETE("TransListenAddress"),
+  VPORT(TransPort),
   V(TransProxyType,              STRING,   "default"),
   OBSOLETE("TunnelDirConns"),
   V(UpdateBridgesFromAuthority,  BOOL,     "0"),
   V(UseBridges,                  BOOL,     "0"),
   VAR("UseEntryGuards",          BOOL,     UseEntryGuards_option, "1"),
-  V(UseEntryGuardsAsDirGuards,   BOOL,     "1"),
+  OBSOLETE("UseEntryGuardsAsDirGuards"),
   V(UseGuardFraction,            AUTOBOOL, "auto"),
   V(UseMicrodescriptors,         AUTOBOOL, "auto"),
   OBSOLETE("UseNTorHandshake"),
   V(User,                        STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
   V(AuthDirSharedRandomness,     BOOL,     "1"),
+  V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
   OBSOLETE("V2AuthoritativeDirectory"),
   VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir,   "0"),
@@ -510,11 +580,13 @@ static config_var_t option_vars_[] = {
                                  "10800, 21600, 43200"),
   /* With the ClientBootstrapConsensus*Download* below:
    * Clients with only authorities will try:
-   *  - 3 authorities over 10 seconds, then wait 60 minutes.
+   *  - at least 3 authorities over 10 seconds, then exponentially backoff,
+   *    with the next attempt 3-21 seconds later,
    * Clients with authorities and fallbacks will try:
-   *  - 2 authorities and 4 fallbacks over 21 seconds, then wait 60 minutes.
+   *  - at least 2 authorities and 4 fallbacks over 21 seconds, then
+   *    exponentially backoff, with the next attempts 4-33 seconds later,
    * Clients will also retry when an application request arrives.
-   * After a number of failed reqests, clients retry every 3 days + 1 hour.
+   * After a number of failed requests, clients retry every 3 days + 1 hour.
    *
    * Clients used to try 2 authorities over 10 seconds, then wait for
    * 60 minutes or an application request.
@@ -564,7 +636,6 @@ static const config_var_t testing_tor_network_defaults[] = {
   V(EnforceDistinctSubnets,      BOOL,     "0"),
   V(AssumeReachable,             BOOL,     "1"),
   V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(AuthDirMaxServersPerAuthAddr,UINT,     "0"),
   V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL,
     "0, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"),
   V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL,
@@ -620,35 +691,8 @@ static const config_deprecation_t option_deprecation_notes_[] = {
   /* Deprecated since 0.2.9.2-alpha... */
   { "AllowDotExit", "Unrestricted use of the .exit notation can be used for "
     "a wide variety of application-level attacks." },
-  { "AllowInvalidNodes", "There is no reason to enable this option; at best "
-    "it will make you easier to track." },
-  { "AllowSingleHopCircuits", "Almost no relays actually allow single-hop "
-    "exits, making this option pointless." },
-  { "AllowSingleHopExits", "Turning this on will make your relay easier "
-    "to abuse." },
   { "ClientDNSRejectInternalAddresses", "Turning this on makes your client "
     "easier to fingerprint, and may open you to esoteric attacks." },
-  { "ExcludeSingleHopRelays", "Turning it on makes your client easier to "
-    "fingerprint." },
-  { "FastFirstHopPK", "Changing this option does not make your client more "
-    "secure, but does make it easier to fingerprint." },
-  { "CloseHSClientCircuitsImmediatelyOnTimeout", "This option makes your "
-    "client easier to fingerprint." },
-  { "CloseHSServiceRendCircuitsImmediatelyOnTimeout", "This option makes "
-    "your hidden services easier to fingerprint." },
-  { "WarnUnsafeSocks", "Changing this option makes it easier for you "
-    "to accidentally lose your anonymity by leaking DNS information" },
-  { "TLSECGroup", "The default is a nice secure choice; the other option "
-    "is less secure." },
-  { "ControlListenAddress", "Use ControlPort instead." },
-  { "DirListenAddress", "Use DirPort instead, possibly with the "
-    "NoAdvertise sub-option" },
-  { "DNSListenAddress", "Use DNSPort instead." },
-  { "SocksListenAddress", "Use SocksPort instead." },
-  { "TransListenAddress", "Use TransPort instead." },
-  { "NATDListenAddress", "Use NATDPort instead." },
-  { "ORListenAddress", "Use ORPort instead, possibly with the "
-    "NoAdvertise sub-option" },
   /* End of options deprecated since 0.2.9.2-alpha. */
 
   { NULL, NULL }
@@ -665,7 +709,9 @@ static int options_transition_affects_workers(
       const or_options_t *old_options, const or_options_t *new_options);
 static int options_transition_affects_descriptor(
       const or_options_t *old_options, const or_options_t *new_options);
-static int check_nickname_list(char **lst, const char *name, char **msg);
+static int normalize_nickname_list(config_line_t **normalized_out,
+                                   const config_line_t *lst, const char *name,
+                                   char **msg);
 static char *get_bindaddr_from_transport_listen_line(const char *line,
                                                      const char *transport);
 static int parse_ports(or_options_t *options, int validate_only,
@@ -802,7 +848,7 @@ set_options(or_options_t *new_val, char **msg)
             tor_free(line);
           }
         } else {
-          smartlist_add(elements, tor_strdup(options_format.vars[i].name));
+          smartlist_add_strdup(elements, options_format.vars[i].name);
           smartlist_add(elements, NULL);
         }
       }
@@ -873,6 +919,7 @@ or_options_free(or_options_t *options)
   tor_free(options->BridgePassword_AuthDigest_);
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
+  config_free_lines(options->MyFamily);
   config_free(&options_format, options);
 }
 
@@ -1482,21 +1529,32 @@ get_effective_bwburst(const or_options_t *options)
   return (uint32_t)bw;
 }
 
-/** Return True if any changes from <b>old_options</b> to
- * <b>new_options</b> needs us to refresh our TLS context. */
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the guard susbsystem.
+ */
 static int
-options_transition_requires_fresh_tls_context(const or_options_t *old_options,
-                                              const or_options_t *new_options)
+options_transition_affects_guards(const or_options_t *old,
+                                  const or_options_t *new)
 {
-  tor_assert(new_options);
-
-  if (!old_options)
-    return 0;
-
-  if (!opt_streq(old_options->TLSECGroup, new_options->TLSECGroup))
-    return 1;
-
-  return 0;
+  /* NOTE: Make sure this function stays in sync with
+   * entry_guards_set_filtered_flags */
+
+  tor_assert(old);
+  tor_assert(new);
+
+  return
+    (old->UseEntryGuards != new->UseEntryGuards ||
+     old->UseBridges != new->UseBridges ||
+     old->ClientUseIPv4 != new->ClientUseIPv4 ||
+     old->ClientUseIPv6 != new->ClientUseIPv6 ||
+     old->FascistFirewall != new->FascistFirewall ||
+     !routerset_equal(old->ExcludeNodes, new->ExcludeNodes) ||
+     !routerset_equal(old->EntryNodes, new->EntryNodes) ||
+     !smartlist_strings_eq(old->FirewallPorts, new->FirewallPorts) ||
+     !config_lines_eq(old->Bridges, new->Bridges) ||
+     !config_lines_eq(old->ReachableORAddresses, new->ReachableORAddresses) ||
+     !config_lines_eq(old->ReachableDirAddresses, new->ReachableDirAddresses));
 }
 
 /** Fetch the active option list, and take actions based on it. All of the
@@ -1518,6 +1576,8 @@ options_act(const or_options_t *old_options)
   const int transition_affects_workers =
     old_options && options_transition_affects_workers(old_options, options);
   int old_ewma_enabled;
+  const int transition_affects_guards =
+    old_options && options_transition_affects_guards(old_options, options);
 
   /* disable ptrace and later, other basic debugging techniques */
   {
@@ -1687,13 +1747,6 @@ options_act(const or_options_t *old_options)
       log_warn(LD_BUG,"Error initializing keys; exiting");
       return -1;
     }
-  } else if (old_options &&
-             options_transition_requires_fresh_tls_context(old_options,
-                                                           options)) {
-    if (router_initialize_tls_context() < 0) {
-      log_warn(LD_BUG,"Error initializing TLS context.");
-      return -1;
-    }
   }
 
   /* Write our PID to the PID file. If we do not have write permissions we
@@ -1714,6 +1767,15 @@ options_act(const or_options_t *old_options)
     return -1;
   }
 
+  if (server_mode(options)) {
+    static int cdm_initialized = 0;
+    if (cdm_initialized == 0) {
+      cdm_initialized = 1;
+      consdiffmgr_configure(NULL);
+      consdiffmgr_validate();
+    }
+  }
+
   if (init_control_cookie_authentication(options->CookieAuthentication) < 0) {
     log_warn(LD_CONFIG,"Error creating control cookie authentication file.");
     return -1;
@@ -1794,6 +1856,7 @@ options_act(const or_options_t *old_options)
   if (old_options) {
     int revise_trackexithosts = 0;
     int revise_automap_entries = 0;
+    int abandon_circuits = 0;
     if ((options->UseEntryGuards && !old_options->UseEntryGuards) ||
         options->UseBridges != old_options->UseBridges ||
         (options->UseBridges &&
@@ -1810,6 +1873,16 @@ options_act(const or_options_t *old_options)
                "Changed to using entry guards or bridges, or changed "
                "preferred or excluded node lists. "
                "Abandoning previous circuits.");
+      abandon_circuits = 1;
+    }
+
+    if (transition_affects_guards) {
+      if (guards_update_all()) {
+        abandon_circuits = 1;
+      }
+    }
+
+    if (abandon_circuits) {
       circuit_mark_all_unused_circs();
       circuit_mark_all_dirty_circs_as_unusable();
       revise_trackexithosts = 1;
@@ -1840,7 +1913,7 @@ options_act(const or_options_t *old_options)
       addressmap_clear_invalid_automaps(options);
 
 /* How long should we delay counting bridge stats after becoming a bridge?
- * We use this so we don't count people who used our bridge thinking it is
+ * We use this so we don't count clients who used our bridge thinking it is
  * a relay. If you change this, don't forget to change the log message
  * below. It's 4 hours (the time it takes to stop being used by clients)
  * plus some extra time for clock skew. */
@@ -1868,9 +1941,16 @@ options_act(const or_options_t *old_options)
     if (transition_affects_workers) {
       log_info(LD_GENERAL,
                "Worker-related options changed. Rotating workers.");
+      const int server_mode_turned_on =
+        server_mode(options) && !server_mode(old_options);
+      const int dir_server_mode_turned_on =
+        dir_server_mode(options) && !dir_server_mode(old_options);
 
-      if (server_mode(options) && !server_mode(old_options)) {
+      if (server_mode_turned_on || dir_server_mode_turned_on) {
         cpu_init();
+      }
+
+      if (server_mode_turned_on) {
         ip_address_changed(0);
         if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
           inform_testing_reachability();
@@ -1891,6 +1971,8 @@ options_act(const or_options_t *old_options)
   /* Only collect directory-request statistics on relays and bridges. */
   options->DirReqStatistics = options->DirReqStatistics_option &&
     server_mode(options);
+  options->HiddenServiceStatistics =
+    options->HiddenServiceStatistics_option && server_mode(options);
 
   if (options->CellStatistics || options->DirReqStatistics ||
       options->EntryStatistics || options->ExitPortStatistics ||
@@ -1905,7 +1987,6 @@ options_act(const or_options_t *old_options)
       options->CellStatistics = 0;
       options->EntryStatistics = 0;
       options->ConnDirectionStatistics = 0;
-      options->HiddenServiceStatistics = 0;
       options->ExitPortStatistics = 0;
     }
 
@@ -1991,13 +2072,6 @@ options_act(const or_options_t *old_options)
       !options->BridgeAuthoritativeDir)
     rep_hist_desc_stats_term();
 
-  /* Check if we need to parse and add the EntryNodes config option. */
-  if (options->EntryNodes &&
-      (!old_options ||
-       !routerset_equal(old_options->EntryNodes,options->EntryNodes) ||
-       !routerset_equal(old_options->ExcludeNodes,options->ExcludeNodes)))
-    entry_nodes_should_be_added();
-
   /* Since our options changed, we might need to regenerate and upload our
    * server descriptor.
    */
@@ -2252,7 +2326,7 @@ print_usage(void)
   printf(
 "Copyright (c) 2001-2004, Roger Dingledine\n"
 "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n"
-"Copyright (c) 2007-2016, The Tor Project, Inc.\n\n"
+"Copyright (c) 2007-2017, The Tor Project, Inc.\n\n"
 "tor -f <torrc> [args]\n"
 "See man page for options, or https://www.torproject.org/ for "
 "documentation.\n");
@@ -2333,8 +2407,8 @@ using_default_dir_authorities(const or_options_t *options)
  * Fail if one or more of the following is true:
  *   - DNS name in <b>options-\>Address</b> cannot be resolved.
  *   - <b>options-\>Address</b> is a local host address.
- *   - Attempt to getting local hostname fails.
- *   - Attempt to getting network interface address fails.
+ *   - Attempt at getting local hostname fails.
+ *   - Attempt at getting network interface address fails.
  *
  * Return 0 if all is well, or -1 if we can't find a suitable
  * public IP address.
@@ -2714,13 +2788,13 @@ compute_publishserverdescriptor(or_options_t *options)
 #define MIN_REND_POST_PERIOD (10*60)
 #define MIN_REND_POST_PERIOD_TESTING (5)
 
-/** Higest allowable value for PredictedPortsRelevanceTime; if this is
- * too high, our selection of exits will decrease for an extended
- * period of time to an uncomfortable level .*/
-#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60)
+/** Highest allowable value for CircuitsAvailableTimeout.
+ * If this is too large, client connections will stay open for too long,
+ * incurring extra padding overhead. */
+#define MAX_CIRCS_AVAILABLE_TIME (24*60*60)
 
 /** Highest allowable value for RendPostPeriod. */
-#define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2)
+#define MAX_DIR_PERIOD ((7*24*60*60)/2)
 
 /** Lowest allowable value for MaxCircuitDirtiness; if this is too low, Tor
  * will generate too many circuits and potentially overload the network. */
@@ -2846,8 +2920,7 @@ options_validate_single_onion(or_options_t *options, char **msg)
       !options->Tor2webMode) {
     REJECT("HiddenServiceNonAnonymousMode is incompatible with using Tor as "
            "an anonymous client. Please set Socks/Trans/NATD/DNSPort to 0, or "
-           "HiddenServiceNonAnonymousMode to 0, or use the non-anonymous "
-           "Tor2webMode.");
+           "revert HiddenServiceNonAnonymousMode to 0.");
   }
 
   /* If you run a hidden service in non-anonymous mode, the hidden service
@@ -2857,12 +2930,12 @@ options_validate_single_onion(or_options_t *options, char **msg)
     REJECT("Non-anonymous (Tor2web) mode is incompatible with using Tor as a "
            "hidden service. Please remove all HiddenServiceDir lines, or use "
            "a version of tor compiled without --enable-tor2web-mode, or use "
-           " HiddenServiceNonAnonymousMode.");
+           "HiddenServiceNonAnonymousMode.");
   }
 
   if (rend_service_allow_non_anonymous_connection(options)
       && options->UseEntryGuards) {
-    /* Single Onion services only use entry guards when uploading descriptors,
+    /* Single Onion services only use entry guards when uploading descriptors;
      * all other connections are one-hop. Further, Single Onions causes the
      * hidden service code to do things which break the path bias
      * detector, and it's far easier to turn off entry guards (and
@@ -2904,8 +2977,12 @@ options_validate(or_options_t *old_options, or_options_t *options,
   tor_assert(msg);
   *msg = NULL;
 
+  if (parse_ports(options, 1, msg, &n_ports,
+                  &world_writable_control_socket) < 0)
+    return -1;
+
   /* Set UseEntryGuards from the configured value, before we check it below.
-   * We change UseEntryGuards whenn it's incompatible with other options,
+   * We change UseEntryGuards when it's incompatible with other options,
    * but leave UseEntryGuards_option with the original value.
    * Always use the value of UseEntryGuards, not UseEntryGuards_option. */
   options->UseEntryGuards = options->UseEntryGuards_option;
@@ -2922,10 +2999,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
         "for details.", uname);
   }
 
-  if (parse_ports(options, 1, msg, &n_ports,
-                  &world_writable_control_socket) < 0)
-    return -1;
-
   if (parse_outbound_addresses(options, 1, msg) < 0)
     return -1;
 
@@ -2991,7 +3064,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     if (!strcasecmp(options->TransProxyType, "default")) {
       options->TransProxyType_parsed = TPT_DEFAULT;
     } else if (!strcasecmp(options->TransProxyType, "pf-divert")) {
-#if !defined(__OpenBSD__) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined( DARWIN )
       /* Later versions of OS X have pf */
       REJECT("pf-divert is a OpenBSD-specific "
              "and OS X/Darwin-specific feature.");
@@ -3018,14 +3091,12 @@ options_validate(or_options_t *old_options, or_options_t *options,
 
     if (strcasecmp(options->TransProxyType, "default") &&
         !options->TransPort_set) {
-      REJECT("Cannot use TransProxyType without any valid TransPort or "
-             "TransListenAddress.");
+      REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
 #else
   if (options->TransPort_set)
-    REJECT("TransPort and TransListenAddress are disabled "
-           "in this build.");
+    REJECT("TransPort is disabled in this build.");
 #endif
 
   if (options->TokenBucketRefillInterval <= 0
@@ -3062,15 +3133,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
-  if (options->TLSECGroup && (strcasecmp(options->TLSECGroup, "P256") &&
-                              strcasecmp(options->TLSECGroup, "P224"))) {
-    COMPLAIN("Unrecognized TLSECGroup: Falling back to the default.");
-    tor_free(options->TLSECGroup);
-  }
-  if (!evaluate_ecgroup_for_tls(options->TLSECGroup)) {
-    REJECT("Unsupported TLSECGroup.");
-  }
-
   if (options->ExcludeNodes && options->StrictNodes) {
     COMPLAIN("You have asked to exclude certain relays from all positions "
              "in your circuits. Expect hidden services and other Tor "
@@ -3257,23 +3319,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "of the Internet, so they must not set Reachable*Addresses "
            "or FascistFirewall or FirewallPorts or ClientUseIPv4 0.");
 
-  /* We check if Reachable*Addresses blocks all addresses in
-   * parse_reachable_addresses(). */
-
-#define WARN_PLEASE_USE_IPV6_LOG_MSG \
-        "ClientPreferIPv6%sPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges."
-
-  if (!fascist_firewall_use_ipv6(options)
-      && options->ClientPreferIPv6ORPort == 1)
-    log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "OR");
-
-  if (!fascist_firewall_use_ipv6(options)
-      && options->ClientPreferIPv6DirPort == 1)
-    log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "Dir");
-
-#undef WARN_PLEASE_USE_IPV6_LOG_MSG
-
   if (options->UseBridges &&
       server_mode(options))
     REJECT("Servers must be able to freely connect to the rest "
@@ -3285,33 +3330,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->UseBridges && options->EntryNodes)
     REJECT("You cannot set both UseBridges and EntryNodes.");
 
+  /* If we have UseBridges as 1 and UseEntryGuards as 0, we end up bypassing
+   * the use of bridges */
+  if (options->UseBridges && !options->UseEntryGuards)
+    REJECT("Setting UseBridges requires also setting UseEntryGuards.");
+
   options->MaxMemInQueues =
     compute_real_max_mem_in_queues(options->MaxMemInQueues_raw,
                                    server_mode(options));
   options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3;
 
-  options->AllowInvalid_ = 0;
-
-  if (options->AllowInvalidNodes) {
-    SMARTLIST_FOREACH_BEGIN(options->AllowInvalidNodes, const char *, cp) {
-        if (!strcasecmp(cp, "entry"))
-          options->AllowInvalid_ |= ALLOW_INVALID_ENTRY;
-        else if (!strcasecmp(cp, "exit"))
-          options->AllowInvalid_ |= ALLOW_INVALID_EXIT;
-        else if (!strcasecmp(cp, "middle"))
-          options->AllowInvalid_ |= ALLOW_INVALID_MIDDLE;
-        else if (!strcasecmp(cp, "introduction"))
-          options->AllowInvalid_ |= ALLOW_INVALID_INTRODUCTION;
-        else if (!strcasecmp(cp, "rendezvous"))
-          options->AllowInvalid_ |= ALLOW_INVALID_RENDEZVOUS;
-        else {
-          tor_asprintf(msg,
-              "Unrecognized value '%s' in AllowInvalidNodes", cp);
-          return -1;
-        }
-    } SMARTLIST_FOREACH_END(cp);
-  }
-
   if (!options->SafeLogging ||
       !strcasecmp(options->SafeLogging, "0")) {
     options->SafeLogging_ = SAFELOG_SCRUB_NONE;
@@ -3347,6 +3375,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
     options->DirPort_set = 0;
   }
 
+  if (server_mode(options) && options->ConnectionPadding != -1) {
+    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
+  }
+
+  if (server_mode(options) && options->ReducedConnectionPadding != 0) {
+    REJECT("Relays cannot set ReducedConnectionPadding. ");
+  }
+
   if (options->MinUptimeHidServDirectoryV2 < 0) {
     log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at "
                         "least 0 seconds. Changing to 0.");
@@ -3368,17 +3404,17 @@ options_validate(or_options_t *old_options, or_options_t *options,
     options->RendPostPeriod = MAX_DIR_PERIOD;
   }
 
-  if (options->PredictedPortsRelevanceTime >
-      MAX_PREDICTED_CIRCS_RELEVANCE) {
-    log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; "
-             "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE);
-    options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE;
-  }
-
   /* Check the Single Onion Service options */
   if (options_validate_single_onion(options, msg) < 0)
     return -1;
 
+  if (options->CircuitsAvailableTimeout > MAX_CIRCS_AVAILABLE_TIME) {
+    // options_t is immutable for new code (the above code is older),
+    // so just make the user fix the value themselves rather than
+    // silently keep a shadow value lower than what they asked for.
+    REJECT("CircuitsAvailableTimeout is too large. Max is 24 hours.");
+  }
+
 #ifdef ENABLE_TOR2WEB_MODE
   if (options->Tor2webMode && options->UseEntryGuards) {
     /* tor2web mode clients do not (and should not) use entry guards
@@ -3428,6 +3464,20 @@ options_validate(or_options_t *old_options, or_options_t *options,
     return -1;
   }
 
+  /* Inform the hidden service operator that pinning EntryNodes can possibly
+   * be harmful for the service anonymity. */
+  if (options->EntryNodes &&
+      routerset_is_list(options->EntryNodes) &&
+      (options->RendConfigLines != NULL)) {
+    log_warn(LD_CONFIG,
+             "EntryNodes is set with multiple entries and at least one "
+             "hidden service is configured. Pinning entry nodes can possibly "
+             "be harmful to the service anonymity. Because of this, we "
+             "recommend you either don't do that or make sure you know what "
+             "you are doing. For more details, please look at "
+             "https://trac.torproject.org/projects/tor/ticket/21155.");
+  }
+
   /* Single Onion Services: non-anonymous hidden services */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     log_warn(LD_CONFIG,
@@ -3453,7 +3503,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     int severity = LOG_NOTICE;
     /* Be a little quieter if we've deliberately disabled
      * LearnCircuitBuildTimeout. */
-    if (circuit_build_times_disabled()) {
+    if (circuit_build_times_disabled_(options, 1)) {
       severity = LOG_INFO;
     }
     log_fn(severity, LD_CONFIG, "You disabled LearnCircuitBuildTimeout, but "
@@ -3783,13 +3833,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "have it group-readable.");
   }
 
-  if (options->MyFamily && options->BridgeRelay) {
+  if (options->MyFamily_lines && options->BridgeRelay) {
     log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
              "supported: it can reveal bridge fingerprints to censors. "
              "You should also make sure you aren't listing this bridge's "
              "fingerprint in any other MyFamily.");
   }
-  if (check_nickname_list(&options->MyFamily, "MyFamily", msg))
+  if (normalize_nickname_list(&options->MyFamily,
+                              options->MyFamily_lines, "MyFamily", msg))
     return -1;
   for (cl = options->NodeFamilies; cl; cl = cl->next) {
     routerset_t *rs = routerset_new();
@@ -3986,13 +4037,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "AlternateDirAuthority and AlternateBridgeAuthority configured.");
   }
 
-  if (options->AllowSingleHopExits && !options->DirAuthorities) {
-    COMPLAIN("You have set AllowSingleHopExits; now your relay will allow "
-             "others to make one-hop exits. However, since by default most "
-             "clients avoid relays that set this option, most clients will "
-             "ignore you.");
-  }
-
 #define CHECK_DEFAULT(arg)                                              \
   STMT_BEGIN                                                            \
     if (!options->TestingTorNetwork &&                                  \
@@ -4258,8 +4302,8 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 }
 
 /* If we have less than 300 MB suggest disabling dircache */
-#define DIRCACHE_MIN_MB_BANDWIDTH 300
-#define DIRCACHE_MIN_BANDWIDTH (DIRCACHE_MIN_MB_BANDWIDTH*ONE_MEGABYTE)
+#define DIRCACHE_MIN_MEM_MB 300
+#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
 #define STRINGIFY(val) #val
 
 /** Create a warning message for emitting if we are a dircache but may not have
@@ -4279,21 +4323,21 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
     }
   }
   if (options->DirCache) {
-    if (total_mem < DIRCACHE_MIN_BANDWIDTH) {
+    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
       if (options->BridgeRelay) {
         *msg = tor_strdup("Running a Bridge with less than "
-                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
-                      "not recommended.");
+                      STRINGIFY(DIRCACHE_MIN_MEM_MB) " MB of memory is not "
+                      "recommended.");
       } else {
         *msg = tor_strdup("Being a directory cache (default) with less than "
-                      STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is "
-                      "not recommended and may consume most of the available "
+                      STRINGIFY(DIRCACHE_MIN_MEM_MB) " MB of memory is not "
+                      "recommended and may consume most of the available "
                       "resources, consider disabling this functionality by "
                       "setting the DirCache option to 0.");
       }
     }
   } else {
-    if (total_mem >= DIRCACHE_MIN_BANDWIDTH) {
+    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
       *msg = tor_strdup("DirCache is disabled and we are configured as a "
                "relay. This may disqualify us from becoming a guard in the "
                "future.");
@@ -4418,7 +4462,6 @@ options_transition_allowed(const or_options_t *old,
     } while (0)
 
     SB_NOCHANGE_STR(Address);
-    SB_NOCHANGE_STR(PidFile);
     SB_NOCHANGE_STR(ServerDNSResolvConfFile);
     SB_NOCHANGE_STR(DirPortFrontPage);
     SB_NOCHANGE_STR(CookieAuthFile);
@@ -4458,6 +4501,7 @@ options_transition_affects_workers(const or_options_t *old_options,
       old_options->SafeLogging_ != new_options->SafeLogging_ ||
       old_options->ClientOnly != new_options->ClientOnly ||
       server_mode(old_options) != server_mode(new_options) ||
+      dir_server_mode(old_options) != dir_server_mode(new_options) ||
       public_server_mode(old_options) != public_server_mode(new_options) ||
       !config_lines_eq(old_options->Logs, new_options->Logs) ||
       old_options->LogMessageDomains != new_options->LogMessageDomains)
@@ -4499,7 +4543,7 @@ options_transition_affects_descriptor(const or_options_t *old_options,
       get_effective_bwburst(old_options) !=
         get_effective_bwburst(new_options) ||
       !opt_streq(old_options->ContactInfo, new_options->ContactInfo) ||
-      !opt_streq(old_options->MyFamily, new_options->MyFamily) ||
+      !config_lines_eq(old_options->MyFamily, new_options->MyFamily) ||
       !opt_streq(old_options->AccountingStart, new_options->AccountingStart) ||
       old_options->AccountingMax != new_options->AccountingMax ||
       old_options->AccountingRule != new_options->AccountingRule ||
@@ -4595,27 +4639,36 @@ get_default_conf_file(int defaults_file)
 #endif
 }
 
-/** Verify whether lst is a string containing valid-looking comma-separated
- * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname
- * or fingerprint that needs it. Return 0 on success.
+/** Verify whether lst is a list of strings containing valid-looking
+ * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
+ * to any nickname or fingerprint that needs it. Also splits comma-separated
+ * list elements into multiple elements. Return 0 on success.
  * Warn and return -1 on failure.
  */
 static int
-check_nickname_list(char **lst, const char *name, char **msg)
+normalize_nickname_list(config_line_t **normalized_out,
+                        const config_line_t *lst, const char *name,
+                        char **msg)
 {
-  int r = 0;
-  smartlist_t *sl;
-  int changes = 0;
-
-  if (!*lst)
+  if (!lst)
     return 0;
-  sl = smartlist_new();
 
-  smartlist_split_string(sl, *lst, ",",
-    SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+  config_line_t *new_nicknames = NULL;
+  config_line_t **new_nicknames_next = &new_nicknames;
+
+  const config_line_t *cl;
+  for (cl = lst; cl; cl = cl->next) {
+    const char *line = cl->value;
+    if (!line)
+      continue;
 
-  SMARTLIST_FOREACH_BEGIN(sl, char *, s)
+    int valid_line = 1;
+    smartlist_t *sl = smartlist_new();
+    smartlist_split_string(sl, line, ",",
+      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
     {
+      char *normalized = NULL;
       if (!is_legal_nickname_or_hexdigest(s)) {
         // check if first char is dollar
         if (s[0] != '$') {
@@ -4624,36 +4677,45 @@ check_nickname_list(char **lst, const char *name, char **msg)
           tor_asprintf(&prepended, "$%s", s);
 
           if (is_legal_nickname_or_hexdigest(prepended)) {
-            // The nickname is valid when it's prepended, swap the current
-            // version with a prepended one
-            tor_free(s);
-            SMARTLIST_REPLACE_CURRENT(sl, s, prepended);
-            changes = 1;
-            continue;
+            // The nickname is valid when it's prepended, set it as the
+            // normalized version
+            normalized = prepended;
+          } else {
+            // Still not valid, free and fallback to error message
+            tor_free(prepended);
           }
-
-          // Still not valid, free and fallback to error message
-          tor_free(prepended);
         }
 
-        tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
-        r = -1;
-        break;
+        if (!normalized) {
+          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
+          valid_line = 0;
+          break;
+        }
+      } else {
+        normalized = tor_strdup(s);
       }
-    }
-  SMARTLIST_FOREACH_END(s);
 
-  // Replace the caller's nickname list with a fixed one
-  if (changes && r == 0) {
-    char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL);
-    tor_free(*lst);
-    *lst = newNicknames;
+      config_line_t *next = tor_malloc_zero(sizeof(*next));
+      next->key = tor_strdup(cl->key);
+      next->value = normalized;
+      next->next = NULL;
+
+      *new_nicknames_next = next;
+      new_nicknames_next = &next->next;
+    } SMARTLIST_FOREACH_END(s);
+
+    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
+    smartlist_free(sl);
+
+    if (!valid_line) {
+      config_free_lines(new_nicknames);
+      return -1;
+    }
   }
 
-  SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
-  smartlist_free(sl);
+  *normalized_out = new_nicknames;
 
-  return r;
+  return 0;
 }
 
 /** Learn config file name from command line arguments, or use the default.
@@ -4855,9 +4917,21 @@ options_init_from_torrc(int argc, char **argv)
     printf("OpenSSL \t\t%-15s\t\t%s\n",
                       crypto_openssl_get_header_version_str(),
                       crypto_openssl_get_version_str());
-    printf("Zlib    \t\t%-15s\t\t%s\n",
-                      tor_zlib_get_header_version_str(),
-                      tor_zlib_get_version_str());
+    if (tor_compress_supports_method(ZLIB_METHOD)) {
+      printf("Zlib    \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(ZLIB_METHOD),
+                        tor_compress_header_version_str(ZLIB_METHOD));
+    }
+    if (tor_compress_supports_method(LZMA_METHOD)) {
+      printf("Liblzma \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(LZMA_METHOD),
+                        tor_compress_header_version_str(LZMA_METHOD));
+    }
+    if (tor_compress_supports_method(ZSTD_METHOD)) {
+      printf("Libzstd \t\t%-15s\t\t%s\n",
+                        tor_compress_version_str(ZSTD_METHOD),
+                        tor_compress_header_version_str(ZSTD_METHOD));
+    }
     //TODO: Hex versions?
     exit(0);
   }
@@ -4997,6 +5071,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   config_line_t *cl;
   int retval;
   setopt_err_t err = SETOPT_ERR_MISC;
+  int cf_has_include = 0;
   tor_assert(msg);
 
   oldoptions = global_options; /* get_options unfortunately asserts if
@@ -5013,7 +5088,8 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     if (!body)
       continue;
     /* get config lines, assign them */
-    retval = config_get_lines(body, &cl, 1);
+    retval = config_get_lines_include(body, &cl, 1,
+                                      body == cf ? &cf_has_include : NULL);
     if (retval < 0) {
       err = SETOPT_ERR_PARSE;
       goto err;
@@ -5041,6 +5117,8 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     goto err;
   }
 
+  newoptions->IncludeUsed = cf_has_include;
+
   /* If this is a testing network configuration, change defaults
    * for a list of dependent config options, re-initialize newoptions
    * with the new defaults, and assign all options to it second time. */
@@ -5084,7 +5162,8 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       if (!body)
         continue;
       /* get config lines, assign them */
-      retval = config_get_lines(body, &cl, 1);
+      retval = config_get_lines_include(body, &cl, 1,
+                                        body == cf ? &cf_has_include : NULL);
       if (retval < 0) {
         err = SETOPT_ERR_PARSE;
         goto err;
@@ -5107,6 +5186,8 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     }
   }
 
+  newoptions->IncludeUsed = cf_has_include;
+
   /* Validate newoptions */
   if (options_validate(oldoptions, newoptions, newdefaultoptions,
                        0, msg) < 0) {
@@ -5210,35 +5291,35 @@ addressmap_register_auto(const char *from, const char *to,
   int from_wildcard = 0, to_wildcard = 0;
 
   *msg = "whoops, forgot the error message";
-  if (1) {
-    if (!strcmp(to, "*") || !strcmp(from, "*")) {
-      *msg = "can't remap from or to *";
-      return -1;
-    }
-    /* Detect asterisks in expressions of type: '*.example.com' */
-    if (!strncmp(from,"*.",2)) {
-      from += 2;
-      from_wildcard = 1;
-    }
-    if (!strncmp(to,"*.",2)) {
-      to += 2;
-      to_wildcard = 1;
-    }
 
-    if (to_wildcard && !from_wildcard) {
-      *msg =  "can only use wildcard (i.e. '*.') if 'from' address "
-        "uses wildcard also";
-      return -1;
-    }
+  if (!strcmp(to, "*") || !strcmp(from, "*")) {
+    *msg = "can't remap from or to *";
+    return -1;
+  }
+  /* Detect asterisks in expressions of type: '*.example.com' */
+  if (!strncmp(from,"*.",2)) {
+    from += 2;
+    from_wildcard = 1;
+  }
+  if (!strncmp(to,"*.",2)) {
+    to += 2;
+    to_wildcard = 1;
+  }
 
-    if (address_is_invalid_destination(to, 1)) {
-      *msg = "destination is invalid";
-      return -1;
-    }
+  if (to_wildcard && !from_wildcard) {
+    *msg =  "can only use wildcard (i.e. '*.') if 'from' address "
+      "uses wildcard also";
+    return -1;
+  }
 
-    addressmap_register(from, tor_strdup(to), expires, addrmap_source,
-                        from_wildcard, to_wildcard);
+  if (address_is_invalid_destination(to, 1)) {
+    *msg = "destination is invalid";
+    return -1;
   }
+
+  addressmap_register(from, tor_strdup(to), expires, addrmap_source,
+                      from_wildcard, to_wildcard);
+
   return 0;
 }
 
@@ -5305,7 +5386,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
                            SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
 
     if (smartlist_len(elts) == 0)
-      smartlist_add(elts, tor_strdup("stdout"));
+      smartlist_add_strdup(elts, "stdout");
 
     if (smartlist_len(elts) == 1 &&
         (!strcasecmp(smartlist_get(elts,0), "stdout") ||
@@ -5840,7 +5921,7 @@ get_options_from_transport_options_line(const char *line,const char *transport)
     }
 
     /* add it to the options smartlist */
-    smartlist_add(options, tor_strdup(option));
+    smartlist_add_strdup(options, option);
     log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
   } SMARTLIST_FOREACH_END(option);
 
@@ -6165,6 +6246,7 @@ port_cfg_new(size_t namelen)
   tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1);
   port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1);
   cfg->entry_cfg.ipv4_traffic = 1;
+  cfg->entry_cfg.ipv6_traffic = 1;
   cfg->entry_cfg.dns_request = 1;
   cfg->entry_cfg.onion_traffic = 1;
   cfg->entry_cfg.cache_ipv4_answers = 1;
@@ -6342,14 +6424,9 @@ warn_client_dns_cache(const char *option, int disabling)
 /**
  * Parse port configuration for a single port type.
  *
- * Read entries of the "FooPort" type from the list <b>ports</b>, and
- * entries of the "FooListenAddress" type from the list
- * <b>listenaddrs</b>.  Two syntaxes are supported: a legacy syntax
- * where FooPort is at most a single entry containing a port number and
- * where FooListenAddress has any number of address:port combinations;
- * and a new syntax where there are no FooListenAddress entries and
- * where FooPort can have any number of entries of the format
- * "[Address:][Port] IsolationOptions".
+ * Read entries of the "FooPort" type from the list <b>ports</b>.  Syntax is
+ * that FooPort can have any number of entries of the format
+ *  "[Address:][Port] IsolationOptions".
  *
  * In log messages, describe the port type as <b>portname</b>.
  *
@@ -6363,9 +6440,6 @@ warn_client_dns_cache(const char *option, int disabling)
  * ports are not on a local address.  If CL_PORT_FORBID_NONLOCAL is set,
  * this is a control port with no password set: don't even allow it.
  *
- * Unless CL_PORT_ALLOW_EXTRA_LISTENADDR is set in <b>flags</b>, warn
- * if FooListenAddress is set but FooPort is 0.
- *
  * If CL_PORT_SERVER_OPTIONS is set in <b>flags</b>, do not allow stream
  * isolation options in the FooPort entries; instead allow the
  * server-port option set.
@@ -6380,7 +6454,6 @@ warn_client_dns_cache(const char *option, int disabling)
 STATIC int
 parse_port_config(smartlist_t *out,
                   const config_line_t *ports,
-                  const config_line_t *listenaddrs,
                   const char *portname,
                   int listener_type,
                   const char *defaultaddr,
@@ -6397,90 +6470,12 @@ parse_port_config(smartlist_t *out,
   const unsigned forbid_nonlocal = flags & CL_PORT_FORBID_NONLOCAL;
   const unsigned default_to_group_writable =
     flags & CL_PORT_DFLT_GROUP_WRITABLE;
-  const unsigned allow_spurious_listenaddr =
-    flags & CL_PORT_ALLOW_EXTRA_LISTENADDR;
   const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES;
   const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET;
   int got_zero_port=0, got_nonzero_port=0;
   char *unix_socket_path = NULL;
 
-  /* FooListenAddress is deprecated; let's make it work like it used to work,
-   * though. */
-  if (listenaddrs) {
-    int mainport = defaultport;
-
-    if (ports && ports->next) {
-      log_warn(LD_CONFIG, "%sListenAddress can't be used when there are "
-               "multiple %sPort lines", portname, portname);
-      return -1;
-    } else if (ports) {
-      if (!strcmp(ports->value, "auto")) {
-        mainport = CFG_AUTO_PORT;
-      } else {
-        int ok;
-        mainport = (int)tor_parse_long(ports->value, 10, 0, 65535, &ok, NULL);
-        if (!ok) {
-          log_warn(LD_CONFIG, "%sListenAddress can only be used with a single "
-                   "%sPort with value \"auto\" or 1-65535 and no options set.",
-                   portname, portname);
-          return -1;
-        }
-      }
-    }
-
-    if (mainport == 0) {
-      if (allow_spurious_listenaddr)
-        return 1; /*DOCDOC*/
-      log_warn(LD_CONFIG, "%sPort must be defined if %sListenAddress is used",
-               portname, portname);
-      return -1;
-    }
-
-    if (use_server_options && out) {
-      /* Add a no_listen port. */
-      port_cfg_t *cfg = port_cfg_new(0);
-      cfg->type = listener_type;
-      cfg->port = mainport;
-      tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */
-      cfg->server_cfg.no_listen = 1;
-      cfg->server_cfg.bind_ipv4_only = 1;
-      /* cfg->entry_cfg defaults are already set by port_cfg_new */
-      smartlist_add(out, cfg);
-    }
-
-    for (; listenaddrs; listenaddrs = listenaddrs->next) {
-      tor_addr_t addr;
-      uint16_t port = 0;
-      if (tor_addr_port_lookup(listenaddrs->value, &addr, &port) < 0) {
-        log_warn(LD_CONFIG, "Unable to parse %sListenAddress '%s'",
-                 portname, listenaddrs->value);
-        return -1;
-      }
-      if (out) {
-        port_cfg_t *cfg = port_cfg_new(0);
-        cfg->type = listener_type;
-        cfg->port = port ? port : mainport;
-        tor_addr_copy(&cfg->addr, &addr);
-        cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
-        cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
-        cfg->server_cfg.no_advertise = 1;
-        smartlist_add(out, cfg);
-      }
-    }
-
-    if (warn_nonlocal && out) {
-      if (is_control)
-        warn_nonlocal_controller_ports(out, forbid_nonlocal);
-      else if (is_ext_orport)
-        warn_nonlocal_ext_orports(out, portname);
-      else
-        warn_nonlocal_client_ports(out, portname, listener_type);
-    }
-    return 0;
-  } /* end if (listenaddrs) */
-
-  /* No ListenAddress lines. If there's no FooPort, then maybe make a default
-   * one. */
+  /* If there's no FooPort, then maybe make a default one. */
   if (! ports) {
     if (defaultport && defaultaddr && out) {
       port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
@@ -6518,7 +6513,7 @@ parse_port_config(smartlist_t *out,
     /* This must be kept in sync with port_cfg_new's defaults */
     int no_listen = 0, no_advertise = 0, all_addrs = 0,
       bind_ipv4_only = 0, bind_ipv6_only = 0,
-      ipv4_traffic = 1, ipv6_traffic = 0, prefer_ipv6 = 0, dns_request = 1,
+      ipv4_traffic = 1, ipv6_traffic = 1, prefer_ipv6 = 0, dns_request = 1,
       onion_traffic = 1,
       cache_ipv4 = 1, use_cached_ipv4 = 0,
       cache_ipv6 = 0, use_cached_ipv6 = 0,
@@ -6881,6 +6876,7 @@ parse_port_config(smartlist_t *out,
     SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
     smartlist_clear(elts);
     tor_free(addrport);
+    tor_free(unix_socket_path);
   }
 
   if (warn_nonlocal && out) {
@@ -6951,36 +6947,35 @@ parse_ports(or_options_t *options, int validate_only,
   const unsigned gw_flag = options->SocksSocketsGroupWritable ?
     CL_PORT_DFLT_GROUP_WRITABLE : 0;
   if (parse_port_config(ports,
-             options->SocksPort_lines, options->SocksListenAddress,
+             options->SocksPort_lines,
              "Socks", CONN_TYPE_AP_LISTENER,
              "127.0.0.1", 9050,
-             CL_PORT_WARN_NONLOCAL|CL_PORT_ALLOW_EXTRA_LISTENADDR|
-             CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) {
-    *msg = tor_strdup("Invalid SocksPort/SocksListenAddress configuration");
+             CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) {
+    *msg = tor_strdup("Invalid SocksPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->DNSPort_lines, options->DNSListenAddress,
+                        options->DNSPort_lines,
                         "DNS", CONN_TYPE_AP_DNS_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL|CL_PORT_TAKES_HOSTNAMES) < 0) {
-    *msg = tor_strdup("Invalid DNSPort/DNSListenAddress configuration");
+    *msg = tor_strdup("Invalid DNSPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->TransPort_lines, options->TransListenAddress,
+                        options->TransPort_lines,
                         "Trans", CONN_TYPE_AP_TRANS_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL) < 0) {
-    *msg = tor_strdup("Invalid TransPort/TransListenAddress configuration");
+    *msg = tor_strdup("Invalid TransPort configuration");
     goto err;
   }
   if (parse_port_config(ports,
-                        options->NATDPort_lines, options->NATDListenAddress,
+                        options->NATDPort_lines,
                         "NATD", CONN_TYPE_AP_NATD_LISTENER,
                         "127.0.0.1", 0,
                         CL_PORT_WARN_NONLOCAL) < 0) {
-    *msg = tor_strdup("Invalid NatdPort/NatdListenAddress configuration");
+    *msg = tor_strdup("Invalid NatdPort configuration");
     goto err;
   }
   {
@@ -6996,16 +6991,14 @@ parse_ports(or_options_t *options, int validate_only,
 
     if (parse_port_config(ports,
                           options->ControlPort_lines,
-                          options->ControlListenAddress,
                           "Control", CONN_TYPE_CONTROL_LISTENER,
                           "127.0.0.1", 0,
                           control_port_flags) < 0) {
-      *msg = tor_strdup("Invalid ControlPort/ControlListenAddress "
-                        "configuration");
+      *msg = tor_strdup("Invalid ControlPort configuration");
       goto err;
     }
 
-    if (parse_port_config(ports, options->ControlSocket, NULL,
+    if (parse_port_config(ports, options->ControlSocket,
                           "ControlSocket",
                           CONN_TYPE_CONTROL_LISTENER, NULL, 0,
                           control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) {
@@ -7015,15 +7008,15 @@ parse_ports(or_options_t *options, int validate_only,
   }
   if (! options->ClientOnly) {
     if (parse_port_config(ports,
-                          options->ORPort_lines, options->ORListenAddress,
+                          options->ORPort_lines,
                           "OR", CONN_TYPE_OR_LISTENER,
                           "0.0.0.0", 0,
                           CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid ORPort/ORListenAddress configuration");
+      *msg = tor_strdup("Invalid ORPort configuration");
       goto err;
     }
     if (parse_port_config(ports,
-                          options->ExtORPort_lines, NULL,
+                          options->ExtORPort_lines,
                           "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
                           "127.0.0.1", 0,
                           CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
@@ -7031,11 +7024,11 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
     if (parse_port_config(ports,
-                          options->DirPort_lines, options->DirListenAddress,
+                          options->DirPort_lines,
                           "Dir", CONN_TYPE_DIR_LISTENER,
                           "0.0.0.0", 0,
                           CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid DirPort/DirListenAddress configuration");
+      *msg = tor_strdup("Invalid DirPort configuration");
       goto err;
     }
   }
@@ -7756,7 +7749,7 @@ getinfo_helper_config(control_connection_t *conn,
         case CONFIG_TYPE_CSV: type = "CommaList"; break;
         case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break;
         case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependant"; break;
+        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
         case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
         default:
         case CONFIG_TYPE_OBSOLETE:
@@ -7838,60 +7831,83 @@ getinfo_helper_config(control_connection_t *conn,
   return 0;
 }
 
-/** Parse outbound bind address option lines. If <b>validate_only</b>
- * is not 0 update OutboundBindAddressIPv4_ and
- * OutboundBindAddressIPv6_ in <b>options</b>. On failure, set
- * <b>msg</b> (if provided) to a newly allocated string containing a
- * description of the problem and return -1. */
+/* Check whether an address has already been set against the options
+ * depending on address family and destination type. Any exsting
+ * value will lead to a fail, even if it is the same value. If not
+ * set and not only validating, copy it into this location too.
+ * Returns 0 on success or -1 if this address is already set.
+ */
 static int
-parse_outbound_addresses(or_options_t *options, int validate_only, char **msg)
+verify_and_store_outbound_address(sa_family_t family, tor_addr_t *addr,
+       outbound_addr_t type, or_options_t *options, int validate_only)
 {
-  const config_line_t *lines = options->OutboundBindAddress;
-  int found_v4 = 0, found_v6 = 0;
-
+  if (type>=OUTBOUND_ADDR_MAX || (family!=AF_INET && family!=AF_INET6)) {
+    return -1;
+  }
+  int fam_index=0;
+  if (family==AF_INET6) {
+    fam_index=1;
+  }
+  tor_addr_t *dest=&options->OutboundBindAddresses[type][fam_index];
+  if (!tor_addr_is_null(dest)) {
+    return -1;
+  }
   if (!validate_only) {
-    memset(&options->OutboundBindAddressIPv4_, 0,
-           sizeof(options->OutboundBindAddressIPv4_));
-    memset(&options->OutboundBindAddressIPv6_, 0,
-           sizeof(options->OutboundBindAddressIPv6_));
+    tor_addr_copy(dest, addr);
   }
+  return 0;
+}
+
+/* Parse a list of address lines for a specific destination type.
+ * Will store them into the options if not validate_only. If a
+ * problem occurs, a suitable error message is store in msg.
+ * Returns 0 on success or -1 if any address is already set.
+ */
+static int
+parse_outbound_address_lines(const config_line_t *lines, outbound_addr_t type,
+           or_options_t *options, int validate_only, char **msg)
+{
+  tor_addr_t addr;
+  sa_family_t family;
   while (lines) {
-    tor_addr_t addr, *dst_addr = NULL;
-    int af = tor_addr_parse(&addr, lines->value);
-    switch (af) {
-    case AF_INET:
-      if (found_v4) {
-        if (msg)
-          tor_asprintf(msg, "Multiple IPv4 outbound bind addresses "
-                       "configured: %s", lines->value);
-        return -1;
-      }
-      found_v4 = 1;
-      dst_addr = &options->OutboundBindAddressIPv4_;
-      break;
-    case AF_INET6:
-      if (found_v6) {
-        if (msg)
-          tor_asprintf(msg, "Multiple IPv6 outbound bind addresses "
-                       "configured: %s", lines->value);
-        return -1;
-      }
-      found_v6 = 1;
-      dst_addr = &options->OutboundBindAddressIPv6_;
-      break;
-    default:
+    family = tor_addr_parse(&addr, lines->value);
+    if (verify_and_store_outbound_address(family, &addr, type,
+                                 options, validate_only)) {
       if (msg)
-        tor_asprintf(msg, "Outbound bind address '%s' didn't parse.",
-                     lines->value);
+        tor_asprintf(msg, "Multiple%s%s outbound bind addresses "
+                     "configured: %s",
+                     family==AF_INET?" IPv4":(family==AF_INET6?" IPv6":""),
+                     type==OUTBOUND_ADDR_OR?" OR":
+                     (type==OUTBOUND_ADDR_EXIT?" exit":""), lines->value);
       return -1;
     }
-    if (!validate_only)
-      tor_addr_copy(dst_addr, &addr);
     lines = lines->next;
   }
   return 0;
 }
 
+/** Parse outbound bind address option lines. If <b>validate_only</b>
+ * is not 0 update OutboundBindAddresses in <b>options</b>.
+ * Only one address can be set for any of these values.
+ * On failure, set <b>msg</b> (if provided) to a newly allocated string
+ * containing a description of the problem and return -1.
+ */
+static int
+parse_outbound_addresses(or_options_t *options, int validate_only, char **msg)
+{
+  if (!validate_only) {
+    memset(&options->OutboundBindAddresses, 0,
+           sizeof(options->OutboundBindAddresses));
+  }
+  parse_outbound_address_lines(options->OutboundBindAddress,
+                      OUTBOUND_ADDR_EXIT_AND_OR, options, validate_only, msg);
+  parse_outbound_address_lines(options->OutboundBindAddressOR,
+                      OUTBOUND_ADDR_OR, options, validate_only, msg);
+  parse_outbound_address_lines(options->OutboundBindAddressExit,
+                      OUTBOUND_ADDR_EXIT, options, validate_only, msg);
+  return 0;
+}
+
 /** Load one of the geoip files, <a>family</a> determining which
  * one. <a>default_fname</a> is used if on Windows and
  * <a>fname</a> equals "<default>". */
diff --git a/src/or/config.h b/src/or/config.h
index 6645532514..27aec7fe3d 100644
--- a/src/or/config.h
+++ b/src/or/config.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -157,7 +157,7 @@ smartlist_t *get_options_for_server_transport(const char *transport);
 
 #define CL_PORT_NO_STREAM_OPTIONS (1u<<0)
 #define CL_PORT_WARN_NONLOCAL (1u<<1)
-#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2)
+/* Was CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) */
 #define CL_PORT_SERVER_OPTIONS (1u<<3)
 #define CL_PORT_FORBID_NONLOCAL (1u<<4)
 #define CL_PORT_TAKES_HOSTNAMES (1u<<5)
@@ -193,7 +193,6 @@ STATIC int have_enough_mem_for_dircache(const or_options_t *options,
                                         size_t total_mem, char **msg);
 STATIC int parse_port_config(smartlist_t *out,
                   const config_line_t *ports,
-                  const config_line_t *listenaddrs,
                   const char *portname,
                   int listener_type,
                   const char *defaultaddr,
diff --git a/src/or/confparse.c b/src/or/confparse.c
index efcf4f981e..abae7e33dc 100644
--- a/src/or/confparse.c
+++ b/src/or/confparse.c
@@ -1,7 +1,8 @@
+
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +10,16 @@
  *
  * \brief Back-end for parsing and generating key-value files, used to
  *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
  */
 
 #include "or.h"
@@ -67,120 +78,6 @@ config_expand_abbrev(const config_format_t *fmt, const char *option,
   return option;
 }
 
-/** Helper: allocate a new configuration option mapping 'key' to 'val',
- * append it to *<b>lst</b>. */
-void
-config_line_append(config_line_t **lst,
-                   const char *key,
-                   const char *val)
-{
-  config_line_t *newline;
-
-  newline = tor_malloc_zero(sizeof(config_line_t));
-  newline->key = tor_strdup(key);
-  newline->value = tor_strdup(val);
-  newline->next = NULL;
-  while (*lst)
-    lst = &((*lst)->next);
-
-  (*lst) = newline;
-}
-
-/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL
- * if no such key exists. For handling commandline-only options only; other
- * options should be looked up in the appropriate data structure. */
-const config_line_t *
-config_line_find(const config_line_t *lines,
-                 const char *key)
-{
-  const config_line_t *cl;
-  for (cl = lines; cl; cl = cl->next) {
-    if (!strcmp(cl->key, key))
-      return cl;
-  }
-  return NULL;
-}
-
-/** Helper: parse the config string and strdup into key/value
- * strings. Set *result to the list, or NULL if parsing the string
- * failed.  Return 0 on success, -1 on failure. Warn and ignore any
- * misformatted lines.
- *
- * If <b>extended</b> is set, then treat keys beginning with / and with + as
- * indicating "clear" and "append" respectively. */
-int
-config_get_lines(const char *string, config_line_t **result, int extended)
-{
-  config_line_t *list = NULL, **next;
-  char *k, *v;
-  const char *parse_err;
-
-  next = &list;
-  do {
-    k = v = NULL;
-    string = parse_config_line_from_str_verbose(string, &k, &v, &parse_err);
-    if (!string) {
-      log_warn(LD_CONFIG, "Error while parsing configuration: %s",
-               parse_err?parse_err:"<unknown>");
-      config_free_lines(list);
-      tor_free(k);
-      tor_free(v);
-      return -1;
-    }
-    if (k && v) {
-      unsigned command = CONFIG_LINE_NORMAL;
-      if (extended) {
-        if (k[0] == '+') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          command = CONFIG_LINE_APPEND;
-        } else if (k[0] == '/') {
-          char *k_new = tor_strdup(k+1);
-          tor_free(k);
-          k = k_new;
-          tor_free(v);
-          v = tor_strdup("");
-          command = CONFIG_LINE_CLEAR;
-        }
-      }
-      /* This list can get long, so we keep a pointer to the end of it
-       * rather than using config_line_append over and over and getting
-       * n^2 performance. */
-      *next = tor_malloc_zero(sizeof(config_line_t));
-      (*next)->key = k;
-      (*next)->value = v;
-      (*next)->next = NULL;
-      (*next)->command = command;
-      next = &((*next)->next);
-    } else {
-      tor_free(k);
-      tor_free(v);
-    }
-  } while (*string);
-
-  *result = list;
-  return 0;
-}
-
-/**
- * Free all the configuration lines on the linked list <b>front</b>.
- */
-void
-config_free_lines(config_line_t *front)
-{
-  config_line_t *tmp;
-
-  while (front) {
-    tmp = front;
-    front = tmp->next;
-
-    tor_free(tmp->key);
-    tor_free(tmp->value);
-    tor_free(tmp);
-  }
-}
-
 /** If <b>key</b> is a deprecated configuration option, return the message
  * explaining why it is deprecated (which may be an empty string). Return NULL
  * if it is not deprecated. The <b>key</b> field must be fully expanded. */
@@ -620,23 +517,6 @@ config_value_needs_escape(const char *value)
   return 0;
 }
 
-/** Return a newly allocated deep copy of the lines in <b>inp</b>. */
-config_line_t *
-config_lines_dup(const config_line_t *inp)
-{
-  config_line_t *result = NULL;
-  config_line_t **next_out = &result;
-  while (inp) {
-    *next_out = tor_malloc_zero(sizeof(config_line_t));
-    (*next_out)->key = tor_strdup(inp->key);
-    (*next_out)->value = tor_strdup(inp->value);
-    inp = inp->next;
-    next_out = &((*next_out)->next);
-  }
-  (*next_out) = NULL;
-  return result;
-}
-
 /** Return newly allocated line or lines corresponding to <b>key</b> in the
  * configuration <b>options</b>.  If <b>escape_val</b> is true and a
  * value needs to be quoted before it's put in a config file, quote and
@@ -753,11 +633,11 @@ config_get_assigned_option(const config_format_t *fmt, const void *options,
       tor_free(result);
       return NULL;
     case CONFIG_TYPE_LINELIST_S:
-      log_warn(LD_CONFIG,
-               "Can't return context-sensitive '%s' on its own", key);
       tor_free(result->key);
       tor_free(result);
-      return NULL;
+      result = config_lines_dup_and_filter(*(const config_line_t **)value,
+                                           key);
+      break;
     case CONFIG_TYPE_LINELIST:
     case CONFIG_TYPE_LINELIST_V:
       tor_free(result->key);
@@ -1002,36 +882,6 @@ config_free(const config_format_t *fmt, void *options)
   tor_free(options);
 }
 
-/** Return true iff a and b contain identical keys and values in identical
- * order. */
-int
-config_lines_eq(config_line_t *a, config_line_t *b)
-{
-  while (a && b) {
-    if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
-      return 0;
-    a = a->next;
-    b = b->next;
-  }
-  if (a || b)
-    return 0;
-  return 1;
-}
-
-/** Return the number of lines in <b>a</b> whose key is <b>key</b>. */
-int
-config_count_key(const config_line_t *a, const char *key)
-{
-  int n = 0;
-  while (a) {
-    if (!strcasecmp(a->key, key)) {
-      ++n;
-    }
-    a = a->next;
-  }
-  return n;
-}
-
 /** Return true iff the option <b>name</b> has the same value in <b>o1</b>
  * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
  */
@@ -1148,6 +998,11 @@ config_dump(const config_format_t *fmt, const void *default_options,
       config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
 
     for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
       smartlist_add_asprintf(elements, "%s%s %s\n",
                    comment_option ? "# " : "",
                    line->key, line->value);
@@ -1213,6 +1068,8 @@ static struct unit_table_t memory_units[] = {
   { "gbits",     1<<27 },
   { "gbit",      1<<27 },
   { "tb",        U64_LITERAL(1)<<40 },
+  { "tbyte",     U64_LITERAL(1)<<40 },
+  { "tbytes",    U64_LITERAL(1)<<40 },
   { "terabyte",  U64_LITERAL(1)<<40 },
   { "terabytes", U64_LITERAL(1)<<40 },
   { "terabits",  U64_LITERAL(1)<<37 },
@@ -1333,8 +1190,6 @@ config_parse_msec_interval(const char *s, int *ok)
 {
   uint64_t r;
   r = config_parse_units(s, time_msec_units, ok);
-  if (!ok)
-    return -1;
   if (r > INT_MAX) {
     log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
     *ok = 0;
@@ -1352,8 +1207,6 @@ config_parse_interval(const char *s, int *ok)
 {
   uint64_t r;
   r = config_parse_units(s, time_units, ok);
-  if (!ok)
-    return -1;
   if (r > INT_MAX) {
     log_warn(LD_CONFIG, "Interval '%s' is too long", s);
     *ok = 0;
diff --git a/src/or/confparse.h b/src/or/confparse.h
index 8d915d266b..9c4205d07c 100644
--- a/src/or/confparse.h
+++ b/src/or/confparse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CONFPARSE_H
@@ -103,14 +103,7 @@ typedef struct config_format_t {
 #define CAL_WARN_DEPRECATIONS (1u<<2)
 
 void *config_new(const config_format_t *fmt);
-void config_line_append(config_line_t **lst,
-                        const char *key, const char *val);
-config_line_t *config_lines_dup(const config_line_t *inp);
-const config_line_t *config_line_find(const config_line_t *lines,
-                                      const char *key);
 void config_free(const config_format_t *fmt, void *options);
-int config_lines_eq(config_line_t *a, config_line_t *b);
-int config_count_key(const config_line_t *a, const char *key);
 config_line_t *config_get_assigned_option(const config_format_t *fmt,
                                           const void *options, const char *key,
                                           int escape_val);
@@ -131,9 +124,6 @@ const char *config_find_deprecation(const config_format_t *fmt,
                                      const char *key);
 const config_var_t *config_find_option(const config_format_t *fmt,
                                        const char *key);
-
-int config_get_lines(const char *string, config_line_t **result, int extended);
-void config_free_lines(config_line_t *front);
 const char *config_expand_abbrev(const config_format_t *fmt,
                                  const char *option,
                                  int command_line, int warn_obsolete);
diff --git a/src/or/connection.c b/src/or/connection.c
index 791fd95c27..fc0646b885 100644
--- a/src/or/connection.c
+++ b/src/or/connection.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,7 @@
  * they become able to read or write register the fact with the event main
  * loop by calling connection_watch_events(), connection_start_reading(), or
  * connection_start_writing().  When they no longer want to read or write,
- * they call connection_stop_reading() or connection_start_writing().
+ * they call connection_stop_reading() or connection_stop_writing().
  *
  * To queue data to be written on a connection, call
  * connection_write_to_buf().  When data arrives, the
@@ -56,6 +56,7 @@
 
 #define CONNECTION_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
@@ -83,6 +84,7 @@
 #include "ext_orport.h"
 #include "geoip.h"
 #include "main.h"
+#include "hs_common.h"
 #include "nodelist.h"
 #include "policies.h"
 #include "reasons.h"
@@ -133,6 +135,8 @@ static int connection_read_https_proxy_response(connection_t *conn);
 static void connection_send_socks5_connect(connection_t *conn);
 static const char *proxy_type_to_string(int proxy_type);
 static int get_proxy_type(void);
+const tor_addr_t *conn_get_outbound_address(sa_family_t family,
+                  const or_options_t *options, unsigned int conn_type);
 
 /** The last addresses that our network interface seemed to have been
  * binding to.  We use this as one way to detect when our IP changes.
@@ -625,14 +629,19 @@ connection_free_(connection_t *conn)
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     tor_free(dir_conn->requested_resource);
 
-    tor_zlib_free(dir_conn->zlib_state);
-    if (dir_conn->fingerprint_stack) {
-      SMARTLIST_FOREACH(dir_conn->fingerprint_stack, char *, cp, tor_free(cp));
-      smartlist_free(dir_conn->fingerprint_stack);
+    tor_compress_free(dir_conn->compress_state);
+    if (dir_conn->spool) {
+      SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled,
+                        spooled_resource_free(spooled));
+      smartlist_free(dir_conn->spool);
     }
 
-    cached_dir_decref(dir_conn->cached_dir);
     rend_data_free(dir_conn->rend_data);
+    if (dir_conn->guard_state) {
+      /* Cancel before freeing, if it's still there. */
+      entry_guard_cancel(&dir_conn->guard_state);
+    }
+    circuit_guard_state_free(dir_conn->guard_state);
   }
 
   if (SOCKET_OK(conn->s)) {
@@ -644,7 +653,7 @@ connection_free_(connection_t *conn)
   if (conn->type == CONN_TYPE_OR &&
       !tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest)) {
     log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest");
-    connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+    connection_or_clear_identity(TO_OR_CONN(conn));
   }
   if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) {
     connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn));
@@ -675,7 +684,7 @@ connection_free,(connection_t *conn))
   }
   if (connection_speaks_cells(conn)) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest)) {
-      connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+      connection_or_clear_identity(TO_OR_CONN(conn));
     }
   }
   if (conn->type == CONN_TYPE_CONTROL) {
@@ -1784,7 +1793,7 @@ connection_connect_sockaddr,(connection_t *conn,
 
   /*
    * We've got the socket open; give the OOS handler a chance to check
-   * against configuured maximum socket number, but tell it no exhaustion
+   * against configured maximum socket number, but tell it no exhaustion
    * failure.
    */
   connection_check_oos(get_n_open_sockets(), 0);
@@ -1903,6 +1912,55 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
   }
 }
 
+/** Retrieve the outbound address depending on the protocol (IPv4 or IPv6)
+ * and the connection type (relay, exit, ...)
+ * Return a socket address or NULL in case nothing is configured.
+ **/
+const tor_addr_t *
+conn_get_outbound_address(sa_family_t family,
+             const or_options_t *options, unsigned int conn_type)
+{
+  const tor_addr_t *ext_addr = NULL;
+
+  int fam_index;
+  switch (family) {
+    case AF_INET:
+      fam_index = 0;
+      break;
+    case AF_INET6:
+      fam_index = 1;
+      break;
+    default:
+      return NULL;
+  }
+
+  // If an exit connection, use the exit address (if present)
+  if (conn_type == CONN_TYPE_EXIT) {
+    if (!tor_addr_is_null(
+        &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT][fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT]
+                 [fam_index];
+    } else if (!tor_addr_is_null(
+                 &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index];
+    }
+  } else { // All non-exit connections
+    if (!tor_addr_is_null(
+           &options->OutboundBindAddresses[OUTBOUND_ADDR_OR][fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_OR]
+                 [fam_index];
+    } else if (!tor_addr_is_null(
+                 &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index])) {
+      ext_addr = &options->OutboundBindAddresses[OUTBOUND_ADDR_EXIT_AND_OR]
+                 [fam_index];
+    }
+  }
+  return ext_addr;
+}
+
 /** Take conn, make a nonblocking socket; try to connect to
  * addr:port (port arrives in *host order*). If fail, return -1 and if
  * applicable put your best guess about errno into *<b>socket_error</b>.
@@ -1924,26 +1982,15 @@ connection_connect(connection_t *conn, const char *address,
   struct sockaddr *bind_addr = NULL;
   struct sockaddr *dest_addr;
   int dest_addr_len, bind_addr_len = 0;
-  const or_options_t *options = get_options();
-  int protocol_family;
 
   /* Log if we didn't stick to ClientUseIPv4/6 or ClientPreferIPv6OR/DirPort
    */
   connection_connect_log_client_use_ip_version(conn);
 
-  if (tor_addr_family(addr) == AF_INET6)
-    protocol_family = PF_INET6;
-  else
-    protocol_family = PF_INET;
-
   if (!tor_addr_is_loopback(addr)) {
     const tor_addr_t *ext_addr = NULL;
-    if (protocol_family == AF_INET &&
-        !tor_addr_is_null(&options->OutboundBindAddressIPv4_))
-      ext_addr = &options->OutboundBindAddressIPv4_;
-    else if (protocol_family == AF_INET6 &&
-             !tor_addr_is_null(&options->OutboundBindAddressIPv6_))
-      ext_addr = &options->OutboundBindAddressIPv6_;
+    ext_addr = conn_get_outbound_address(tor_addr_family(addr), get_options(),
+                                         conn->type);
     if (ext_addr) {
       memset(&bind_addr_ss, 0, sizeof(bind_addr_ss));
       bind_addr_len = tor_addr_to_sockaddr(ext_addr, 0,
@@ -4011,10 +4058,6 @@ connection_flush(connection_t *conn)
  * its contents compressed or decompressed as they're written.  If zlib is
  * negative, this is the last data to be compressed, and the connection's zlib
  * state should be flushed.
- *
- * If it's a local control connection and a 64k chunk is ready, try to flush
- * it all, so we don't end up with many megabytes of controller info queued at
- * once.
  */
 MOCK_IMPL(void,
 connection_write_to_buf_impl_,(const char *string, size_t len,
@@ -4033,9 +4076,9 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
   if (zlib) {
     dir_connection_t *dir_conn = TO_DIR_CONN(conn);
     int done = zlib < 0;
-    CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf,
-                                                 dir_conn->zlib_state,
-                                                 string, len, done));
+    CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf,
+                                                     dir_conn->compress_state,
+                                                     string, len, done));
   } else {
     CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf));
   }
@@ -4149,12 +4192,12 @@ connection_get_by_type_state_rendquery(int type, int state,
          (type == CONN_TYPE_DIR &&
           TO_DIR_CONN(conn)->rend_data &&
           !rend_cmp_service_ids(rendquery,
-                                TO_DIR_CONN(conn)->rend_data->onion_address))
+                    rend_data_get_address(TO_DIR_CONN(conn)->rend_data)))
          ||
               (CONN_IS_EDGE(conn) &&
                TO_EDGE_CONN(conn)->rend_data &&
                !rend_cmp_service_ids(rendquery,
-                            TO_EDGE_CONN(conn)->rend_data->onion_address))
+                    rend_data_get_address(TO_EDGE_CONN(conn)->rend_data)))
          ));
 }
 
diff --git a/src/or/connection.h b/src/or/connection.h
index d25e002fa4..36e45aef38 100644
--- a/src/or/connection.h
+++ b/src/or/connection.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -141,17 +141,17 @@ MOCK_DECL(void, connection_write_to_buf_impl_,
 /* DOCDOC connection_write_to_buf */
 static void connection_write_to_buf(const char *string, size_t len,
                                     connection_t *conn);
-/* DOCDOC connection_write_to_buf_zlib */
-static void connection_write_to_buf_zlib(const char *string, size_t len,
-                                         dir_connection_t *conn, int done);
+/* DOCDOC connection_write_to_buf_compress */
+static void connection_write_to_buf_compress(const char *string, size_t len,
+                                             dir_connection_t *conn, int done);
 static inline void
 connection_write_to_buf(const char *string, size_t len, connection_t *conn)
 {
   connection_write_to_buf_impl_(string, len, conn, 0);
 }
 static inline void
-connection_write_to_buf_zlib(const char *string, size_t len,
-                             dir_connection_t *conn, int done)
+connection_write_to_buf_compress(const char *string, size_t len,
+                                 dir_connection_t *conn, int done)
 {
   connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1);
 }
diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c
index 12fe2f57c9..8480a35458 100644
--- a/src/or/connection_edge.c
+++ b/src/or/connection_edge.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,7 @@
  *   <li>DNS lookup streams, created on the exit side in response to
  *     a RELAY_RESOLVE cell from a client.
  *   <li>Tunneled directory streams, created on the directory cache side
- *     in response to a RELAY_BEGINDIR cell.  These streams attach directly
+ *     in response to a RELAY_BEGIN_DIR cell.  These streams attach directly
  *     to a dir_connection_t object without ever using TCP.
  *   </ul>
  *
@@ -75,6 +75,7 @@
 #include "directory.h"
 #include "dirserv.h"
 #include "hibernate.h"
+#include "hs_common.h"
 #include "main.h"
 #include "nodelist.h"
 #include "policies.h"
@@ -329,6 +330,33 @@ relay_send_end_cell_from_edge(streamid_t stream_id, circuit_t *circ,
                                       payload, 1, cpath_layer);
 }
 
+/* If the connection <b>conn</b> is attempting to connect to an external
+ * destination that is an hidden service and the reason is a connection
+ * refused or timeout, log it so the operator can take appropriate actions.
+ * The log statement is a rate limited warning. */
+static void
+warn_if_hs_unreachable(const edge_connection_t *conn, uint8_t reason)
+{
+  tor_assert(conn);
+
+  if (conn->base_.type == CONN_TYPE_EXIT &&
+      connection_edge_is_rendezvous_stream(conn) &&
+      (reason == END_STREAM_REASON_CONNECTREFUSED ||
+       reason == END_STREAM_REASON_TIMEOUT)) {
+#define WARN_FAILED_HS_CONNECTION 300
+    static ratelim_t warn_limit = RATELIM_INIT(WARN_FAILED_HS_CONNECTION);
+    char *m;
+    if ((m = rate_limit_log(&warn_limit, approx_time()))) {
+      log_warn(LD_EDGE, "Onion service connection to %s failed (%s)",
+               (conn->base_.socket_family == AF_UNIX) ?
+               safe_str(conn->base_.address) :
+               safe_str(fmt_addrport(&conn->base_.addr, conn->base_.port)),
+               stream_end_reason_to_string(reason));
+      tor_free(m);
+    }
+  }
+}
+
 /** Send a relay end cell from stream <b>conn</b> down conn's circuit, and
  * remember that we've done so.  If this is not a client connection, set the
  * relay end cell's reason for closing as <b>reason</b>.
@@ -386,6 +414,9 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
               conn->base_.s);
     connection_edge_send_command(conn, RELAY_COMMAND_END,
                                  payload, payload_len);
+    /* We'll log warn if the connection was an hidden service and couldn't be
+     * made because the service wasn't available. */
+    warn_if_hs_unreachable(conn, control_reason);
   } else {
     log_debug(LD_EDGE,"No circ to send end on conn "
               "(fd "TOR_SOCKET_T_FORMAT").",
@@ -831,7 +862,8 @@ connection_ap_rescan_and_attach_pending(void)
 #endif
 
 /** Tell any AP streams that are listed as waiting for a new circuit to try
- * again, either attaching to an available circ or launching a new one.
+ * again.  If there is an available circuit for a stream, attach it. Otherwise,
+ * launch a new circuit.
  *
  * If <b>retry</b> is false, only check the list if it contains at least one
  * streams that we have not yet tried to attach to a circuit.
@@ -846,8 +878,9 @@ connection_ap_attach_pending(int retry)
   if (untried_pending_connections == 0 && !retry)
     return;
 
-  /* Don't allow modifications to pending_entry_connections while we are
-   * iterating over it. */
+  /* Don't allow any modifications to list while we are iterating over
+   * it.  We'll put streams back on this list if we can't attach them
+   * immediately. */
   smartlist_t *pending = pending_entry_connections;
   pending_entry_connections = smartlist_new();
 
@@ -866,9 +899,7 @@ connection_ap_attach_pending(int retry)
       continue;
     }
     if (conn->state != AP_CONN_STATE_CIRCUIT_WAIT) {
-      // XXXX 030 -- this is downgraded in 0.2.9, since we apparently
-      // XXXX are running into it in practice.  It's harmless.
-      log_info(LD_BUG, "%p is no longer in circuit_wait. Its current state "
+      log_warn(LD_BUG, "%p is no longer in circuit_wait. Its current state "
                "is %s. Why is it on pending_entry_connections?",
                entry_conn,
                conn_state_to_string(conn->type, conn->state));
@@ -876,6 +907,7 @@ connection_ap_attach_pending(int retry)
       continue;
     }
 
+    /* Okay, we're through the sanity checks. Try to handle this stream. */
     if (connection_ap_handshake_attach_circuit(entry_conn) < 0) {
       if (!conn->marked_for_close)
         connection_mark_unattached_ap(entry_conn,
@@ -885,12 +917,17 @@ connection_ap_attach_pending(int retry)
     if (! conn->marked_for_close &&
         conn->type == CONN_TYPE_AP &&
         conn->state == AP_CONN_STATE_CIRCUIT_WAIT) {
+      /* Is it still waiting for a circuit? If so, we didn't attach it,
+       * so it's still pending.  Put it back on the list.
+       */
       if (!smartlist_contains(pending_entry_connections, entry_conn)) {
         smartlist_add(pending_entry_connections, entry_conn);
         continue;
       }
     }
 
+    /* If we got here, then we either closed the connection, or
+     * we attached it. */
     UNMARK();
   } SMARTLIST_FOREACH_END(entry_conn);
 
@@ -1198,6 +1235,8 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
 
   /* Remember the original address so we can tell the user about what
    * they actually said, not just what it turned into. */
+  /* XXX yes, this is the same as out->orig_address above. One is
+   * in the output, and one is in the connection. */
   if (! conn->original_dest_address) {
     /* Is the 'if' necessary here? XXXX */
     conn->original_dest_address = tor_strdup(conn->socks_request->address);
@@ -1205,7 +1244,7 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
 
   /* First, apply MapAddress and MAPADDRESS mappings. We need to do
    * these only for non-reverse lookups, since they don't exist for those.
-   * We need to do this before we consider automapping, since we might
+   * We also need to do this before we consider automapping, since we might
    * e.g. resolve irc.oftc.net into irconionaddress.onion, at which point
    * we'd need to automap it. */
   if (socks->command != SOCKS_COMMAND_RESOLVE_PTR) {
@@ -1217,9 +1256,12 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
     }
   }
 
-  /* Now, handle automapping.  Automapping happens when we're asked to
-   * resolve a hostname, and AutomapHostsOnResolve is set, and
-   * the hostname has a suffix listed in AutomapHostsSuffixes.
+  /* Now see if we need to create or return an existing Hostname->IP
+   * automapping.  Automapping happens when we're asked to resolve a
+   * hostname, and AutomapHostsOnResolve is set, and the hostname has a
+   * suffix listed in AutomapHostsSuffixes.  It's a handy feature
+   * that lets you have Tor assign e.g. IPv6 addresses for .onion
+   * names, and return them safely from DNSPort.
    */
   if (socks->command == SOCKS_COMMAND_RESOLVE &&
       tor_addr_parse(&addr_tmp, socks->address)<0 &&
@@ -1259,7 +1301,8 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
   }
 
   /* Now handle reverse lookups, if they're in the cache.  This doesn't
-   * happen too often, since client-side DNS caching is off by default. */
+   * happen too often, since client-side DNS caching is off by default,
+   * and very deprecated. */
   if (socks->command == SOCKS_COMMAND_RESOLVE_PTR) {
     unsigned rewrite_flags = 0;
     if (conn->entry_cfg.use_cached_ipv4_answers)
@@ -1288,7 +1331,7 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
      * an internal address?  If so, we should reject it if we're configured to
      * do so. */
     if (options->ClientDNSRejectInternalAddresses) {
-      /* Don't let people try to do a reverse lookup on 10.0.0.1. */
+      /* Don't let clients try to do a reverse lookup on 10.0.0.1. */
       tor_addr_t addr;
       int ok;
       ok = tor_addr_parse_PTR_name(
@@ -1304,11 +1347,12 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
     }
   }
 
-  /* If we didn't automap it before, then this is still the address
-   * that came straight from the user, mapped according to any
-   * MapAddress/MAPADDRESS commands.  Now other mappings, including
-   * previously registered Automap entries, TrackHostExits entries,
-   * and client-side DNS cache entries (not recommended).
+  /* If we didn't automap it before, then this is still the address that
+   * came straight from the user, mapped according to any
+   * MapAddress/MAPADDRESS commands.  Now apply other mappings,
+   * including previously registered Automap entries (IP back to
+   * hostname), TrackHostExits entries, and client-side DNS cache
+   * entries (if they're turned on).
    */
   if (socks->command != SOCKS_COMMAND_RESOLVE_PTR &&
       !out->automap) {
@@ -1373,11 +1417,14 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   time_t now = time(NULL);
   rewrite_result_t rr;
 
+  /* First we'll do the rewrite part.  Let's see if we get a reasonable
+   * answer.
+   */
   memset(&rr, 0, sizeof(rr));
   connection_ap_handshake_rewrite(conn,&rr);
 
   if (rr.should_close) {
-    /* connection_ap_handshake_rewrite told us to close the connection,
+    /* connection_ap_handshake_rewrite told us to close the connection:
      * either because it sent back an answer, or because it sent back an
      * error */
     connection_mark_unattached_ap(conn, rr.end_reason);
@@ -1391,8 +1438,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   const int automap = rr.automap;
   const addressmap_entry_source_t exit_source = rr.exit_source;
 
-  /* Parse the address provided by SOCKS.  Modify it in-place if it
-   * specifies a hidden-service (.onion) or particular exit node (.exit).
+  /* Now, we parse the address to see if it's an .onion or .exit or
+   * other special address.
    */
   const hostname_type_t addresstype = parse_extended_hostname(socks->address);
 
@@ -1406,8 +1453,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   }
 
   /* If this is a .exit hostname, strip off the .name.exit part, and
-   * see whether we're going to connect there, and otherwise handle it.
-   * (The ".exit" part got stripped off by "parse_extended_hostname").
+   * see whether we're willing to connect there, and and otherwise handle the
+   * .exit address.
    *
    * We'll set chosen_exit_name and/or close the connection as appropriate.
    */
@@ -1419,7 +1466,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     const node_t *node = NULL;
 
     /* If this .exit was added by an AUTOMAP, then it came straight from
-     * a user.  Make sure that options->AllowDotExit permits that. */
+     * a user.  Make sure that options->AllowDotExit permits that! */
     if (exit_source == ADDRMAPSRC_AUTOMAP && !options->AllowDotExit) {
       /* Whoops; this one is stale.  It must have gotten added earlier,
        * when AllowDotExit was on. */
@@ -1448,7 +1495,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     }
 
     tor_assert(!automap);
-    /* Now, find the character before the .(name) part. */
+
+    /* Now, find the character before the .(name) part.
+     * (The ".exit" part got stripped off by "parse_extended_hostname").
+     *
+     * We're going to put the exit name into conn->chosen_exit_name, and
+     * look up a node correspondingly. */
     char *s = strrchr(socks->address,'.');
     if (s) {
       /* The address was of the form "(stuff).(name).exit */
@@ -1504,10 +1556,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
        implies no. */
   }
 
-  /* Now, handle everything that isn't a .onion address. */
+  /* Now, we handle everything that isn't a .onion address. */
   if (addresstype != ONION_HOSTNAME) {
     /* Not a hidden-service request.  It's either a hostname or an IP,
-     * possibly with a .exit that we stripped off. */
+     * possibly with a .exit that we stripped off.  We're going to check
+     * if we're allowed to connect/resolve there, and then launch the
+     * appropriate request. */
 
     /* Check for funny characters in the address. */
     if (address_is_invalid_destination(socks->address, 1)) {
@@ -1554,30 +1608,37 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     }
 
     /* Then check if we have a hostname or IP address, and whether DNS or
-     * the IP address family are permitted */
+     * the IP address family are permitted.  Reject if not. */
     tor_addr_t dummy_addr;
     int socks_family = tor_addr_parse(&dummy_addr, socks->address);
     /* family will be -1 for a non-onion hostname that's not an IP */
-    if (socks_family == -1 && !conn->entry_cfg.dns_request) {
-      log_warn(LD_APP, "Refusing to connect to hostname %s "
-               "because Port has NoDNSRequest set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    } else if (socks_family == AF_INET && !conn->entry_cfg.ipv4_traffic) {
-      log_warn(LD_APP, "Refusing to connect to IPv4 address %s because "
-               "Port has NoIPv4Traffic set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
-    } else if (socks_family == AF_INET6 && !conn->entry_cfg.ipv6_traffic) {
-      log_warn(LD_APP, "Refusing to connect to IPv6 address %s because "
-               "Port has NoIPv6Traffic set.",
-               safe_str_client(socks->address));
-      connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
-      return -1;
+    if (socks_family == -1) {
+      if (!conn->entry_cfg.dns_request) {
+        log_warn(LD_APP, "Refusing to connect to hostname %s "
+                 "because Port has NoDNSRequest set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else if (socks_family == AF_INET) {
+      if (!conn->entry_cfg.ipv4_traffic) {
+        log_warn(LD_APP, "Refusing to connect to IPv4 address %s because "
+                 "Port has NoIPv4Traffic set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else if (socks_family == AF_INET6) {
+      if (!conn->entry_cfg.ipv6_traffic) {
+        log_warn(LD_APP, "Refusing to connect to IPv6 address %s because "
+                 "Port has NoIPv6Traffic set.",
+                 safe_str_client(socks->address));
+        connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY);
+        return -1;
+      }
+    } else {
+      tor_assert_nonfatal_unreached_once();
     }
-    /* No else, we've covered all possible returned value. */
 
     /* See if this is a hostname lookup that we can answer immediately.
      * (For example, an attempt to look up the IP address for an IP address.)
@@ -1598,7 +1659,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       tor_assert(!automap);
       rep_hist_note_used_resolve(now); /* help predict this next time */
     } else if (socks->command == SOCKS_COMMAND_CONNECT) {
-      /* Special handling for attempts to connect */
+      /* Now see if this is a connect request that we can reject immediately */
+
       tor_assert(!automap);
       /* Don't allow connections to port 0. */
       if (socks->port == 0) {
@@ -1607,7 +1669,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         return -1;
       }
       /* You can't make connections to internal addresses, by default.
-       * Exceptions are begindir requests (where the address is meaningless,
+       * Exceptions are begindir requests (where the address is meaningless),
        * or cases where you've hand-configured a particular exit, thereby
        * making the local address meaningful. */
       if (options->ClientRejectInternalAddresses &&
@@ -1651,7 +1713,9 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       } /* end "if we should check for internal addresses" */
 
       /* Okay.  We're still doing a CONNECT, and it wasn't a private
-       * address.  Do special handling for literal IP addresses */
+       * address.  Here we do special handling for literal IP addresses,
+       * to see if we should reject this preemptively, and to set up
+       * fields in conn->entry_cfg to tell the exit what AF we want. */
       {
         tor_addr_t addr;
         /* XXX Duplicate call to tor_addr_parse. */
@@ -1694,11 +1758,15 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         }
       }
 
+      /* we never allow IPv6 answers on socks4. (TODO: Is this smart?) */
       if (socks->socks_version == 4)
         conn->entry_cfg.ipv6_traffic = 0;
 
       /* Still handling CONNECT. Now, check for exit enclaves.  (Which we
-       * don't do on BEGINDIR, or there is a chosen exit.)
+       * don't do on BEGIN_DIR, or when there is a chosen exit.)
+       *
+       * TODO: Should we remove this?  Exit enclaves are nutty and don't
+       * work very well
        */
       if (!conn->use_begindir && !conn->chosen_exit_name && !circ) {
         /* see if we can find a suitable enclave exit */
@@ -1722,7 +1790,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
         if (consider_plaintext_ports(conn, socks->port) < 0)
           return -1;
 
-      /* Remember the port so that we do predicted requests there. */
+      /* Remember the port so that we will predict that more requests
+         there will happen in the future. */
       if (!conn->use_begindir) {
         /* help predict this next time */
         rep_hist_note_used_port(now, socks->port);
@@ -1731,7 +1800,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       rep_hist_note_used_resolve(now); /* help predict this next time */
       /* no extra processing needed */
     } else {
-      /* We should only be doing CONNECT or RESOLVE! */
+      /* We should only be doing CONNECT, RESOLVE, or RESOLVE_PTR! */
       tor_fragile_assert();
     }
 
@@ -1747,6 +1816,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     if (circ) {
       rv = connection_ap_handshake_attach_chosen_circuit(conn, circ, cpath);
     } else {
+      /* We'll try to attach it at the next event loop, or whenever
+       * we call connection_ap_attach_pending() */
       connection_ap_mark_as_pending_circuit(conn);
       rv = 0;
     }
@@ -1820,24 +1891,26 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
     if (rend_data == NULL) {
       return -1;
     }
+    const char *onion_address = rend_data_get_address(rend_data);
     log_info(LD_REND,"Got a hidden service request for ID '%s'",
-             safe_str_client(rend_data->onion_address));
+             safe_str_client(onion_address));
 
-    /* Lookup the given onion address. If invalid, stop right now else we
-     * might have it in the cache or not, it will be tested later on. */
+    /* Lookup the given onion address. If invalid, stop right now.
+     * Otherwise, we might have it in the cache or not. */
     unsigned int refetch_desc = 0;
     rend_cache_entry_t *entry = NULL;
     const int rend_cache_lookup_result =
-      rend_cache_lookup_entry(rend_data->onion_address, -1, &entry);
+      rend_cache_lookup_entry(onion_address, -1, &entry);
     if (rend_cache_lookup_result < 0) {
       switch (-rend_cache_lookup_result) {
       case EINVAL:
         /* We should already have rejected this address! */
         log_warn(LD_BUG,"Invalid service name '%s'",
-            safe_str_client(rend_data->onion_address));
+                 safe_str_client(onion_address));
         connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
         return -1;
       case ENOENT:
+        /* We didn't have this; we should look it up. */
         refetch_desc = 1;
         break;
       default:
@@ -1847,8 +1920,9 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       }
     }
 
-    /* Help predict this next time. We're not sure if it will need
-     * a stable circuit yet, but we know we'll need *something*. */
+    /* Help predict that we'll want to do hidden service circuits in the
+     * future. We're not sure if it will need a stable circuit yet, but
+     * we know we'll need *something*. */
     rep_hist_note_used_internal(now, 0, 1);
 
     /* Now we have a descriptor but is it usable or not? If not, refetch.
@@ -1858,14 +1932,17 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
       connection_ap_mark_as_non_pending_circuit(conn);
       base_conn->state = AP_CONN_STATE_RENDDESC_WAIT;
       log_info(LD_REND, "Unknown descriptor %s. Fetching.",
-          safe_str_client(rend_data->onion_address));
+               safe_str_client(onion_address));
       rend_client_refetch_v2_renddesc(rend_data);
       return 0;
     }
 
-    /* We have the descriptor so launch a connection to the HS. */
+    /* We have the descriptor!  So launch a connection to the HS. */
     base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT;
     log_info(LD_REND, "Descriptor is here. Great.");
+
+    /* We'll try to attach it at the next event loop, or whenever
+     * we call connection_ap_attach_pending() */
     connection_ap_mark_as_pending_circuit(conn);
     return 0;
   }
@@ -1883,7 +1960,7 @@ get_pf_socket(void)
   if (pf_socket >= 0)
     return pf_socket;
 
-#ifdef OPENBSD
+#if defined(OpenBSD)
   /* only works on OpenBSD */
   pf = tor_open_cloexec("/dev/pf", O_RDONLY, 0);
 #else
@@ -2451,8 +2528,10 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn)
     /* Sensitive directory connections must have an anonymous path length.
      * Otherwise, directory connections are typically one-hop.
      * This matches the earlier check for directory connection path anonymity
-     * in directory_initiate_command_rend(). */
-    if (is_sensitive_dir_purpose(linked_dir_conn_base->purpose)) {
+     * in directory_initiate_request(). */
+    if (purpose_needs_anonymity(linked_dir_conn_base->purpose,
+                    TO_DIR_CONN(linked_dir_conn_base)->router_purpose,
+                    TO_DIR_CONN(linked_dir_conn_base)->requested_resource)) {
       assert_circ_anonymity_ok(circ, options);
     }
   } else {
@@ -2923,7 +3002,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
   return;
 }
 
-/** Read a RELAY_BEGIN or RELAY_BEGINDIR cell from <b>cell</b>, decode it, and
+/** Read a RELAY_BEGIN or RELAY_BEGIN_DIR cell from <b>cell</b>, decode it, and
  * place the result in <b>bcell</b>.  On success return 0; on failure return
  * <0 and set *<b>end_reason_out</b> to the end reason we should send back to
  * the client.
@@ -3013,14 +3092,21 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   char *address = NULL;
   uint16_t port = 0;
   or_circuit_t *or_circ = NULL;
+  origin_circuit_t *origin_circ = NULL;
+  crypt_path_t *layer_hint = NULL;
   const or_options_t *options = get_options();
   begin_cell_t bcell;
   int rv;
   uint8_t end_reason=0;
 
   assert_circuit_ok(circ);
-  if (!CIRCUIT_IS_ORIGIN(circ))
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
     or_circ = TO_OR_CIRCUIT(circ);
+  } else {
+    tor_assert(circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    layer_hint = origin_circ->cpath->prev;
+  }
 
   relay_header_unpack(&rh, cell->payload);
   if (rh.length > RELAY_PAYLOAD_SIZE)
@@ -3045,7 +3131,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     return -END_CIRC_REASON_TORPROTOCOL;
   } else if (rv == -1) {
     tor_free(bcell.address);
-    relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, NULL);
+    relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, layer_hint);
     return 0;
   }
 
@@ -3055,15 +3141,13 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     port = bcell.port;
 
     if (or_circ && or_circ->p_chan) {
-      if (!options->AllowSingleHopExits &&
-           (or_circ->is_first_hop ||
-            (!connection_or_digest_is_known_relay(
+      if ((or_circ->is_first_hop ||
+           (!connection_or_digest_is_known_relay(
                 or_circ->p_chan->identity_digest) &&
           should_refuse_unknown_exits(options)))) {
-        /* Don't let clients use us as a single-hop proxy, unless the user
-         * has explicitly allowed that in the config. It attracts attackers
-         * and users who'd be better off with, well, single-hop proxies.
-         */
+        /* Don't let clients use us as a single-hop proxy. It attracts
+         * attackers and users who'd be better off with, well, single-hop
+         * proxies. */
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "Attempt by %s to open a stream %s. Closing.",
                safe_str(channel_get_canonical_remote_descr(or_circ->p_chan)),
@@ -3082,7 +3166,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     if (!directory_permits_begindir_requests(options) ||
         circ->purpose != CIRCUIT_PURPOSE_OR) {
       relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_NOTDIRECTORY, NULL);
+                                  END_STREAM_REASON_NOTDIRECTORY, layer_hint);
       return 0;
     }
     /* Make sure to get the 'real' address of the previous hop: the
@@ -3099,7 +3183,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   } else {
     log_warn(LD_BUG, "Got an unexpected command %d", (int)rh.command);
     relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                  END_STREAM_REASON_INTERNAL, NULL);
+                                  END_STREAM_REASON_INTERNAL, layer_hint);
     return 0;
   }
 
@@ -3110,7 +3194,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
     if (bcell.flags & BEGIN_FLAG_IPV4_NOT_OK) {
       tor_free(address);
       relay_send_end_cell_from_edge(rh.stream_id, circ,
-                                    END_STREAM_REASON_EXITPOLICY, NULL);
+                                    END_STREAM_REASON_EXITPOLICY, layer_hint);
       return 0;
     }
   }
@@ -3133,7 +3217,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
   n_stream->deliver_window = STREAMWINDOW_START;
 
   if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
-    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    tor_assert(origin_circ);
     log_info(LD_REND,"begin is for rendezvous. configuring stream.");
     n_stream->base_.address = tor_strdup("(rendezvous)");
     n_stream->base_.state = EXIT_CONN_STATE_CONNECTING;
@@ -3153,7 +3237,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
        * the hidden service. */
       relay_send_end_cell_from_edge(rh.stream_id, circ,
                                     END_STREAM_REASON_DONE,
-                                    origin_circ->cpath->prev);
+                                    layer_hint);
       connection_free(TO_CONN(n_stream));
       tor_free(address);
 
@@ -3479,7 +3563,7 @@ connection_exit_connect_dir(edge_connection_t *exitconn)
  * it is a general stream.
  */
 int
-connection_edge_is_rendezvous_stream(edge_connection_t *conn)
+connection_edge_is_rendezvous_stream(const edge_connection_t *conn)
 {
   tor_assert(conn);
   if (conn->rend_data)
diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h
index 5dfc8af901..e4780b3c7d 100644
--- a/src/or/connection_edge.h
+++ b/src/or/connection_edge.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -60,7 +60,7 @@ void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn,
 int connection_exit_begin_conn(cell_t *cell, circuit_t *circ);
 int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ);
 void connection_exit_connect(edge_connection_t *conn);
-int connection_edge_is_rendezvous_stream(edge_connection_t *conn);
+int connection_edge_is_rendezvous_stream(const edge_connection_t *conn);
 int connection_ap_can_use_exit(const entry_connection_t *conn,
                                const node_t *exit);
 void connection_ap_expire_beginning(void);
diff --git a/src/or/connection_or.c b/src/or/connection_or.c
index 8beedcae72..fcd281da26 100644
--- a/src/or/connection_or.c
+++ b/src/or/connection_or.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,6 +21,7 @@
  * This module also implements the client side of the v3 Tor link handshake,
  **/
 #include "or.h"
+#include "bridges.h"
 #include "buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
@@ -49,9 +50,12 @@
 #include "relay.h"
 #include "rephist.h"
 #include "router.h"
+#include "routerkeys.h"
 #include "routerlist.h"
 #include "ext_orport.h"
 #include "scheduler.h"
+#include "torcert.h"
+#include "channelpadding.h"
 
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
@@ -73,56 +77,25 @@ static void connection_or_mark_bad_for_new_circs(or_connection_t *or_conn);
 
 static void connection_or_change_state(or_connection_t *conn, uint8_t state);
 
-/**************************************************************/
+static void connection_or_check_canonicity(or_connection_t *conn,
+                                           int started_here);
 
-/** Map from identity digest of connected OR or desired OR to a connection_t
- * with that identity digest.  If there is more than one such connection_t,
- * they form a linked list, with next_with_same_id as the next pointer. */
-static digestmap_t *orconn_identity_map = NULL;
+/**************************************************************/
 
 /** Global map between Extended ORPort identifiers and OR
  *  connections. */
 static digestmap_t *orconn_ext_or_id_map = NULL;
 
-/** If conn is listed in orconn_identity_map, remove it, and clear
- * conn->identity_digest.  Otherwise do nothing. */
+/** Clear clear conn->identity_digest and update other data
+ * structures as appropriate.*/
 void
-connection_or_remove_from_identity_map(or_connection_t *conn)
+connection_or_clear_identity(or_connection_t *conn)
 {
-  or_connection_t *tmp;
   tor_assert(conn);
-  if (!orconn_identity_map)
-    return;
-  tmp = digestmap_get(orconn_identity_map, conn->identity_digest);
-  if (!tmp) {
-    if (!tor_digest_is_zero(conn->identity_digest)) {
-      log_warn(LD_BUG, "Didn't find connection '%s' on identity map when "
-               "trying to remove it.",
-               conn->nickname ? conn->nickname : "NULL");
-    }
-    return;
-  }
-  if (conn == tmp) {
-    if (conn->next_with_same_id)
-      digestmap_set(orconn_identity_map, conn->identity_digest,
-                    conn->next_with_same_id);
-    else
-      digestmap_remove(orconn_identity_map, conn->identity_digest);
-  } else {
-    while (tmp->next_with_same_id) {
-      if (tmp->next_with_same_id == conn) {
-        tmp->next_with_same_id = conn->next_with_same_id;
-        break;
-      }
-      tmp = tmp->next_with_same_id;
-    }
-  }
   memset(conn->identity_digest, 0, DIGEST_LEN);
-  conn->next_with_same_id = NULL;
 }
 
-/** Remove all entries from the identity-to-orconn map, and clear
- * all identities in OR conns.*/
+/** Clear all identities in OR conns.*/
 void
 connection_or_clear_identity_map(void)
 {
@@ -130,57 +103,72 @@ connection_or_clear_identity_map(void)
   SMARTLIST_FOREACH(conns, connection_t *, conn,
   {
     if (conn->type == CONN_TYPE_OR) {
-      or_connection_t *or_conn = TO_OR_CONN(conn);
-      memset(or_conn->identity_digest, 0, DIGEST_LEN);
-      or_conn->next_with_same_id = NULL;
+      connection_or_clear_identity(TO_OR_CONN(conn));
     }
   });
-
-  digestmap_free(orconn_identity_map, NULL);
-  orconn_identity_map = NULL;
 }
 
 /** Change conn->identity_digest to digest, and add conn into
- * orconn_digest_map. */
+ * the appropriate digest maps.
+ *
+ * NOTE that this function only allows two kinds of transitions: from
+ * unset identity to set identity, and from idempotent re-settings
+ * of the same identity.  It's not allowed to clear an identity or to
+ * change an identity.  Return 0 on success, and -1 if the transition
+ * is not allowed.
+ **/
 static void
-connection_or_set_identity_digest(or_connection_t *conn, const char *digest)
+connection_or_set_identity_digest(or_connection_t *conn,
+                                  const char *rsa_digest,
+                                  const ed25519_public_key_t *ed_id)
 {
-  or_connection_t *tmp;
+  channel_t *chan = NULL;
   tor_assert(conn);
-  tor_assert(digest);
+  tor_assert(rsa_digest);
 
-  if (!orconn_identity_map)
-    orconn_identity_map = digestmap_new();
-  if (tor_memeq(conn->identity_digest, digest, DIGEST_LEN))
+  if (conn->chan)
+    chan = TLS_CHAN_TO_BASE(conn->chan);
+
+  log_info(LD_HANDSHAKE, "Set identity digest for %p (%s): %s %s.",
+           conn,
+           escaped_safe_str(conn->base_.address),
+           hex_str(rsa_digest, DIGEST_LEN),
+           ed25519_fmt(ed_id));
+  log_info(LD_HANDSHAKE, "   (Previously: %s %s)",
+           hex_str(conn->identity_digest, DIGEST_LEN),
+           chan ? ed25519_fmt(&chan->ed25519_identity) : "<null>");
+
+  const int rsa_id_was_set = ! tor_digest_is_zero(conn->identity_digest);
+  const int ed_id_was_set =
+    chan && !ed25519_public_key_is_zero(&chan->ed25519_identity);
+  const int rsa_changed =
+    tor_memneq(conn->identity_digest, rsa_digest, DIGEST_LEN);
+  const int ed_changed = ed_id_was_set &&
+    (!ed_id || !ed25519_pubkey_eq(ed_id, &chan->ed25519_identity));
+
+  tor_assert(!rsa_changed || !rsa_id_was_set);
+  tor_assert(!ed_changed || !ed_id_was_set);
+
+  if (!rsa_changed && !ed_changed)
     return;
 
   /* If the identity was set previously, remove the old mapping. */
-  if (! tor_digest_is_zero(conn->identity_digest)) {
-    connection_or_remove_from_identity_map(conn);
-    if (conn->chan)
-      channel_clear_identity_digest(TLS_CHAN_TO_BASE(conn->chan));
+  if (rsa_id_was_set) {
+    connection_or_clear_identity(conn);
+    if (chan)
+      channel_clear_identity_digest(chan);
   }
 
-  memcpy(conn->identity_digest, digest, DIGEST_LEN);
+  memcpy(conn->identity_digest, rsa_digest, DIGEST_LEN);
 
-  /* If we're setting the ID to zero, don't add a mapping. */
-  if (tor_digest_is_zero(digest))
+  /* If we're initializing the IDs to zero, don't add a mapping yet. */
+  if (tor_digest_is_zero(rsa_digest) &&
+      (!ed_id || ed25519_public_key_is_zero(ed_id)))
     return;
 
-  tmp = digestmap_set(orconn_identity_map, digest, conn);
-  conn->next_with_same_id = tmp;
-
   /* Deal with channels */
-  if (conn->chan)
-    channel_set_identity_digest(TLS_CHAN_TO_BASE(conn->chan), digest);
-
-#if 1
-  /* Testing code to check for bugs in representation. */
-  for (; tmp; tmp = tmp->next_with_same_id) {
-    tor_assert(tor_memeq(tmp->identity_digest, digest, DIGEST_LEN));
-    tor_assert(tmp != conn);
-  }
-#endif
+  if (chan)
+    channel_set_identity_digest(chan, rsa_digest, ed_id);
 }
 
 /** Remove the Extended ORPort identifier of <b>conn</b> from the
@@ -731,8 +719,8 @@ connection_or_about_to_close(or_connection_t *or_conn)
       const or_options_t *options = get_options();
       connection_or_note_state_when_broken(or_conn);
       rep_hist_note_connect_failed(or_conn->identity_digest, now);
-      entry_guard_register_connect_status(or_conn->identity_digest,0,
-                                          !options->HTTPSProxy, now);
+      /* Tell the new guard API about the channel failure */
+      entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
         control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
@@ -829,24 +817,6 @@ connection_or_update_token_buckets(smartlist_t *conns,
   });
 }
 
-/** How long do we wait before killing non-canonical OR connections with no
- * circuits?  In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15
- * minutes before cancelling these connections, which caused fast relays to
- * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough
- * that it kills most idle connections, without being so low that we cause
- * clients to bounce on and off.
- *
- * For canonical connections, the limit is higher, at 15-22.5 minutes.
- *
- * For each OR connection, we randomly add up to 50% extra to its idle_timeout
- * field, to avoid exposing when exactly the last circuit closed.  Since we're
- * storing idle_timeout in a uint16_t, don't let these values get higher than
- * 12 hours or so without revising connection_or_set_canonical and/or expanding
- * idle_timeout.
- */
-#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180
-#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900
-
 /* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and
  * non-canonical otherwise. Adjust idle_timeout accordingly.
  */
@@ -854,9 +824,6 @@ void
 connection_or_set_canonical(or_connection_t *or_conn,
                             int is_canonical)
 {
-  const unsigned int timeout_base = is_canonical ?
-    IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL;
-
   if (bool_eq(is_canonical, or_conn->is_canonical) &&
       or_conn->idle_timeout != 0) {
     /* Don't recalculate an existing idle_timeout unless the canonical
@@ -865,7 +832,14 @@ connection_or_set_canonical(or_connection_t *or_conn,
   }
 
   or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */
-  or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2);
+  or_conn->idle_timeout = channelpadding_get_channel_idle_timeout(
+          TLS_CHAN_TO_BASE(or_conn->chan), is_canonical);
+
+  log_info(LD_CIRC,
+          "Channel " U64_FORMAT " chose an idle timeout of %d.",
+          or_conn->chan ?
+          U64_PRINTF_ARG(TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier):0,
+          or_conn->idle_timeout);
 }
 
 /** If we don't necessarily know the router we're connecting to, but we
@@ -877,15 +851,47 @@ void
 connection_or_init_conn_from_address(or_connection_t *conn,
                                      const tor_addr_t *addr, uint16_t port,
                                      const char *id_digest,
+                                     const ed25519_public_key_t *ed_id,
                                      int started_here)
 {
-  const node_t *r = node_get_by_id(id_digest);
-  connection_or_set_identity_digest(conn, id_digest);
+  log_debug(LD_HANDSHAKE, "init conn from address %s: %s, %s (%d)",
+            fmt_addr(addr),
+            hex_str((const char*)id_digest, DIGEST_LEN),
+            ed25519_fmt(ed_id),
+            started_here);
+
+  connection_or_set_identity_digest(conn, id_digest, ed_id);
   connection_or_update_token_buckets_helper(conn, 1, get_options());
 
   conn->base_.port = port;
   tor_addr_copy(&conn->base_.addr, addr);
   tor_addr_copy(&conn->real_addr, addr);
+
+  connection_or_check_canonicity(conn, started_here);
+}
+
+/** Check whether the identity of <b>conn</b> matches a known node.  If it
+ * does, check whether the address of conn matches the expected address, and
+ * update the connection's is_canonical flag, nickname, and address fields as
+ * appropriate. */
+static void
+connection_or_check_canonicity(or_connection_t *conn, int started_here)
+{
+  const char *id_digest = conn->identity_digest;
+  const ed25519_public_key_t *ed_id = NULL;
+  const tor_addr_t *addr = &conn->real_addr;
+  if (conn->chan)
+    ed_id = & TLS_CHAN_TO_BASE(conn->chan)->ed25519_identity;
+
+  const node_t *r = node_get_by_id(id_digest);
+  if (r &&
+      node_supports_ed25519_link_authentication(r) &&
+      ! node_ed25519_id_matches(r, ed_id)) {
+    /* If this node is capable of proving an ed25519 ID,
+     * we can't call this a canonical connection unless both IDs match. */
+     r = NULL;
+  }
+
   if (r) {
     tor_addr_port_t node_ap;
     node_get_pref_orport(r, &node_ap);
@@ -907,10 +913,12 @@ connection_or_init_conn_from_address(or_connection_t *conn,
       tor_addr_copy(&conn->base_.addr, &node_ap.addr);
       conn->base_.port = node_ap.port;
     }
+    tor_free(conn->nickname);
     conn->nickname = tor_strdup(node_get_nickname(r));
     tor_free(conn->base_.address);
     conn->base_.address = tor_addr_to_str_dup(&node_ap.addr);
   } else {
+    tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
     conn->nickname[0] = '$';
     base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1,
@@ -956,7 +964,7 @@ connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
  * too old for new circuits? */
 #define TIME_BEFORE_OR_CONN_IS_TOO_OLD (60*60*24*7)
 
-/** Given the head of the linked list for all the or_connections with a given
+/** Given a list of all the or_connections with a given
  * identity, set elements of that list as is_bad_for_new_circs as
  * appropriate. Helper for connection_or_set_bad_connections().
  *
@@ -973,16 +981,19 @@ connection_or_mark_bad_for_new_circs(or_connection_t *or_conn)
  * See channel_is_better() in channel.c for our idea of what makes one OR
  * connection better than another.
  */
-static void
-connection_or_group_set_badness(or_connection_t *head, int force)
+void
+connection_or_group_set_badness_(smartlist_t *group, int force)
 {
-  or_connection_t *or_conn = NULL, *best = NULL;
+  /* XXXX this function should be entirely about channels, not OR
+   * XXXX connections. */
+
+  or_connection_t *best = NULL;
   int n_old = 0, n_inprogress = 0, n_canonical = 0, n_other = 0;
   time_t now = time(NULL);
 
   /* Pass 1: expire everything that's old, and see what the status of
    * everything else is. */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn))
       continue;
@@ -1006,11 +1017,11 @@ connection_or_group_set_badness(or_connection_t *head, int force)
     } else {
       ++n_other;
     }
-  }
+  } SMARTLIST_FOREACH_END(or_conn);
 
   /* Pass 2: We know how about how good the best connection is.
    * expire everything that's worse, and find the very best if we can. */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn))
       continue; /* This one doesn't need to be marked bad. */
@@ -1031,13 +1042,11 @@ connection_or_group_set_badness(or_connection_t *head, int force)
     }
 
     if (!best ||
-        channel_is_better(now,
-                          TLS_CHAN_TO_BASE(or_conn->chan),
-                          TLS_CHAN_TO_BASE(best->chan),
-                          0)) {
+        channel_is_better(TLS_CHAN_TO_BASE(or_conn->chan),
+                          TLS_CHAN_TO_BASE(best->chan))) {
       best = or_conn;
     }
-  }
+  } SMARTLIST_FOREACH_END(or_conn);
 
   if (!best)
     return;
@@ -1056,17 +1065,15 @@ connection_or_group_set_badness(or_connection_t *head, int force)
    *   0.1.2.x dies out, the first case will go away, and the second one is
    *   "mostly harmless", so a fix can wait until somebody is bored.
    */
-  for (or_conn = head; or_conn; or_conn = or_conn->next_with_same_id) {
+  SMARTLIST_FOREACH_BEGIN(group, or_connection_t *, or_conn) {
     if (or_conn->base_.marked_for_close ||
         connection_or_is_bad_for_new_circs(or_conn) ||
         or_conn->base_.state != OR_CONN_STATE_OPEN)
       continue;
     if (or_conn != best &&
-        channel_is_better(now,
-                          TLS_CHAN_TO_BASE(best->chan),
-                          TLS_CHAN_TO_BASE(or_conn->chan), 1)) {
-      /* This isn't the best conn, _and_ the best conn is better than it,
-         even when we're being forgiving. */
+        channel_is_better(TLS_CHAN_TO_BASE(best->chan),
+                          TLS_CHAN_TO_BASE(or_conn->chan))) {
+      /* This isn't the best conn, _and_ the best conn is better than it */
       if (best->is_canonical) {
         log_info(LD_OR,
                  "Marking OR conn to %s:%d as unsuitable for new circuits: "
@@ -1090,24 +1097,7 @@ connection_or_group_set_badness(or_connection_t *head, int force)
         connection_or_mark_bad_for_new_circs(or_conn);
       }
     }
-  }
-}
-
-/** Go through all the OR connections (or if <b>digest</b> is non-NULL, just
- * the OR connections with that digest), and set the is_bad_for_new_circs
- * flag based on the rules in connection_or_group_set_badness() (or just
- * always set it if <b>force</b> is true).
- */
-void
-connection_or_set_bad_connections(const char *digest, int force)
-{
-  if (!orconn_identity_map)
-    return;
-
-  DIGESTMAP_FOREACH(orconn_identity_map, identity, or_connection_t *, conn) {
-    if (!digest || tor_memeq(digest, conn->identity_digest, DIGEST_LEN))
-      connection_or_group_set_badness(conn, force);
-  } DIGESTMAP_FOREACH_END;
+  } SMARTLIST_FOREACH_END(or_conn);
 }
 
 /** <b>conn</b> is in the 'connecting' state, and it failed to complete
@@ -1173,7 +1163,9 @@ connection_or_notify_error(or_connection_t *conn,
 
 MOCK_IMPL(or_connection_t *,
 connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
-                        const char *id_digest, channel_tls_t *chan))
+                        const char *id_digest,
+                        const ed25519_public_key_t *ed_id,
+                        channel_tls_t *chan))
 {
   or_connection_t *conn;
   const or_options_t *options = get_options();
@@ -1193,6 +1185,11 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     log_info(LD_PROTOCOL,"Client asked me to connect to myself. Refusing.");
     return NULL;
   }
+  if (server_mode(options) && router_ed25519_id_is_me(ed_id)) {
+    log_info(LD_PROTOCOL,"Client asked me to connect to myself by Ed25519 "
+             "identity. Refusing.");
+    return NULL;
+  }
 
   conn = or_connection_new(CONN_TYPE_OR, tor_addr_family(&addr));
 
@@ -1205,7 +1202,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
    */
   conn->chan = chan;
   chan->conn = conn;
-  connection_or_init_conn_from_address(conn, &addr, port, id_digest, 1);
+  connection_or_init_conn_from_address(conn, &addr, port, id_digest, ed_id, 1);
   connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
   control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
 
@@ -1552,7 +1549,9 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
   }
 
   if (identity_rcvd) {
-    crypto_pk_get_digest(identity_rcvd, digest_rcvd_out);
+    if (crypto_pk_get_digest(identity_rcvd, digest_rcvd_out) < 0) {
+      return -1;
+    }
   } else {
     memset(digest_rcvd_out, 0, DIGEST_LEN);
   }
@@ -1562,18 +1561,25 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
 
   crypto_pk_free(identity_rcvd);
 
-  if (started_here)
+  if (started_here) {
+    /* A TLS handshake can't teach us an Ed25519 ID, so we set it to NULL
+     * here. */
+    log_debug(LD_HANDSHAKE, "Calling client_learned_peer_id from "
+              "check_valid_tls_handshake");
     return connection_or_client_learned_peer_id(conn,
-                                     (const uint8_t*)digest_rcvd_out);
+                                        (const uint8_t*)digest_rcvd_out,
+                                        NULL);
+  }
 
   return 0;
 }
 
 /** Called when we (as a connection initiator) have definitively,
  * authenticatedly, learned that ID of the Tor instance on the other
- * side of <b>conn</b> is <b>peer_id</b>.  For v1 and v2 handshakes,
+ * side of <b>conn</b> is <b>rsa_peer_id</b> and optionally <b>ed_peer_id</b>.
+ * For v1 and v2 handshakes,
  * this is right after we get a certificate chain in a TLS handshake
- * or renegotiation.  For v3 handshakes, this is right after we get a
+ * or renegotiation.  For v3+ handshakes, this is right after we get a
  * certificate chain in a CERTS cell.
  *
  * If we did not know the ID before, record the one we got.
@@ -1594,12 +1600,31 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn,
  */
 int
 connection_or_client_learned_peer_id(or_connection_t *conn,
-                                     const uint8_t *peer_id)
+                                     const uint8_t *rsa_peer_id,
+                                     const ed25519_public_key_t *ed_peer_id)
 {
   const or_options_t *options = get_options();
-
-  if (tor_digest_is_zero(conn->identity_digest)) {
-    connection_or_set_identity_digest(conn, (const char*)peer_id);
+  channel_tls_t *chan_tls = conn->chan;
+  channel_t *chan = channel_tls_to_base(chan_tls);
+  int changed_identity = 0;
+  tor_assert(chan);
+
+  const int expected_rsa_key =
+    ! tor_digest_is_zero(conn->identity_digest);
+  const int expected_ed_key =
+    ! ed25519_public_key_is_zero(&chan->ed25519_identity);
+
+  log_info(LD_HANDSHAKE, "learned peer id for %p (%s): %s, %s",
+           conn,
+           safe_str_client(conn->base_.address),
+           hex_str((const char*)rsa_peer_id, DIGEST_LEN),
+           ed25519_fmt(ed_peer_id));
+
+  if (! expected_rsa_key && ! expected_ed_key) {
+    log_info(LD_HANDSHAKE, "(we had no ID in mind when we made this "
+             "connection.");
+    connection_or_set_identity_digest(conn,
+                                      (const char*)rsa_peer_id, ed_peer_id);
     tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
     conn->nickname[0] = '$';
@@ -1611,16 +1636,39 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     /* if it's a bridge and we didn't know its identity fingerprint, now
      * we do -- remember it for future attempts. */
     learned_router_identity(&conn->base_.addr, conn->base_.port,
-                            (const char*)peer_id);
+                            (const char*)rsa_peer_id, ed_peer_id);
+    changed_identity = 1;
   }
 
-  if (tor_memneq(peer_id, conn->identity_digest, DIGEST_LEN)) {
+  const int rsa_mismatch = expected_rsa_key &&
+    tor_memneq(rsa_peer_id, conn->identity_digest, DIGEST_LEN);
+  /* It only counts as an ed25519 mismatch if we wanted an ed25519 identity
+   * and didn't get it. It's okay if we get one that we didn't ask for. */
+  const int ed25519_mismatch =
+    expected_ed_key &&
+    (ed_peer_id == NULL ||
+     ! ed25519_pubkey_eq(&chan->ed25519_identity, ed_peer_id));
+
+  if (rsa_mismatch || ed25519_mismatch) {
     /* I was aiming for a particular digest. I didn't get it! */
-    char seen[HEX_DIGEST_LEN+1];
-    char expected[HEX_DIGEST_LEN+1];
-    base16_encode(seen, sizeof(seen), (const char*)peer_id, DIGEST_LEN);
-    base16_encode(expected, sizeof(expected), conn->identity_digest,
+    char seen_rsa[HEX_DIGEST_LEN+1];
+    char expected_rsa[HEX_DIGEST_LEN+1];
+    char seen_ed[ED25519_BASE64_LEN+1];
+    char expected_ed[ED25519_BASE64_LEN+1];
+    base16_encode(seen_rsa, sizeof(seen_rsa),
+                  (const char*)rsa_peer_id, DIGEST_LEN);
+    base16_encode(expected_rsa, sizeof(expected_rsa), conn->identity_digest,
                   DIGEST_LEN);
+    if (ed_peer_id) {
+      ed25519_public_to_base64(seen_ed, ed_peer_id);
+    } else {
+      strlcpy(seen_ed, "no ed25519 key", sizeof(seen_ed));
+    }
+    if (! ed25519_public_key_is_zero(&chan->ed25519_identity)) {
+      ed25519_public_to_base64(expected_ed, &chan->ed25519_identity);
+    } else {
+      strlcpy(expected_ed, "no ed25519 key", sizeof(expected_ed));
+    }
     const int using_hardcoded_fingerprints =
       !networkstatus_get_reasonably_live_consensus(time(NULL),
                                                    usable_consensus_flavor());
@@ -1655,11 +1703,13 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
     }
 
     log_fn(severity, LD_HANDSHAKE,
-           "Tried connecting to router at %s:%d, but identity key was not "
-           "as expected: wanted %s but got %s.%s",
-           conn->base_.address, conn->base_.port, expected, seen, extra_log);
-    entry_guard_register_connect_status(conn->identity_digest, 0, 1,
-                                        time(NULL));
+           "Tried connecting to router at %s:%d, but RSA identity key was not "
+           "as expected: wanted %s + %s but got %s + %s.%s",
+           conn->base_.address, conn->base_.port,
+           expected_rsa, expected_ed, seen_rsa, seen_ed, extra_log);
+
+    /* Tell the new guard API about the channel failure */
+    entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
     control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
                                  END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
@@ -1669,9 +1719,24 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
                                 conn);
     return -1;
   }
+
+  if (!expected_ed_key && ed_peer_id) {
+    log_info(LD_HANDSHAKE, "(we had no Ed25519 ID in mind when we made this "
+             "connection.");
+    connection_or_set_identity_digest(conn,
+                                      (const char*)rsa_peer_id, ed_peer_id);
+    changed_identity = 1;
+  }
+
+  if (changed_identity) {
+    /* If we learned an identity for this connection, then we might have
+     * just discovered it to be canonical. */
+    connection_or_check_canonicity(conn, conn->handshake_state->started_here);
+  }
+
   if (authdir_mode_tests_reachability(options)) {
     dirserv_orconn_tls_done(&conn->base_.addr, conn->base_.port,
-                            (const char*)peer_id);
+                            (const char*)rsa_peer_id, ed_peer_id);
   }
 
   return 0;
@@ -1727,7 +1792,8 @@ connection_tls_finish_handshake(or_connection_t *conn)
   if (tor_tls_used_v1_handshake(conn->tls)) {
     conn->link_proto = 1;
     connection_or_init_conn_from_address(conn, &conn->base_.addr,
-                                         conn->base_.port, digest_rcvd, 0);
+                                         conn->base_.port, digest_rcvd,
+                                         NULL, 0);
     tor_tls_block_renegotiation(conn->tls);
     rep_hist_note_negotiated_link_proto(1, started_here);
     return connection_or_set_state_open(conn);
@@ -1736,7 +1802,8 @@ connection_tls_finish_handshake(or_connection_t *conn)
     if (connection_init_or_handshake_state(conn, started_here) < 0)
       return -1;
     connection_or_init_conn_from_address(conn, &conn->base_.addr,
-                                         conn->base_.port, digest_rcvd, 0);
+                                         conn->base_.port, digest_rcvd,
+                                         NULL, 0);
     return connection_or_send_versions(conn, 0);
   }
 }
@@ -1775,6 +1842,11 @@ connection_init_or_handshake_state(or_connection_t *conn, int started_here)
   s->started_here = started_here ? 1 : 0;
   s->digest_sent_data = 1;
   s->digest_received_data = 1;
+  if (! started_here && get_current_link_cert_cert()) {
+    s->own_link_cert = tor_cert_dup(get_current_link_cert_cert());
+  }
+  s->certs = or_handshake_certs_new();
+  s->certs->started_here = s->started_here;
   return 0;
 }
 
@@ -1786,8 +1858,8 @@ or_handshake_state_free(or_handshake_state_t *state)
     return;
   crypto_digest_free(state->digest_sent);
   crypto_digest_free(state->digest_received);
-  tor_x509_cert_free(state->auth_cert);
-  tor_x509_cert_free(state->id_cert);
+  or_handshake_certs_free(state->certs);
+  tor_cert_free(state->own_link_cert);
   memwipe(state, 0xBE, sizeof(or_handshake_state_t));
   tor_free(state);
 }
@@ -1908,12 +1980,23 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (cell->command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+
   connection_write_to_buf(networkcell.body, cell_network_size, TO_CONN(conn));
 
   /* Touch the channel's active timestamp if there is one */
-  if (conn->chan)
+  if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
+    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+      if (cell->command == CELL_PADDING)
+        rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+    }
+  }
+
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0);
 }
@@ -2019,7 +2102,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn)
 }
 
 /** Array of recognized link protocol versions. */
-static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4 };
+static const uint16_t or_protocol_versions[] = { 1, 2, 3, 4, 5 };
 /** Number of versions in <b>or_protocol_versions</b>. */
 static const int n_or_protocol_versions =
   (int)( sizeof(or_protocol_versions)/sizeof(uint16_t) );
@@ -2140,66 +2223,187 @@ connection_or_send_netinfo,(or_connection_t *conn))
   return 0;
 }
 
+/** Helper used to add an encoded certs to a cert cell */
+static void
+add_certs_cell_cert_helper(certs_cell_t *certs_cell,
+                           uint8_t cert_type,
+                           const uint8_t *cert_encoded,
+                           size_t cert_len)
+{
+  tor_assert(cert_len <= UINT16_MAX);
+  certs_cell_cert_t *ccc = certs_cell_cert_new();
+  ccc->cert_type = cert_type;
+  ccc->cert_len = cert_len;
+  certs_cell_cert_setlen_body(ccc, cert_len);
+  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
+
+  certs_cell_add_certs(certs_cell, ccc);
+}
+
+/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
+ * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
+ * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
+ * (If <b>cert</b> is NULL, take no action.) */
+static void
+add_x509_cert(certs_cell_t *certs_cell,
+              uint8_t cert_type,
+              const tor_x509_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  const uint8_t *cert_encoded = NULL;
+  size_t cert_len;
+  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+
+  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
+}
+
+/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
+ * that we are building in <b>certs_cell</b>.  Set its type field to
+ * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
+static void
+add_ed25519_cert(certs_cell_t *certs_cell,
+                 uint8_t cert_type,
+                 const tor_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  add_certs_cell_cert_helper(certs_cell, cert_type,
+                             cert->encoded, cert->encoded_len);
+}
+
+#ifdef TOR_UNIT_TESTS
+int certs_cell_ed25519_disabled_for_testing = 0;
+#else
+#define certs_cell_ed25519_disabled_for_testing 0
+#endif
+
 /** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
  * on failure. */
 int
 connection_or_send_certs_cell(or_connection_t *conn)
 {
-  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL,
-    *using_link_cert = NULL;
+  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
   tor_x509_cert_t *own_link_cert = NULL;
-  const uint8_t *link_encoded = NULL, *id_encoded = NULL;
-  size_t link_len, id_len;
   var_cell_t *cell;
-  size_t cell_len;
-  ssize_t pos;
+
+  certs_cell_t *certs_cell = NULL;
 
   tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
 
   if (! conn->handshake_state)
     return -1;
+
   const int conn_in_server_mode = ! conn->handshake_state->started_here;
+
+  /* Get the encoded values of the X509 certificates */
   if (tor_tls_get_my_certs(conn_in_server_mode,
                            &global_link_cert, &id_cert) < 0)
     return -1;
+
   if (conn_in_server_mode) {
-    using_link_cert = own_link_cert = tor_tls_get_own_cert(conn->tls);
+    own_link_cert = tor_tls_get_own_cert(conn->tls);
+  }
+  tor_assert(id_cert);
+
+  certs_cell = certs_cell_new();
+
+  /* Start adding certs.  First the link cert or auth1024 cert. */
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(own_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
   } else {
-    using_link_cert = global_link_cert;
+    tor_assert(global_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
   }
-  tor_x509_cert_get_der(using_link_cert, &link_encoded, &link_len);
-  tor_x509_cert_get_der(id_cert, &id_encoded, &id_len);
 
-  cell_len = 1 /* 1 byte: num certs in cell */ +
-             2 * ( 1 + 2 ) /* For each cert: 1 byte for type, 2 for length */ +
-             link_len + id_len;
-  cell = var_cell_new(cell_len);
-  cell->command = CELL_CERTS;
-  cell->payload[0] = 2;
-  pos = 1;
+  /* Next the RSA->RSA ID cert */
+  add_x509_cert(certs_cell,
+                OR_CERT_TYPE_ID_1024, id_cert);
 
-  if (conn_in_server_mode)
-    cell->payload[pos] = OR_CERT_TYPE_TLS_LINK; /* Link cert  */
-  else
-    cell->payload[pos] = OR_CERT_TYPE_AUTH_1024; /* client authentication */
-  set_uint16(&cell->payload[pos+1], htons(link_len));
-  memcpy(&cell->payload[pos+3], link_encoded, link_len);
-  pos += 3 + link_len;
+  /* Next the Ed25519 certs */
+  add_ed25519_cert(certs_cell,
+                   CERTTYPE_ED_ID_SIGN,
+                   get_master_signing_key_cert());
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
+                        certs_cell_ed25519_disabled_for_testing);
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_LINK,
+                     conn->handshake_state->own_link_cert);
+  } else {
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_AUTH,
+                     get_current_auth_key_cert());
+  }
+
+  /* And finally the crosscert. */
+  {
+    const uint8_t *crosscert=NULL;
+    size_t crosscert_len;
+    get_master_rsa_crosscert(&crosscert, &crosscert_len);
+    if (crosscert) {
+      add_certs_cell_cert_helper(certs_cell,
+                               CERTTYPE_RSA1024_ID_EDID,
+                               crosscert, crosscert_len);
+    }
+  }
 
-  cell->payload[pos] = OR_CERT_TYPE_ID_1024; /* ID cert */
-  set_uint16(&cell->payload[pos+1], htons(id_len));
-  memcpy(&cell->payload[pos+3], id_encoded, id_len);
-  pos += 3 + id_len;
+  /* We've added all the certs; make the cell. */
+  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
 
-  tor_assert(pos == (int)cell_len); /* Otherwise we just smashed the heap */
+  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
+  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
+  cell = var_cell_new(alloc_len);
+  cell->command = CELL_CERTS;
+  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
+  tor_assert(enc_len > 0 && enc_len <= alloc_len);
+  cell->payload_len = enc_len;
 
   connection_or_write_var_cell_to_buf(cell, conn);
   var_cell_free(cell);
+  certs_cell_free(certs_cell);
   tor_x509_cert_free(own_link_cert);
 
   return 0;
 }
 
+/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
+ * we can send and receive. */
+int
+authchallenge_type_is_supported(uint16_t challenge_type)
+{
+  switch (challenge_type) {
+     case AUTHTYPE_RSA_SHA256_TLSSECRET:
+     case AUTHTYPE_ED25519_SHA256_RFC5705:
+       return 1;
+     case AUTHTYPE_RSA_SHA256_RFC5705:
+     default:
+       return 0;
+  }
+}
+
+/** Return true iff <b>challenge_type_a</b> is one that we would rather
+ * use than <b>challenge_type_b</b>. */
+int
+authchallenge_type_is_better(uint16_t challenge_type_a,
+                             uint16_t challenge_type_b)
+{
+  /* Any supported type is better than an unsupported one;
+   * all unsupported types are equally bad. */
+  if (!authchallenge_type_is_supported(challenge_type_a))
+    return 0;
+  if (!authchallenge_type_is_supported(challenge_type_b))
+    return 1;
+  /* It happens that types are superior in numerically ascending order.
+   * If that ever changes, this must change too. */
+  return (challenge_type_a > challenge_type_b);
+}
+
 /** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
  * on success, -1 on failure. */
 int
@@ -2214,17 +2418,26 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
 
   auth_challenge_cell_t *ac = auth_challenge_cell_new();
 
+  tor_assert(sizeof(ac->challenge) == 32);
   crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
 
   auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
+  /* Disabled, because everything that supports this method also supports
+   * the much-superior ED25519_SHA256_RFC5705 */
+  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
+  auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
   auth_challenge_cell_set_n_methods(ac,
                                     auth_challenge_cell_getlen_methods(ac));
 
   cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
   ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
                                            ac);
-  if (len != cell->payload_len)
+  if (len != cell->payload_len) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
     goto done;
+    /* LCOV_EXCL_STOP */
+  }
   cell->command = CELL_AUTH_CHALLENGE;
 
   connection_or_write_var_cell_to_buf(cell, conn);
@@ -2238,8 +2451,8 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
 }
 
 /** Compute the main body of an AUTHENTICATE cell that a client can use
- * to authenticate itself on a v3 handshake for <b>conn</b>.  Write it to the
- * <b>outlen</b>-byte buffer at <b>out</b>.
+ * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
+ * in a var_cell_t.
  *
  * If <b>server</b> is true, only calculate the first
  * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
@@ -2255,24 +2468,44 @@ connection_or_send_auth_challenge_cell(or_connection_t *conn)
  *
  * Return the length of the cell body on success, and -1 on failure.
  */
-int
+var_cell_t *
 connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                             uint8_t *out, size_t outlen,
+                                             const int authtype,
                                              crypto_pk_t *signing_key,
-                                             int server)
+                                      const ed25519_keypair_t *ed_signing_key,
+                                      int server)
 {
   auth1_t *auth = NULL;
   auth_ctx_t *ctx = auth_ctx_new();
-  int result;
+  var_cell_t *result = NULL;
+  int old_tlssecrets_algorithm = 0;
+  const char *authtype_str = NULL;
 
-  /* assert state is reasonable XXXX */
+  int is_ed = 0;
 
-  ctx->is_ed = 0;
+  /* assert state is reasonable XXXX */
+  switch (authtype) {
+  case AUTHTYPE_RSA_SHA256_TLSSECRET:
+    authtype_str = "AUTH0001";
+    old_tlssecrets_algorithm = 1;
+    break;
+  case AUTHTYPE_RSA_SHA256_RFC5705:
+    authtype_str = "AUTH0002";
+    break;
+  case AUTHTYPE_ED25519_SHA256_RFC5705:
+    authtype_str = "AUTH0003";
+    is_ed = 1;
+    break;
+  default:
+    tor_assert(0);
+    break;
+  }
 
   auth = auth1_new();
+  ctx->is_ed = is_ed;
 
   /* Type: 8 bytes. */
-  memcpy(auth1_getarray_type(auth), "AUTH0001", 8);
+  memcpy(auth1_getarray_type(auth), authtype_str, 8);
 
   {
     const tor_x509_cert_t *id_cert=NULL;
@@ -2282,7 +2515,7 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
       goto err;
     my_digests = tor_x509_cert_get_id_digests(id_cert);
     their_digests =
-      tor_x509_cert_get_id_digests(conn->handshake_state->id_cert);
+      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
     tor_assert(my_digests);
     tor_assert(their_digests);
     my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
@@ -2298,6 +2531,22 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
     memcpy(auth->sid, server_id, 32);
   }
 
+  if (is_ed) {
+    const ed25519_public_key_t *my_ed_id, *their_ed_id;
+    if (!conn->handshake_state->certs->ed_id_sign) {
+      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
+      goto err;
+    }
+    my_ed_id = get_master_identity_key();
+    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
+
+    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
+    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
+
+    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
+    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
+  }
+
   {
     crypto_digest_t *server_d, *client_d;
     if (server) {
@@ -2324,7 +2573,8 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
       cert = tor_tls_get_peer_cert(conn->tls);
     }
     if (!cert) {
-      log_warn(LD_OR, "Unable to find cert when making AUTH1 data.");
+      log_warn(LD_OR, "Unable to find cert when making %s data.",
+               authtype_str);
       goto err;
     }
 
@@ -2335,36 +2585,79 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
   }
 
   /* HMAC of clientrandom and serverrandom using master key : 32 octets */
-  tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets);
+  if (old_tlssecrets_algorithm) {
+    tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets);
+  } else {
+    char label[128];
+    tor_snprintf(label, sizeof(label),
+                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
+    tor_tls_export_key_material(conn->tls, auth->tlssecrets,
+                                auth->cid, sizeof(auth->cid),
+                                label);
+  }
 
   /* 8 octets were reserved for the current time, but we're trying to get out
    * of the habit of sending time around willynilly.  Fortunately, nothing
    * checks it.  That's followed by 16 bytes of nonce. */
   crypto_rand((char*)auth->rand, 24);
 
+  ssize_t maxlen = auth1_encoded_len(auth, ctx);
+  if (ed_signing_key && is_ed) {
+    maxlen += ED25519_SIG_LEN;
+  } else if (signing_key && !is_ed) {
+    maxlen += crypto_pk_keysize(signing_key);
+  }
+
+  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
+  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
+  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
+  const size_t outlen = maxlen;
   ssize_t len;
+
+  result->command = CELL_AUTHENTICATE;
+  set_uint16(result->payload, htons(authtype));
+
   if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
-    log_warn(LD_OR, "Unable to encode signed part of AUTH1 data.");
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
     goto err;
+    /* LCOV_EXCL_STOP */
   }
 
   if (server) {
     auth1_t *tmp = NULL;
     ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
     if (!tmp) {
-      log_warn(LD_OR, "Unable to parse signed part of AUTH1 data.");
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
+               "we just encoded");
       goto err;
+      /* LCOV_EXCL_STOP */
     }
-    result = (int) (tmp->end_of_fixed_part - out);
+    result->payload_len = (tmp->end_of_signed - result->payload);
+
     auth1_free(tmp);
     if (len2 != len) {
-      log_warn(LD_OR, "Mismatched length when re-parsing AUTH1 data.");
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
       goto err;
+      /* LCOV_EXCL_STOP */
     }
     goto done;
   }
 
-  if (signing_key) {
+  if (ed_signing_key && is_ed) {
+    ed25519_signature_t sig;
+    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    auth1_setlen_sig(auth, ED25519_SIG_LEN);
+    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
+
+  } else if (signing_key && !is_ed) {
     auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
 
     char d[32];
@@ -2379,18 +2672,24 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn,
     }
 
     auth1_setlen_sig(auth, siglen);
+  }
 
-    len = auth1_encode(out, outlen, auth, ctx);
-    if (len < 0) {
-      log_warn(LD_OR, "Unable to encode signed AUTH1 data.");
-      goto err;
-    }
+  len = auth1_encode(out, outlen, auth, ctx);
+  if (len < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
   }
-  result = (int) len;
+  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
+  result->payload_len = len + AUTH_CELL_HEADER_LEN;
+  set_uint16(result->payload+2, htons(len));
+
   goto done;
 
  err:
-  result = -1;
+  var_cell_free(result);
+  result = NULL;
  done:
   auth1_free(auth);
   auth_ctx_free(ctx);
@@ -2404,44 +2703,29 @@ connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
 {
   var_cell_t *cell;
   crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
-  int authlen;
-  size_t cell_maxlen;
   /* XXXX make sure we're actually supposed to send this! */
 
   if (!pk) {
     log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
     return -1;
   }
-  if (authtype != AUTHTYPE_RSA_SHA256_TLSSECRET) {
+  if (! authchallenge_type_is_supported(authtype)) {
     log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
              "authentication type %d", authtype);
     return -1;
   }
 
-  cell_maxlen = 4 + /* overhead */
-    V3_AUTH_BODY_LEN + /* Authentication body */
-    crypto_pk_keysize(pk) + /* Max signature length */
-    16 /* add a few extra bytes just in case. */;
-
-  cell = var_cell_new(cell_maxlen);
-  cell->command = CELL_AUTHENTICATE;
-  set_uint16(cell->payload, htons(AUTHTYPE_RSA_SHA256_TLSSECRET));
-  /* skip over length ; we don't know that yet. */
-
-  authlen = connection_or_compute_authenticate_cell_body(conn,
-                                                         cell->payload+4,
-                                                         cell_maxlen-4,
-                                                         pk,
-                                                         0 /* not server */);
-  if (authlen < 0) {
+  cell = connection_or_compute_authenticate_cell_body(conn,
+                                                 authtype,
+                                                 pk,
+                                                 get_current_auth_keypair(),
+                                                 0 /* not server */);
+  if (! cell) {
+    /* LCOV_EXCL_START */
     log_warn(LD_BUG, "Unable to compute authenticate cell!");
-    var_cell_free(cell);
     return -1;
+    /* LCOV_EXCL_STOP */
   }
-  tor_assert(authlen + 4 <= cell->payload_len);
-  set_uint16(cell->payload+2, htons(authlen));
-  cell->payload_len = authlen + 4;
-
   connection_or_write_var_cell_to_buf(cell, conn);
   var_cell_free(cell);
 
diff --git a/src/or/connection_or.h b/src/or/connection_or.h
index 2e8c6066cc..fe85a3f5fd 100644
--- a/src/or/connection_or.h
+++ b/src/or/connection_or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,14 +12,13 @@
 #ifndef TOR_CONNECTION_OR_H
 #define TOR_CONNECTION_OR_H
 
-void connection_or_remove_from_identity_map(or_connection_t *conn);
+void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
 void clear_broken_connection_map(int disable);
 or_connection_t *connection_or_get_for_extend(const char *digest,
                                               const tor_addr_t *target_addr,
                                               const char **msg_out,
                                               int *launch_out);
-void connection_or_set_bad_connections(const char *digest, int force);
 
 void connection_or_block_renegotiation(or_connection_t *conn);
 int connection_or_reached_eof(or_connection_t *conn);
@@ -40,7 +39,9 @@ void connection_or_notify_error(or_connection_t *conn,
 MOCK_DECL(or_connection_t *,
           connection_or_connect,
           (const tor_addr_t *addr, uint16_t port,
-           const char *id_digest, channel_tls_t *chan));
+           const char *id_digest,
+           const ed25519_public_key_t *ed_id,
+           channel_tls_t *chan));
 
 void connection_or_close_normally(or_connection_t *orconn, int flush);
 MOCK_DECL(void,connection_or_close_for_error,
@@ -59,10 +60,12 @@ int connection_init_or_handshake_state(or_connection_t *conn,
 void connection_or_init_conn_from_address(or_connection_t *conn,
                                           const tor_addr_t *addr,
                                           uint16_t port,
-                                          const char *id_digest,
+                                          const char *rsa_id_digest,
+                                          const ed25519_public_key_t *ed_id,
                                           int started_here);
 int connection_or_client_learned_peer_id(or_connection_t *conn,
-                                         const uint8_t *peer_id);
+                              const uint8_t *rsa_peer_id,
+                              const ed25519_public_key_t *ed_peer_id);
 time_t connection_or_client_used(or_connection_t *conn);
 MOCK_DECL(int, connection_or_get_num_circuits, (or_connection_t *conn));
 void or_handshake_state_free(or_handshake_state_t *state);
@@ -84,10 +87,14 @@ int connection_or_send_versions(or_connection_t *conn, int v3_plus);
 MOCK_DECL(int,connection_or_send_netinfo,(or_connection_t *conn));
 int connection_or_send_certs_cell(or_connection_t *conn);
 int connection_or_send_auth_challenge_cell(or_connection_t *conn);
-int connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                                 uint8_t *out, size_t outlen,
-                                                 crypto_pk_t *signing_key,
-                                                 int server);
+int authchallenge_type_is_supported(uint16_t challenge_type);
+int authchallenge_type_is_better(uint16_t challenge_type_a,
+                                 uint16_t challenge_type_b);
+var_cell_t *connection_or_compute_authenticate_cell_body(or_connection_t *conn,
+                                       const int authtype,
+                                       crypto_pk_t *signing_key,
+                                       const ed25519_keypair_t *ed_signing_key,
+                                       int server);
 MOCK_DECL(int,connection_or_send_authenticate_cell,
           (or_connection_t *conn, int type));
 
@@ -102,6 +109,14 @@ void var_cell_free(var_cell_t *cell);
 
 /* DOCDOC */
 #define MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS 4
+#define MIN_LINK_PROTO_FOR_CHANNEL_PADDING 5
+#define MAX_LINK_PROTO MIN_LINK_PROTO_FOR_CHANNEL_PADDING
+
+void connection_or_group_set_badness_(smartlist_t *group, int force);
+
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
+#endif
 
 #endif
 
diff --git a/src/or/conscache.c b/src/or/conscache.c
new file mode 100644
index 0000000000..33a5495974
--- /dev/null
+++ b/src/or/conscache.c
@@ -0,0 +1,626 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+
+#include "config.h"
+#include "conscache.h"
+#include "storagedir.h"
+
+#define CCE_MAGIC 0x17162253
+
+#ifdef _WIN32
+/* On Windows, unlink won't work on a file if the file is actively mmap()ed.
+ * That forces us to be less aggressive about unlinking files, and causes other
+ * changes throughout our logic.
+ */
+#define MUST_UNMAP_TO_UNLINK
+#endif
+
+/**
+ * A consensus_cache_entry_t is a reference-counted handle to an
+ * item in a consensus_cache_t.  It can be mmapped into RAM, or not,
+ * depending whether it's currently in use.
+ */
+struct consensus_cache_entry_t {
+  uint32_t magic; /**< Must be set to CCE_MAGIC */
+  HANDLE_ENTRY(consensus_cache_entry, consensus_cache_entry_t);
+  int32_t refcnt; /**< Reference count. */
+  unsigned can_remove : 1; /**< If true, we want to delete this file. */
+  /** If true, we intend to unmap this file as soon as we're done with it. */
+  unsigned release_aggressively : 1;
+
+  /** Filename for this object within the storage_dir_t */
+  char *fname;
+  /** Labels associated with this object. Immutable once the object
+   * is created. */
+  config_line_t *labels;
+  /** Pointer to the cache that includes this entry (if any). */
+  consensus_cache_t *in_cache;
+
+  /** Since what time has this object been mapped into RAM, but with the cache
+   * being the only having a reference to it? */
+  time_t unused_since;
+  /** mmaped contents of the underlying file.  May be NULL */
+  tor_mmap_t *map;
+  /** Length of the body within <b>map</b>. */
+  size_t bodylen;
+  /** Pointer to the body within <b>map</b>. */
+  const uint8_t *body;
+};
+
+/**
+ * A consensus_cache_t holds a directory full of labeled items.
+ */
+struct consensus_cache_t {
+  /** Underling storage_dir_t to handle persistence */
+  storage_dir_t *dir;
+  /** List of all the entries in the directory. */
+  smartlist_t *entries;
+
+  /** The maximum number of entries that we'd like to allow in this cache.
+   * This is the same as the storagedir limit when MUST_UNMAP_TO_UNLINK is
+   * not defined. */
+  unsigned max_entries;
+};
+
+static void consensus_cache_clear(consensus_cache_t *cache);
+static void consensus_cache_rescan(consensus_cache_t *);
+static void consensus_cache_entry_map(consensus_cache_t *,
+                                      consensus_cache_entry_t *);
+static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);
+
+/**
+ * Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
+ * data directory, to hold up to <b>max_entries</b> of data.
+ */
+consensus_cache_t *
+consensus_cache_open(const char *subdir, int max_entries)
+{
+  int storagedir_max_entries;
+  consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
+  char *directory = get_datadir_fname(subdir);
+  cache->max_entries = max_entries;
+
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* If we can't unlink the files that we're still using, then we need to
+   * tell the storagedir backend to allow far more files than this consensus
+   * cache actually wants, so that it can hold files which, from this cache's
+   * perspective, have become useless.
+   */
+#define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
+  storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
+#else
+  /* Otherwise, we can just tell the storagedir to use the same limits
+   * as this cache. */
+  storagedir_max_entries = max_entries;
+#endif
+
+  cache->dir = storage_dir_new(directory, storagedir_max_entries);
+  tor_free(directory);
+  if (!cache->dir) {
+    tor_free(cache);
+    return NULL;
+  }
+
+  consensus_cache_rescan(cache);
+  return cache;
+}
+
+/** Return true if it's okay to put more entries in this cache than
+ * its official file limit.
+ *
+ * (We need this method on Windows, where we can't unlink files that are still
+ * in use, and therefore might need to temporarily exceed the file limit until
+ * the no-longer-wanted files are deletable.)
+ */
+int
+consensus_cache_may_overallocate(consensus_cache_t *cache)
+{
+  (void) cache;
+#ifdef MUST_UNMAP_TO_UNLINK
+  return 1;
+#else
+  return 0;
+#endif
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that <b>cache</b> will need.
+ */
+int
+consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                      struct sandbox_cfg_elem **cfg)
+{
+#ifdef MUST_UNMAP_TO_UNLINK
+  /* Our Linux sandbox doesn't support huge file lists like the one that would
+   * be generated by using VERY_LARGE_STORAGEDIR_LIMIT above in
+   * consensus_cache_open().  Since the Linux sandbox is the only one we have
+   * right now, we just assert that we never reach this point when we've had
+   * to use VERY_LARGE_STORAGEDIR_LIMIT.
+   *
+   * If at some point in the future we have a different sandbox mechanism that
+   * can handle huge file lists, we can remove this assertion or make it
+   * conditional.
+   */
+  tor_assert_nonfatal_unreached();
+#endif
+  return storage_dir_register_with_sandbox(cache->dir, cfg);
+}
+
+/**
+ * Helper: clear all entries from <b>cache</b> (but do not delete
+ * any that aren't marked for removal
+ */
+static void
+consensus_cache_clear(consensus_cache_t *cache)
+{
+  consensus_cache_delete_pending(cache, 0);
+
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    ent->in_cache = NULL;
+    consensus_cache_entry_decref(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(cache->entries);
+  cache->entries = NULL;
+}
+
+/**
+ * Drop all storage held by <b>cache</b>.
+ */
+void
+consensus_cache_free(consensus_cache_t *cache)
+{
+  if (! cache)
+    return;
+
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+  storage_dir_free(cache->dir);
+  tor_free(cache);
+}
+
+/**
+ * Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
+ * labeling that data with <b>labels</b>.  On failure, return NULL. On
+ * success, return a newly created consensus_cache_entry_t.
+ *
+ * The returned value will be owned by the cache, and you will have a
+ * reference to it.  Call consensus_cache_entry_decref() when you are
+ * done with it.
+ *
+ * The provided <b>labels</b> MUST have distinct keys: if they don't,
+ * this API does not specify which values (if any) for the duplicate keys
+ * will be considered.
+ */
+consensus_cache_entry_t *
+consensus_cache_add(consensus_cache_t *cache,
+                    const config_line_t *labels,
+                    const uint8_t *data,
+                    size_t datalen)
+{
+  char *fname = NULL;
+  int r = storage_dir_save_labeled_to_file(cache->dir,
+                                            labels, data, datalen, &fname);
+  if (r < 0 || fname == NULL) {
+    return NULL;
+  }
+  consensus_cache_entry_t *ent =
+    tor_malloc_zero(sizeof(consensus_cache_entry_t));
+  ent->magic = CCE_MAGIC;
+  ent->fname = fname;
+  ent->labels = config_lines_dup(labels);
+  ent->in_cache = cache;
+  ent->unused_since = TIME_MAX;
+  smartlist_add(cache->entries, ent);
+  /* Start the reference count at 2: the caller owns one copy, and the
+   * cache owns another.
+   */
+  ent->refcnt = 2;
+
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
+ * Return NULL if no such entry exists.
+ *
+ * Does not adjust reference counts.
+ */
+consensus_cache_entry_t *
+consensus_cache_find_first(consensus_cache_t *cache,
+                           const char *key,
+                           const char *value)
+{
+  smartlist_t *tmp = smartlist_new();
+  consensus_cache_find_all(tmp, cache, key, value);
+  consensus_cache_entry_t *ent = NULL;
+  if (smartlist_len(tmp))
+    ent = smartlist_get(tmp, 0);
+  smartlist_free(tmp);
+  return ent;
+}
+
+/**
+ * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
+ *
+ * Do not add any entry that has been marked for removal.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_find_all(smartlist_t *out,
+                         consensus_cache_t *cache,
+                         const char *key,
+                         const char *value)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    if (ent->can_remove == 1) {
+      /* We want to delete this; pretend it isn't there. */
+      continue;
+    }
+    if (! key) {
+      smartlist_add(out, ent);
+      continue;
+    }
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (found_val && !strcmp(value, found_val)) {
+      smartlist_add(out, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Given a list of consensus_cache_entry_t, remove all those entries
+ * that do not have <b>key</b>=<b>value</b> in their labels.
+ *
+ * Does not adjust reference counts.
+ */
+void
+consensus_cache_filter_list(smartlist_t *lst,
+                            const char *key,
+                            const char *value)
+{
+  if (BUG(lst == NULL))
+    return; // LCOV_EXCL_LINE
+  if (key == NULL)
+    return;
+  SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
+    const char *found_val = consensus_cache_entry_get_value(ent, key);
+    if (! found_val || strcmp(value, found_val)) {
+      SMARTLIST_DEL_CURRENT(lst, ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * If <b>ent</b> has a label with the given <b>key</b>, return its
+ * value.  Otherwise return NULL.
+ *
+ * The return value is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const char *
+consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                const char *key)
+{
+  const config_line_t *match = config_line_find(ent->labels, key);
+  if (match)
+    return match->value;
+  else
+    return NULL;
+}
+
+/**
+ * Return a pointer to the labels in <b>ent</b>.
+ *
+ * This pointer is only guaranteed to be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+const config_line_t *
+consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
+{
+  return ent->labels;
+}
+
+/**
+ * Increase the reference count of <b>ent</b>.
+ */
+void
+consensus_cache_entry_incref(consensus_cache_entry_t *ent)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+  ++ent->refcnt;
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Release a reference held to <b>ent</b>.
+ *
+ * If it was the last reference, ent will be freed. Therefore, you must not
+ * use <b>ent</b> after calling this function.
+ */
+void
+consensus_cache_entry_decref(consensus_cache_entry_t *ent)
+{
+  if (! ent)
+    return;
+  if (BUG(ent->refcnt <= 0))
+    return; // LCOV_EXCL_LINE
+  if (BUG(ent->magic != CCE_MAGIC))
+    return; // LCOV_EXCL_LINE
+
+  --ent->refcnt;
+
+  if (ent->refcnt == 1 && ent->in_cache) {
+    /* Only the cache has a reference: we don't need to keep the file
+     * mapped */
+    if (ent->map) {
+      if (ent->release_aggressively) {
+        consensus_cache_entry_unmap(ent);
+      } else {
+        ent->unused_since = approx_time();
+      }
+    }
+    return;
+  }
+
+  if (ent->refcnt > 0)
+    return;
+
+  /* Refcount is zero; we can free it. */
+  if (ent->map) {
+    consensus_cache_entry_unmap(ent);
+  }
+  tor_free(ent->fname);
+  config_free_lines(ent->labels);
+  consensus_cache_entry_handles_clear(ent);
+  memwipe(ent, 0, sizeof(consensus_cache_entry_t));
+  tor_free(ent);
+}
+
+/**
+ * Mark <b>ent</b> for deletion from the cache.  Deletion will not occur
+ * until the cache is the only place that holds a reference to <b>ent</b>.
+ */
+void
+consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
+{
+  ent->can_remove = 1;
+}
+
+/**
+ * Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
+ * any longer than we're actually using it.
+ */
+void
+consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
+{
+  ent->release_aggressively = 1;
+}
+
+/**
+ * Try to read the body of <b>ent</b> into memory if it isn't already
+ * loaded.  On success, set *<b>body_out</b> to the body, *<b>sz_out</b>
+ * to its size, and return 0.  On failure return -1.
+ *
+ * The resulting body pointer will only be valid for as long as you
+ * hold a reference to <b>ent</b>.
+ */
+int
+consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                               const uint8_t **body_out,
+                               size_t *sz_out)
+{
+  if (BUG(ent->magic != CCE_MAGIC))
+    return -1; // LCOV_EXCL_LINE
+
+  if (! ent->map) {
+    if (! ent->in_cache)
+      return -1;
+
+    consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
+                              (consensus_cache_entry_t *)ent);
+    if (! ent->map) {
+      return -1;
+    }
+  }
+
+  *body_out = ent->body;
+  *sz_out = ent->bodylen;
+  return 0;
+}
+
+/**
+ * Unmap every mmap'd element of <b>cache</b> that has been unused
+ * since <b>cutoff</b>.
+ */
+void
+consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+      /* Somebody is using this entry right now */
+      continue;
+    }
+    if (ent->unused_since > cutoff) {
+      /* Has been unused only for a little while */
+      continue;
+    }
+    if (ent->map == NULL) {
+      /* Not actually mapped. */
+      continue;
+    }
+    consensus_cache_entry_unmap(ent);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Return the number of currently unused filenames available in this cache.
+ */
+int
+consensus_cache_get_n_filenames_available(consensus_cache_t *cache)
+{
+  tor_assert(cache);
+  int max = cache->max_entries;
+  int used = smartlist_len(storage_dir_list(cache->dir));
+#ifdef MUST_UNMAP_TO_UNLINK
+  if (used > max)
+    return 0;
+#else
+  tor_assert_nonfatal(max >= used);
+#endif
+  return max - used;
+}
+
+/**
+ * Delete every element of <b>cache</b> has been marked with
+ * consensus_cache_entry_mark_for_removal. If <b>force</b> is false,
+ * retain those entries which are in use by something other than the cache.
+ */
+void
+consensus_cache_delete_pending(consensus_cache_t *cache, int force)
+{
+  SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
+    tor_assert_nonfatal(ent->in_cache == cache);
+    int force_ent = force;
+#ifdef MUST_UNMAP_TO_UNLINK
+    /* We cannot delete anything with an active mmap on win32, so no
+     * force-deletion. */
+    if (ent->map) {
+      force_ent = 0;
+    }
+#endif
+    if (! force_ent) {
+      if (ent->refcnt > 1 || BUG(ent->in_cache == NULL)) {
+        /* Somebody is using this entry right now */
+        continue;
+      }
+    }
+    if (ent->can_remove == 0) {
+      /* Don't want to delete this. */
+      continue;
+    }
+    if (BUG(ent->refcnt <= 0)) {
+      continue; // LCOV_EXCL_LINE
+    }
+
+    SMARTLIST_DEL_CURRENT(cache->entries, ent);
+    ent->in_cache = NULL;
+    char *fname = tor_strdup(ent->fname); /* save a copy */
+    consensus_cache_entry_decref(ent);
+    storage_dir_remove_file(cache->dir, fname);
+    tor_free(fname);
+  } SMARTLIST_FOREACH_END(ent);
+}
+
+/**
+ * Internal helper: rescan <b>cache</b> and rebuild its list of entries.
+ */
+static void
+consensus_cache_rescan(consensus_cache_t *cache)
+{
+  if (cache->entries) {
+    consensus_cache_clear(cache);
+  }
+
+  cache->entries = smartlist_new();
+  const smartlist_t *fnames = storage_dir_list(cache->dir);
+  SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
+    tor_mmap_t *map = NULL;
+    config_line_t *labels = NULL;
+    const uint8_t *body;
+    size_t bodylen;
+    map = storage_dir_map_labeled(cache->dir, fname,
+                                  &labels, &body, &bodylen);
+    if (! map) {
+      /* The ERANGE error might come from tor_mmap_file() -- it means the file
+       * was empty. EINVAL might come from ..map_labeled() -- it means the
+       * file was misformatted. In both cases, we should just delete it.
+       */
+      if (errno == ERANGE || errno == EINVAL) {
+        log_warn(LD_FS, "Found %s file %s in consensus cache; removing it.",
+                 errno == ERANGE ? "empty" : "misformatted",
+                 escaped(fname));
+        storage_dir_remove_file(cache->dir, fname);
+      } else {
+        /* Can't load this; continue */
+        log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
+                 escaped(fname), strerror(errno));
+      }
+      continue;
+    }
+    consensus_cache_entry_t *ent =
+      tor_malloc_zero(sizeof(consensus_cache_entry_t));
+    ent->magic = CCE_MAGIC;
+    ent->fname = tor_strdup(fname);
+    ent->labels = labels;
+    ent->refcnt = 1;
+    ent->in_cache = cache;
+    ent->unused_since = TIME_MAX;
+    smartlist_add(cache->entries, ent);
+    tor_munmap_file(map); /* don't actually need to keep this around */
+  } SMARTLIST_FOREACH_END(fname);
+}
+
+/**
+ * Make sure that <b>ent</b> is mapped into RAM.
+ */
+static void
+consensus_cache_entry_map(consensus_cache_t *cache,
+                          consensus_cache_entry_t *ent)
+{
+  if (ent->map)
+    return;
+
+  ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
+                                     NULL, &ent->body, &ent->bodylen);
+  ent->unused_since = TIME_MAX;
+}
+
+/**
+ * Unmap <b>ent</b> from RAM.
+ *
+ * Do not call this if something other than the cache is holding a reference
+ * to <b>ent</b>
+ */
+static void
+consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
+{
+  ent->unused_since = TIME_MAX;
+  if (!ent->map)
+    return;
+
+  tor_munmap_file(ent->map);
+  ent->map = NULL;
+  ent->body = NULL;
+  ent->bodylen = 0;
+  ent->unused_since = TIME_MAX;
+}
+
+HANDLE_IMPL(consensus_cache_entry, consensus_cache_entry_t, )
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Testing only: Return true iff <b>ent</b> is mapped into memory.
+ *
+ * (In normal operation, this information is not exposed.)
+ */
+int
+consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent)
+{
+  if (ent->map) {
+    tor_assert(ent->body);
+    return 1;
+  } else {
+    tor_assert(!ent->body);
+    return 0;
+  }
+}
+#endif
+
diff --git a/src/or/conscache.h b/src/or/conscache.h
new file mode 100644
index 0000000000..a0d74c4e08
--- /dev/null
+++ b/src/or/conscache.h
@@ -0,0 +1,62 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSCACHE_H
+#define TOR_CONSCACHE_H
+
+#include "handles.h"
+
+typedef struct consensus_cache_entry_t consensus_cache_entry_t;
+typedef struct consensus_cache_t consensus_cache_t;
+
+HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, )
+
+consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
+void consensus_cache_free(consensus_cache_t *cache);
+struct sandbox_cfg_elem;
+int consensus_cache_may_overallocate(consensus_cache_t *cache);
+int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
+                                          struct sandbox_cfg_elem **cfg);
+void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
+void consensus_cache_delete_pending(consensus_cache_t *cache,
+                                    int force);
+int consensus_cache_get_n_filenames_available(consensus_cache_t *cache);
+consensus_cache_entry_t *consensus_cache_add(consensus_cache_t *cache,
+                                             const config_line_t *labels,
+                                             const uint8_t *data,
+                                             size_t datalen);
+
+consensus_cache_entry_t *consensus_cache_find_first(
+                                             consensus_cache_t *cache,
+                                             const char *key,
+                                             const char *value);
+
+void consensus_cache_find_all(smartlist_t *out,
+                              consensus_cache_t *cache,
+                              const char *key,
+                              const char *value);
+void consensus_cache_filter_list(smartlist_t *lst,
+                                 const char *key,
+                                 const char *value);
+
+const char *consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
+                                            const char *key);
+const config_line_t *consensus_cache_entry_get_labels(
+                                          const consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
+void consensus_cache_entry_decref(consensus_cache_entry_t *ent);
+
+void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
+void consensus_cache_entry_mark_for_aggressive_release(
+                                            consensus_cache_entry_t *ent);
+int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
+                                   const uint8_t **body_out,
+                                   size_t *sz_out);
+
+#ifdef TOR_UNIT_TESTS
+int consensus_cache_entry_is_mapped(consensus_cache_entry_t *ent);
+#endif
+
+#endif
+
diff --git a/src/or/consdiff.c b/src/or/consdiff.c
new file mode 100644
index 0000000000..deaf465fe7
--- /dev/null
+++ b/src/or/consdiff.c
@@ -0,0 +1,1415 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiff.c
+ * \brief Consensus diff implementation, including both the generation and the
+ * application of diffs in a minimal ed format.
+ *
+ * The consensus diff application is done in consdiff_apply_diff, which relies
+ * on apply_ed_diff for the main ed diff part and on some digest helper
+ * functions to check the digest hashes found in the consensus diff header.
+ *
+ * The consensus diff generation is more complex. consdiff_gen_diff generates
+ * it, relying on gen_ed_diff to generate the ed diff and some digest helper
+ * functions to generate the digest hashes.
+ *
+ * gen_ed_diff is the tricky bit. In it simplest form, it will take quadratic
+ * time and linear space to generate an ed diff given two smartlists. As shown
+ * in its comment section, calling calc_changes on the entire two consensuses
+ * will calculate what is to be added and what is to be deleted in the diff.
+ * Its comment section briefly explains how it works.
+ *
+ * In our case specific to consensuses, we take advantage of the fact that
+ * consensuses list routers sorted by their identities. We use that
+ * information to avoid running calc_changes on the whole smartlists.
+ * gen_ed_diff will navigate through the two consensuses identity by identity
+ * and will send small couples of slices to calc_changes, keeping the running
+ * time near-linear. This is explained in more detail in the gen_ed_diff
+ * comments.
+ *
+ * The allocation strategy tries to save time and memory by avoiding needless
+ * copies.  Instead of actually splitting the inputs into separate strings, we
+ * allocate cdline_t objects, each of which represents a line in the original
+ * object or in the output.  We use memarea_t allocators to manage the
+ * temporary memory we use when generating or applying diffs.
+ **/
+
+#define CONSDIFF_PRIVATE
+
+#include "or.h"
+#include "consdiff.h"
+#include "memarea.h"
+#include "routerparse.h"
+
+static const char* ns_diff_version = "network-status-diff-version 1";
+static const char* hash_token = "hash";
+
+static char *consensus_join_lines(const smartlist_t *inp);
+
+/** Return true iff a and b have the same contents. */
+STATIC int
+lines_eq(const cdline_t *a, const cdline_t *b)
+{
+  return a->len == b->len && fast_memeq(a->s, b->s, a->len);
+}
+
+/** Return true iff a has the same contents as the nul-terminated string b. */
+STATIC int
+line_str_eq(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  cdline_t bline = { b, (uint32_t)len };
+  return lines_eq(a, &bline);
+}
+
+/** Return true iff a begins with the same contents as the nul-terminated
+ * string b. */
+static int
+line_starts_with_str(const cdline_t *a, const char *b)
+{
+  const size_t len = strlen(b);
+  tor_assert(len <= UINT32_MAX);
+  return a->len >= len && fast_memeq(a->s, b, len);
+}
+
+/** Return a new cdline_t holding as its contents the nul-terminated
+ * string s. Use the provided memory area for storage. */
+static cdline_t *
+cdline_linecpy(memarea_t *area, const char *s)
+{
+  size_t len = strlen(s);
+  const char *ss = memarea_memdup(area, s, len);
+  cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+  line->s = ss;
+  line->len = (uint32_t)len;
+  return line;
+}
+
+/** Add a cdline_t to <b>lst</b> holding as its contents the nul-terminated
+ * string s.  Use the provided memory area for storage. */
+STATIC void
+smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
+{
+  smartlist_add(lst, cdline_linecpy(area, s));
+}
+
+/** Compute the digest of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest,(const char *cons,
+                          consensus_digest_t *digest_out))
+{
+  int r = crypto_digest256((char*)digest_out->sha3_256,
+                           cons, strlen(cons), DIGEST_SHA3_256);
+  return r;
+}
+
+/** Compute the digest-as-signed of <b>cons</b>, and store the result in
+ * <b>digest_out</b>. Return 0 on success, -1 on failure. */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_compute_digest_as_signed,(const char *cons,
+                                    consensus_digest_t *digest_out))
+{
+  return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
+                                                    cons);
+}
+
+/** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
+/* This is a separate, mockable function so that we can override it when
+ * fuzzing. */
+MOCK_IMPL(STATIC int,
+consensus_digest_eq,(const uint8_t *d1,
+                     const uint8_t *d2))
+{
+  return fast_memeq(d1, d2, DIGEST256_LEN);
+}
+
+/** Create (allocate) a new slice from a smartlist. Assumes that the start
+ * and the end indexes are within the bounds of the initial smartlist. The end
+ * element is not part of the resulting slice. If end is -1, the slice is to
+ * reach the end of the smartlist.
+ */
+STATIC smartlist_slice_t *
+smartlist_slice(const smartlist_t *list, int start, int end)
+{
+  int list_len = smartlist_len(list);
+  tor_assert(start >= 0);
+  tor_assert(start <= list_len);
+  if (end == -1) {
+    end = list_len;
+  }
+  tor_assert(start <= end);
+
+  smartlist_slice_t *slice = tor_malloc(sizeof(smartlist_slice_t));
+  slice->list = list;
+  slice->offset = start;
+  slice->len = end - start;
+  return slice;
+}
+
+/** Helper: Compute the longest common subsequence lengths for the two slices.
+ * Used as part of the diff generation to find the column at which to split
+ * slice2 while still having the optimal solution.
+ * If direction is -1, the navigation is reversed. Otherwise it must be 1.
+ * The length of the resulting integer array is that of the second slice plus
+ * one.
+ */
+STATIC int *
+lcs_lengths(const smartlist_slice_t *slice1, const smartlist_slice_t *slice2,
+            int direction)
+{
+  size_t a_size = sizeof(int) * (slice2->len+1);
+
+  /* Resulting lcs lengths. */
+  int *result = tor_malloc_zero(a_size);
+  /* Copy of the lcs lengths from the last iteration. */
+  int *prev = tor_malloc(a_size);
+
+  tor_assert(direction == 1 || direction == -1);
+
+  int si = slice1->offset;
+  if (direction == -1) {
+    si += (slice1->len-1);
+  }
+
+  for (int i = 0; i < slice1->len; ++i, si+=direction) {
+
+    const cdline_t *line1 = smartlist_get(slice1->list, si);
+    /* Store the last results. */
+    memcpy(prev, result, a_size);
+
+    int sj = slice2->offset;
+    if (direction == -1) {
+      sj += (slice2->len-1);
+    }
+
+    for (int j = 0; j < slice2->len; ++j, sj+=direction) {
+
+      const cdline_t *line2 = smartlist_get(slice2->list, sj);
+      if (lines_eq(line1, line2)) {
+        /* If the lines are equal, the lcs is one line longer. */
+        result[j + 1] = prev[j] + 1;
+      } else {
+        /* If not, see what lcs parent path is longer. */
+        result[j + 1] = MAX(result[j], prev[j + 1]);
+      }
+    }
+  }
+  tor_free(prev);
+  return result;
+}
+
+/** Helper: Trim any number of lines that are equally at the start or the end
+ * of both slices.
+ */
+STATIC void
+trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2)
+{
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, slice1->offset);
+    const cdline_t *line2 = smartlist_get(slice2->list, slice2->offset);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    slice1->offset++; slice1->len--;
+    slice2->offset++; slice2->len--;
+  }
+
+  int i1 = (slice1->offset+slice1->len)-1;
+  int i2 = (slice2->offset+slice2->len)-1;
+
+  while (slice1->len>0 && slice2->len>0) {
+    const cdline_t *line1 = smartlist_get(slice1->list, i1);
+    const cdline_t *line2 = smartlist_get(slice2->list, i2);
+    if (!lines_eq(line1, line2)) {
+      break;
+    }
+    i1--;
+    slice1->len--;
+    i2--;
+    slice2->len--;
+  }
+}
+
+/** Like smartlist_string_pos, but uses a cdline_t, and is restricted to the
+ * bounds of the slice.
+ */
+STATIC int
+smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                           const cdline_t *string)
+{
+  int end = slice->offset + slice->len;
+  for (int i = slice->offset; i < end; ++i) {
+    const cdline_t *el = smartlist_get(slice->list, i);
+    if (lines_eq(el, string)) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/** Helper: Set all the appropriate changed booleans to true. The first slice
+ * must be of length 0 or 1. All the lines of slice1 and slice2 which are not
+ * present in the other slice will be set to changed in their bool array.
+ * The two changed bool arrays are passed in the same order as the slices.
+ */
+STATIC void
+set_changed(bitarray_t *changed1, bitarray_t *changed2,
+            const smartlist_slice_t *slice1, const smartlist_slice_t *slice2)
+{
+  int toskip = -1;
+  tor_assert(slice1->len == 0 || slice1->len == 1);
+
+  if (slice1->len == 1) {
+    const cdline_t *line_common = smartlist_get(slice1->list, slice1->offset);
+    toskip = smartlist_slice_string_pos(slice2, line_common);
+    if (toskip == -1) {
+      bitarray_set(changed1, slice1->offset);
+    }
+  }
+  int end = slice2->offset + slice2->len;
+  for (int i = slice2->offset; i < end; ++i) {
+    if (i != toskip) {
+      bitarray_set(changed2, i);
+    }
+  }
+}
+
+/*
+ * Helper: Given that slice1 has been split by half into top and bot, we want
+ * to fetch the column at which to split slice2 so that we are still on track
+ * to the optimal diff solution, i.e. the shortest one. We use lcs_lengths
+ * since the shortest diff is just another way to say the longest common
+ * subsequence.
+ */
+static int
+optimal_column_to_split(const smartlist_slice_t *top,
+                        const smartlist_slice_t *bot,
+                        const smartlist_slice_t *slice2)
+{
+  int *lens_top = lcs_lengths(top, slice2, 1);
+  int *lens_bot = lcs_lengths(bot, slice2, -1);
+  int column=0, max_sum=-1;
+
+  for (int i = 0; i < slice2->len+1; ++i) {
+    int sum = lens_top[i] + lens_bot[slice2->len-i];
+    if (sum > max_sum) {
+      column = i;
+      max_sum = sum;
+    }
+  }
+  tor_free(lens_top);
+  tor_free(lens_bot);
+
+  return column;
+}
+
+/**
+ * Helper: Figure out what elements are new or gone on the second smartlist
+ * relative to the first smartlist, and store the booleans in the bitarrays.
+ * True on the first bitarray means the element is gone, true on the second
+ * bitarray means it's new.
+ *
+ * In its base case, either of the smartlists is of length <= 1 and we can
+ * quickly see what elements are new or are gone. In the other case, we will
+ * split one smartlist by half and we'll use optimal_column_to_split to find
+ * the optimal column at which to split the second smartlist so that we are
+ * finding the smallest diff possible.
+ */
+STATIC void
+calc_changes(smartlist_slice_t *slice1,
+             smartlist_slice_t *slice2,
+             bitarray_t *changed1, bitarray_t *changed2)
+{
+  trim_slices(slice1, slice2);
+
+  if (slice1->len <= 1) {
+    set_changed(changed1, changed2, slice1, slice2);
+
+  } else if (slice2->len <= 1) {
+    set_changed(changed2, changed1, slice2, slice1);
+
+  /* Keep on splitting the slices in two. */
+  } else {
+    smartlist_slice_t *top, *bot, *left, *right;
+
+    /* Split the first slice in half. */
+    int mid = slice1->len/2;
+    top = smartlist_slice(slice1->list, slice1->offset, slice1->offset+mid);
+    bot = smartlist_slice(slice1->list, slice1->offset+mid,
+        slice1->offset+slice1->len);
+
+    /* Split the second slice by the optimal column. */
+    int mid2 = optimal_column_to_split(top, bot, slice2);
+    left = smartlist_slice(slice2->list, slice2->offset, slice2->offset+mid2);
+    right = smartlist_slice(slice2->list, slice2->offset+mid2,
+        slice2->offset+slice2->len);
+
+    calc_changes(top, left, changed1, changed2);
+    calc_changes(bot, right, changed1, changed2);
+    tor_free(top);
+    tor_free(bot);
+    tor_free(left);
+    tor_free(right);
+  }
+}
+
+/* This table is from crypto.c. The SP and PAD defines are different. */
+#define NOT_VALID_BASE64 255
+#define X NOT_VALID_BASE64
+#define SP NOT_VALID_BASE64
+#define PAD NOT_VALID_BASE64
+static const uint8_t base64_compare_table[256] = {
+  X, X, X, X, X, X, X, X, X, SP, SP, SP, X, SP, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  SP, X, X, X, X, X, X, X, X, X, X, 62, X, X, X, 63,
+  52, 53, 54, 55, 56, 57, 58, 59, 60, 61, X, X, X, PAD, X, X,
+  X, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+  15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, X, X, X, X, X,
+  X, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+  41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+  X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,
+};
+
+/** Helper: Get the identity hash from a router line, assuming that the line
+ * at least appears to be a router line and thus starts with "r ".
+ *
+ * If an identity hash is found, store it (without decoding it) in
+ * <b>hash_out</b>, and return 0.  On failure, return -1.
+ */
+STATIC int
+get_id_hash(const cdline_t *line, cdline_t *hash_out)
+{
+  if (line->len < 2)
+    return -1;
+
+  /* Skip the router name. */
+  const char *hash = memchr(line->s + 2, ' ', line->len - 2);
+  if (!hash) {
+    return -1;
+  }
+
+  hash++;
+  const char *hash_end = hash;
+  /* Stop when the first non-base64 character is found. Use unsigned chars to
+   * avoid negative indexes causing crashes.
+   */
+  while (base64_compare_table[*((unsigned char*)hash_end)]
+           != NOT_VALID_BASE64 &&
+         hash_end < line->s + line->len) {
+    hash_end++;
+  }
+
+  /* Empty hash. */
+  if (hash_end == hash) {
+    return -1;
+  }
+
+  hash_out->s = hash;
+  /* Always true because lines are limited to this length */
+  tor_assert(hash_end >= hash);
+  tor_assert((size_t)(hash_end - hash) <= UINT32_MAX);
+  hash_out->len = (uint32_t)(hash_end - hash);
+
+  return 0;
+}
+
+/** Helper: Check that a line is a valid router entry. We must at least be
+ * able to fetch a proper identity hash from it for it to be valid.
+ */
+STATIC int
+is_valid_router_entry(const cdline_t *line)
+{
+  if (line->len < 2 || fast_memneq(line->s, "r ", 2))
+    return 0;
+  cdline_t tmp;
+  return (get_id_hash(line, &tmp) == 0);
+}
+
+/** Helper: Find the next router line starting at the current position.
+ * Assumes that cur is lower than the length of the smartlist, i.e. it is a
+ * line within the bounds of the consensus. The only exception is when we
+ * don't want to skip the first line, in which case cur will be -1.
+ */
+STATIC int
+next_router(const smartlist_t *cons, int cur)
+{
+  int len = smartlist_len(cons);
+  tor_assert(cur >= -1 && cur < len);
+
+  if (++cur >= len) {
+    return len;
+  }
+
+  const cdline_t *line = smartlist_get(cons, cur);
+  while (!is_valid_router_entry(line)) {
+    if (++cur >= len) {
+      return len;
+    }
+    line = smartlist_get(cons, cur);
+  }
+  return cur;
+}
+
+/** Helper: compare two base64-encoded identity hashes, which may be of
+ * different lengths. Comparison ends when the first non-base64 char is found.
+ */
+STATIC int
+base64cmp(const cdline_t *hash1, const cdline_t *hash2)
+{
+  /* NULL is always lower, useful for last_hash which starts at NULL. */
+  if (!hash1->s && !hash2->s) {
+    return 0;
+  }
+  if (!hash1->s) {
+    return -1;
+  }
+  if (!hash2->s) {
+    return 1;
+  }
+
+  /* Don't index with a char; char may be signed. */
+  const unsigned char *a = (unsigned char*)hash1->s;
+  const unsigned char *b = (unsigned char*)hash2->s;
+  const unsigned char *a_end = a + hash1->len;
+  const unsigned char *b_end = b + hash2->len;
+  while (1) {
+    uint8_t av = base64_compare_table[*a];
+    uint8_t bv = base64_compare_table[*b];
+    if (av == NOT_VALID_BASE64) {
+      if (bv == NOT_VALID_BASE64) {
+        /* Both ended with exactly the same characters. */
+        return 0;
+      } else {
+        /* hash2 goes on longer than hash1 and thus hash1 is lower. */
+        return -1;
+      }
+    } else if (bv == NOT_VALID_BASE64) {
+      /* hash1 goes on longer than hash2 and thus hash1 is greater. */
+      return 1;
+    } else if (av < bv) {
+      /* The first difference shows that hash1 is lower. */
+      return -1;
+    } else if (av > bv) {
+      /* The first difference shows that hash1 is greater. */
+      return 1;
+    } else {
+      a++;
+      b++;
+      if (a == a_end) {
+        if (b == b_end) {
+          return 0;
+        } else {
+          return -1;
+        }
+      } else if (b == b_end) {
+        return 1;
+      }
+    }
+  }
+}
+
+/** Structure used to remember the previous and current identity hash of
+ * the "r " lines in a consensus, to enforce well-ordering. */
+typedef struct router_id_iterator_t {
+  cdline_t last_hash;
+  cdline_t hash;
+} router_id_iterator_t;
+
+/**
+ * Initializer for a router_id_iterator_t.
+ */
+#define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } }
+
+/** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance
+ * the index to the next router line ("r ...") in the consensus, or to
+ * an index one after the end of the list if there is no such line.
+ *
+ * Use <b>iter</b> to record the hash of the found router line, if any,
+ * and to enforce ordering on the hashes.  If the hashes are mis-ordered,
+ * return -1.  Else, return 0.
+ **/
+static int
+find_next_router_line(const smartlist_t *cons,
+                      const char *consname,
+                      int *idxp,
+                      router_id_iterator_t *iter)
+{
+  *idxp = next_router(cons, *idxp);
+  if (*idxp < smartlist_len(cons)) {
+    memcpy(&iter->last_hash, &iter->hash, sizeof(cdline_t));
+    if (get_id_hash(smartlist_get(cons, *idxp), &iter->hash) < 0 ||
+        base64cmp(&iter->hash, &iter->last_hash) <= 0) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+               "the %s consensus doesn't have its router entries sorted "
+               "properly.", consname);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Line-prefix indicating the beginning of the signatures section that we
+ * intend to delete. */
+#define START_OF_SIGNATURES_SECTION "directory-signature "
+
+/** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t)
+ * to remove the signatures from it.  If the footer is removed, return a
+ * cdline_t containing a delete command to delete the footer, allocated in
+ * <b>area</>.  If no footer is removed, return NULL.
+ *
+ * We remove the signatures here because they are not themselves signed, and
+ * as such there might be different encodings for them.
+ */
+static cdline_t *
+preprocess_consensus(memarea_t *area,
+                     smartlist_t *cons)
+{
+  int idx;
+  int dirsig_idx = -1;
+  for (idx = 0; idx < smartlist_len(cons); ++idx) {
+    cdline_t *line = smartlist_get(cons, idx);
+    if (line_starts_with_str(line, START_OF_SIGNATURES_SECTION)) {
+      dirsig_idx = idx;
+      break;
+    }
+  }
+  if (dirsig_idx >= 0) {
+    char buf[64];
+    while (smartlist_len(cons) > dirsig_idx)
+      smartlist_del(cons, dirsig_idx);
+    tor_snprintf(buf, sizeof(buf), "%d,$d", dirsig_idx+1);
+    return cdline_linecpy(area, buf);
+  } else {
+    return NULL;
+  }
+}
+
+/** Generate an ed diff as a smartlist from two consensuses, also given as
+ * smartlists. Will return NULL if the diff could not be generated, which can
+ * happen if any lines the script had to add matched "." or if the routers
+ * were not properly ordered.
+ *
+ * All cdline_t objects in the resulting object are either references to lines
+ * in one of the inputs, or are newly allocated lines in the provided memarea.
+ *
+ * This implementation is consensus-specific. To generate an ed diff for any
+ * given input in quadratic time, you can replace all the code until the
+ * navigation in reverse order with the following:
+ *
+ *   int len1 = smartlist_len(cons1);
+ *   int len2 = smartlist_len(cons2);
+ *   bitarray_t *changed1 = bitarray_init_zero(len1);
+ *   bitarray_t *changed2 = bitarray_init_zero(len2);
+ *   cons1_sl = smartlist_slice(cons1, 0, -1);
+ *   cons2_sl = smartlist_slice(cons2, 0, -1);
+ *   calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+ */
+STATIC smartlist_t *
+gen_ed_diff(const smartlist_t *cons1_orig, const smartlist_t *cons2,
+            memarea_t *area)
+{
+  smartlist_t *cons1 = smartlist_new();
+  smartlist_add_all(cons1, cons1_orig);
+  cdline_t *remove_trailer = preprocess_consensus(area, cons1);
+
+  int len1 = smartlist_len(cons1);
+  int len2 = smartlist_len(cons2);
+  smartlist_t *result = smartlist_new();
+
+  if (remove_trailer) {
+    /* There's a delete-the-trailer line at the end, so add it here. */
+    smartlist_add(result, remove_trailer);
+  }
+
+  /* Initialize the changed bitarrays to zero, so that calc_changes only needs
+   * to set the ones that matter and leave the rest untouched.
+   */
+  bitarray_t *changed1 = bitarray_init_zero(len1);
+  bitarray_t *changed2 = bitarray_init_zero(len2);
+  int i1=-1, i2=-1;
+  int start1=0, start2=0;
+
+  /* To check that hashes are ordered properly */
+  router_id_iterator_t iter1 = ROUTER_ID_ITERATOR_INIT;
+  router_id_iterator_t iter2 = ROUTER_ID_ITERATOR_INIT;
+
+  /* i1 and i2 are initialized at the first line of each consensus. They never
+   * reach past len1 and len2 respectively, since next_router doesn't let that
+   * happen. i1 and i2 are advanced by at least one line at each iteration as
+   * long as they have not yet reached len1 and len2, so the loop is
+   * guaranteed to end, and each pair of (i1,i2) will be inspected at most
+   * once.
+   */
+  while (i1 < len1 || i2 < len2) {
+
+    /* Advance each of the two navigation positions by one router entry if not
+     * yet at the end.
+     */
+    if (i1 < len1) {
+      if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    if (i2 < len2) {
+      if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+        goto error_cleanup;
+      }
+    }
+
+    /* If we have reached the end of both consensuses, there is no need to
+     * compare hashes anymore, since this is the last iteration.
+     */
+    if (i1 < len1 || i2 < len2) {
+
+      /* Keep on advancing the lower (by identity hash sorting) position until
+       * we have two matching positions. The only other possible outcome is
+       * that a lower position reaches the end of the consensus before it can
+       * reach a hash that is no longer the lower one. Since there will always
+       * be a lower hash for as long as the loop runs, one of the two indexes
+       * will always be incremented, thus assuring that the loop must end
+       * after a finite number of iterations. If that cannot be because said
+       * consensus has already reached the end, both are extended to their
+       * respecting ends since we are done.
+       */
+      int cmp = base64cmp(&iter1.hash, &iter2.hash);
+      while (cmp != 0) {
+        if (i1 < len1 && cmp < 0) {
+          if (find_next_router_line(cons1, "base", &i1, &iter1) < 0) {
+            goto error_cleanup;
+          }
+          if (i1 == len1) {
+            /* We finished the first consensus, so grab all the remaining
+             * lines of the second consensus and finish up.
+             */
+            i2 = len2;
+            break;
+          }
+        } else if (i2 < len2 && cmp > 0) {
+          if (find_next_router_line(cons2, "target", &i2, &iter2) < 0) {
+            goto error_cleanup;
+          }
+          if (i2 == len2) {
+            /* We finished the second consensus, so grab all the remaining
+             * lines of the first consensus and finish up.
+             */
+            i1 = len1;
+            break;
+          }
+        } else {
+          i1 = len1;
+          i2 = len2;
+          break;
+        }
+        cmp = base64cmp(&iter1.hash, &iter2.hash);
+      }
+    }
+
+    /* Make slices out of these chunks (up to the common router entry) and
+     * calculate the changes for them.
+     * Error if any of the two slices are longer than 10K lines. That should
+     * never happen with any pair of real consensuses. Feeding more than 10K
+     * lines to calc_changes would be very slow anyway.
+     */
+#define MAX_LINE_COUNT (10000)
+    if (i1-start1 > MAX_LINE_COUNT || i2-start2 > MAX_LINE_COUNT) {
+      log_warn(LD_CONSDIFF, "Refusing to generate consensus diff because "
+          "we found too few common router ids.");
+      goto error_cleanup;
+    }
+
+    smartlist_slice_t *cons1_sl = smartlist_slice(cons1, start1, i1);
+    smartlist_slice_t *cons2_sl = smartlist_slice(cons2, start2, i2);
+    calc_changes(cons1_sl, cons2_sl, changed1, changed2);
+    tor_free(cons1_sl);
+    tor_free(cons2_sl);
+    start1 = i1, start2 = i2;
+  }
+
+  /* Navigate the changes in reverse order and generate one ed command for
+   * each chunk of changes.
+   */
+  i1=len1-1, i2=len2-1;
+  char buf[128];
+  while (i1 >= 0 || i2 >= 0) {
+
+    int start1x, start2x, end1, end2, added, deleted;
+
+    /* We are at a point were no changed bools are true, so just keep going. */
+    if (!(i1 >= 0 && bitarray_is_set(changed1, i1)) &&
+        !(i2 >= 0 && bitarray_is_set(changed2, i2))) {
+      if (i1 >= 0) {
+        i1--;
+      }
+      if (i2 >= 0) {
+        i2--;
+      }
+      continue;
+    }
+
+    end1 = i1, end2 = i2;
+
+    /* Grab all contiguous changed lines */
+    while (i1 >= 0 && bitarray_is_set(changed1, i1)) {
+      i1--;
+    }
+    while (i2 >= 0 && bitarray_is_set(changed2, i2)) {
+      i2--;
+    }
+
+    start1x = i1+1, start2x = i2+1;
+    added = end2-i2, deleted = end1-i1;
+
+    if (added == 0) {
+      if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%id", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%id", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+    } else {
+      int i;
+      if (deleted == 0) {
+        tor_snprintf(buf, sizeof(buf), "%ia", start1x);
+        smartlist_add_linecpy(result, area, buf);
+      } else if (deleted == 1) {
+        tor_snprintf(buf, sizeof(buf), "%ic", start1x+1);
+        smartlist_add_linecpy(result, area, buf);
+      } else {
+        tor_snprintf(buf, sizeof(buf), "%i,%ic", start1x+1, start1x+deleted);
+        smartlist_add_linecpy(result, area, buf);
+      }
+
+      for (i = start2x; i <= end2; ++i) {
+        cdline_t *line = smartlist_get(cons2, i);
+        if (line_str_eq(line, ".")) {
+          log_warn(LD_CONSDIFF, "Cannot generate consensus diff because "
+              "one of the lines to be added is \".\".");
+          goto error_cleanup;
+        }
+        smartlist_add(result, line);
+      }
+      smartlist_add_linecpy(result, area, ".");
+    }
+  }
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  return result;
+
+ error_cleanup:
+
+  smartlist_free(cons1);
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+
+  smartlist_free(result);
+
+  return NULL;
+}
+
+/* Helper: Read a base-10 number between 0 and INT32_MAX from <b>s</b> and
+ * store it in <b>num_out</b>.  Advance <b>s</b> to the characer immediately
+ * after the number.  Return 0 on success, -1 on failure. */
+static int
+get_linenum(const char **s, int *num_out)
+{
+  int ok;
+  char *next;
+  if (!TOR_ISDIGIT(**s)) {
+    return -1;
+  }
+  *num_out = (int) tor_parse_long(*s, 10, 0, INT32_MAX, &ok, &next);
+  if (ok && next) {
+    *s = next;
+    return 0;
+  } else {
+    return -1;
+  }
+}
+
+/** Apply the ed diff, starting at <b>diff_starting_line</b>, to the consensus
+ * and return a new consensus, also as a line-based smartlist. Will return
+ * NULL if the ed diff is not properly formatted.
+ *
+ * All cdline_t objects in the resulting object are references to lines
+ * in one of the inputs; nothing is copied.
+ */
+STATIC smartlist_t *
+apply_ed_diff(const smartlist_t *cons1, const smartlist_t *diff,
+              int diff_starting_line)
+{
+  int diff_len = smartlist_len(diff);
+  int j = smartlist_len(cons1);
+  smartlist_t *cons2 = smartlist_new();
+
+  for (int i=diff_starting_line; i<diff_len; ++i) {
+    const cdline_t *diff_cdline = smartlist_get(diff, i);
+    char diff_line[128];
+
+    if (diff_cdline->len > sizeof(diff_line) - 1) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was far too long");
+      goto error_cleanup;
+    }
+    /* Copy the line to make it nul-terminated. */
+    memcpy(diff_line, diff_cdline->s, diff_cdline->len);
+    diff_line[diff_cdline->len] = 0;
+    const char *ptr = diff_line;
+    int start = 0, end = 0;
+    int had_range = 0;
+    int end_was_eof = 0;
+    if (get_linenum(&ptr, &start) < 0) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "an ed command was missing a line number.");
+      goto error_cleanup;
+    }
+    if (*ptr == ',') {
+      /* Two-item range */
+      had_range = 1;
+      ++ptr;
+      if (*ptr == '$') {
+        end_was_eof = 1;
+        end = smartlist_len(cons1);
+        ++ptr;
+      } else if (get_linenum(&ptr, &end) < 0) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an ed command was missing a range end line number.");
+        goto error_cleanup;
+      }
+      /* Incoherent range. */
+      if (end <= start) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+                 "an invalid range was found in an ed command.");
+        goto error_cleanup;
+      }
+    } else {
+      /* We'll take <n1> as <n1>,<n1> for simplicity. */
+      end = start;
+    }
+
+    if (end > j) {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "its commands are not properly sorted in reverse order.");
+      goto error_cleanup;
+    }
+
+    if (*ptr == '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "a line with no ed command was found");
+      goto error_cleanup;
+    }
+
+    if (*(ptr+1) != '\0') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "an ed command longer than one char was found.");
+      goto error_cleanup;
+    }
+
+    char action = *ptr;
+
+    switch (action) {
+      case 'a':
+      case 'c':
+      case 'd':
+        break;
+      default:
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "an unrecognised ed command was found.");
+        goto error_cleanup;
+    }
+
+    /** $ is not allowed with non-d actions. */
+    if (end_was_eof && action != 'd') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+               "it wanted to use $ with a command other than delete");
+      goto error_cleanup;
+    }
+
+    /* 'a' commands are not allowed to have ranges. */
+    if (had_range && action == 'a') {
+      log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+          "it wanted to add lines after a range.");
+      goto error_cleanup;
+    }
+
+    /* Add unchanged lines. */
+    for (; j && j > end; --j) {
+      cdline_t *cons_line = smartlist_get(cons1, j-1);
+      smartlist_add(cons2, cons_line);
+    }
+
+    /* Ignore removed lines. */
+    if (action == 'c' || action == 'd') {
+      while (--j >= start) {
+        /* Skip line */
+      }
+    }
+
+    /* Add new lines in reverse order, since it will all be reversed at the
+     * end.
+     */
+    if (action == 'a' || action == 'c') {
+      int added_end = i;
+
+      i++; /* Skip the line with the range and command. */
+      while (i < diff_len) {
+        if (line_str_eq(smartlist_get(diff, i), ".")) {
+          break;
+        }
+        if (++i == diff_len) {
+          log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+              "it has lines to be inserted that don't end with a \".\".");
+          goto error_cleanup;
+        }
+      }
+
+      int added_i = i-1;
+
+      /* It would make no sense to add zero new lines. */
+      if (added_i == added_end) {
+        log_warn(LD_CONSDIFF, "Could not apply consensus diff because "
+            "it has an ed command that tries to insert zero lines.");
+        goto error_cleanup;
+      }
+
+      while (added_i > added_end) {
+        cdline_t *added_line = smartlist_get(diff, added_i--);
+        smartlist_add(cons2, added_line);
+      }
+    }
+  }
+
+  /* Add remaining unchanged lines. */
+  for (; j > 0; --j) {
+    cdline_t *cons_line = smartlist_get(cons1, j-1);
+    smartlist_add(cons2, cons_line);
+  }
+
+  /* Reverse the whole thing since we did it from the end. */
+  smartlist_reverse(cons2);
+  return cons2;
+
+ error_cleanup:
+
+  smartlist_free(cons2);
+
+  return NULL;
+}
+
+/** Generate a consensus diff as a smartlist from two given consensuses, also
+ * as smartlists. Will return NULL if the consensus diff could not be
+ * generated. Neither of the two consensuses are modified in any way, so it's
+ * up to the caller to free their resources.
+ */
+smartlist_t *
+consdiff_gen_diff(const smartlist_t *cons1,
+                  const smartlist_t *cons2,
+                  const consensus_digest_t *digests1,
+                  const consensus_digest_t *digests2,
+                  memarea_t *area)
+{
+  smartlist_t *ed_diff = gen_ed_diff(cons1, cons2, area);
+  /* ed diff could not be generated - reason already logged by gen_ed_diff. */
+  if (!ed_diff) {
+    goto error_cleanup;
+  }
+
+  /* See that the script actually produces what we want. */
+  smartlist_t *ed_cons2 = apply_ed_diff(cons1, ed_diff, 0);
+  if (!ed_cons2) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff could not be tested to successfully generate "
+        "the target consensus.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  int cons2_eq = 1;
+  if (smartlist_len(cons2) == smartlist_len(ed_cons2)) {
+    SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) {
+      const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx);
+      if (! lines_eq(line1, line2) ) {
+        cons2_eq = 0;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(line1);
+  } else {
+    cons2_eq = 0;
+  }
+  smartlist_free(ed_cons2);
+  if (!cons2_eq) {
+    /* LCOV_EXCL_START -- impossible if diff generation is correct. */
+    log_warn(LD_BUG|LD_CONSDIFF, "Refusing to generate consensus diff because "
+        "the generated ed diff did not generate the target consensus "
+        "successfully when tested.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  char cons1_hash_hex[HEX_DIGEST256_LEN+1];
+  char cons2_hash_hex[HEX_DIGEST256_LEN+1];
+  base16_encode(cons1_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests1->sha3_256, DIGEST256_LEN);
+  base16_encode(cons2_hash_hex, HEX_DIGEST256_LEN+1,
+                (const char*)digests2->sha3_256, DIGEST256_LEN);
+
+  /* Create the resulting consensus diff. */
+  char buf[160];
+  smartlist_t *result = smartlist_new();
+  tor_snprintf(buf, sizeof(buf), "%s", ns_diff_version);
+  smartlist_add_linecpy(result, area, buf);
+  tor_snprintf(buf, sizeof(buf), "%s %s %s", hash_token,
+      cons1_hash_hex, cons2_hash_hex);
+  smartlist_add_linecpy(result, area, buf);
+  smartlist_add_all(result, ed_diff);
+  smartlist_free(ed_diff);
+  return result;
+
+ error_cleanup:
+
+  if (ed_diff) {
+    /* LCOV_EXCL_START -- ed_diff is NULL except in unreachable cases above */
+    smartlist_free(ed_diff);
+    /* LCOV_EXCL_STOP */
+  }
+
+  return NULL;
+}
+
+/** Fetch the digest of the base consensus in the consensus diff, encoded in
+ * base16 as found in the diff itself. digest1_out and digest2_out must be of
+ * length DIGEST256_LEN or larger if not NULL.
+ */
+int
+consdiff_get_digests(const smartlist_t *diff,
+                     char *digest1_out,
+                     char *digest2_out)
+{
+  smartlist_t *hash_words = NULL;
+  const cdline_t *format;
+  char cons1_hash[DIGEST256_LEN], cons2_hash[DIGEST256_LEN];
+  char *cons1_hash_hex, *cons2_hash_hex;
+  if (smartlist_len(diff) < 2) {
+    log_info(LD_CONSDIFF, "The provided consensus diff is too short.");
+    goto error_cleanup;
+  }
+
+  /* Check that it's the format and version we know. */
+  format = smartlist_get(diff, 0);
+  if (!line_str_eq(format, ns_diff_version)) {
+    log_warn(LD_CONSDIFF, "The provided consensus diff format is not known.");
+    goto error_cleanup;
+  }
+
+  /* Grab the base16 digests. */
+  hash_words = smartlist_new();
+  {
+    const cdline_t *line2 = smartlist_get(diff, 1);
+    char *h = tor_memdup_nulterm(line2->s, line2->len);
+    smartlist_split_string(hash_words, h, " ", 0, 0);
+    tor_free(h);
+  }
+
+  /* There have to be three words, the first of which must be hash_token. */
+  if (smartlist_len(hash_words) != 3 ||
+      strcmp(smartlist_get(hash_words, 0), hash_token)) {
+    log_info(LD_CONSDIFF, "The provided consensus diff does not include "
+        "the necessary digests.");
+    goto error_cleanup;
+  }
+
+  /* Expected hashes as found in the consensus diff header. They must be of
+   * length HEX_DIGEST256_LEN, normally 64 hexadecimal characters.
+   * If any of the decodings fail, error to make sure that the hashes are
+   * proper base16-encoded digests.
+   */
+  cons1_hash_hex = smartlist_get(hash_words, 1);
+  cons2_hash_hex = smartlist_get(hash_words, 2);
+  if (strlen(cons1_hash_hex) != HEX_DIGEST256_LEN ||
+      strlen(cons2_hash_hex) != HEX_DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "base16-encoded digests of incorrect size.");
+    goto error_cleanup;
+  }
+
+  if (base16_decode(cons1_hash, DIGEST256_LEN,
+          cons1_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN ||
+      base16_decode(cons2_hash, DIGEST256_LEN,
+          cons2_hash_hex, HEX_DIGEST256_LEN) != DIGEST256_LEN) {
+    log_info(LD_CONSDIFF, "The provided consensus diff includes "
+        "malformed digests.");
+    goto error_cleanup;
+  }
+
+  if (digest1_out) {
+    memcpy(digest1_out, cons1_hash, DIGEST256_LEN);
+  }
+  if (digest2_out) {
+    memcpy(digest2_out, cons2_hash, DIGEST256_LEN);
+  }
+
+  SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+  smartlist_free(hash_words);
+  return 0;
+
+ error_cleanup:
+
+  if (hash_words) {
+    SMARTLIST_FOREACH(hash_words, char *, cp, tor_free(cp));
+    smartlist_free(hash_words);
+  }
+  return 1;
+}
+
+/** Apply the consensus diff to the given consensus and return a new
+ * consensus, also as a line-based smartlist. Will return NULL if the diff
+ * could not be applied. Neither the consensus nor the diff are modified in
+ * any way, so it's up to the caller to free their resources.
+ */
+char *
+consdiff_apply_diff(const smartlist_t *cons1,
+                    const smartlist_t *diff,
+                    const consensus_digest_t *digests1)
+{
+  smartlist_t *cons2 = NULL;
+  char *cons2_str = NULL;
+  char e_cons1_hash[DIGEST256_LEN];
+  char e_cons2_hash[DIGEST256_LEN];
+
+  if (consdiff_get_digests(diff, e_cons1_hash, e_cons2_hash) != 0) {
+    goto error_cleanup;
+  }
+
+  /* See that the consensus that was given to us matches its hash. */
+  if (!consensus_digest_eq(digests1->sha3_256,
+                           (const uint8_t*)e_cons1_hash)) {
+    char hex_digest1[HEX_DIGEST256_LEN+1];
+    char e_hex_digest1[HEX_DIGEST256_LEN+1];
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the base consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    base16_encode(hex_digest1, HEX_DIGEST256_LEN+1,
+                  (const char *)digests1->sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest1, HEX_DIGEST256_LEN+1,
+                  e_cons1_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest1, e_hex_digest1);
+    goto error_cleanup;
+  }
+
+  /* Grab the ed diff and calculate the resulting consensus. */
+  /* Skip the first two lines. */
+  cons2 = apply_ed_diff(cons1, diff, 2);
+
+  /* ed diff could not be applied - reason already logged by apply_ed_diff. */
+  if (!cons2) {
+    goto error_cleanup;
+  }
+
+  cons2_str = consensus_join_lines(cons2);
+
+  consensus_digest_t cons2_digests;
+  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+    /* LCOV_EXCL_START -- digest can't fail */
+    log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
+        "resulting from applying a consensus diff.");
+    goto error_cleanup;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* See that the resulting consensus matches its hash. */
+  if (!consensus_digest_eq(cons2_digests.sha3_256,
+                           (const uint8_t*)e_cons2_hash)) {
+    log_warn(LD_CONSDIFF, "Refusing to apply consensus diff because "
+        "the resulting consensus doesn't match the digest as found in "
+        "the consensus diff header.");
+    char hex_digest2[HEX_DIGEST256_LEN+1];
+    char e_hex_digest2[HEX_DIGEST256_LEN+1];
+    base16_encode(hex_digest2, HEX_DIGEST256_LEN+1,
+        (const char *)cons2_digests.sha3_256, DIGEST256_LEN);
+    base16_encode(e_hex_digest2, HEX_DIGEST256_LEN+1,
+        e_cons2_hash, DIGEST256_LEN);
+    log_warn(LD_CONSDIFF, "Expected: %s; found: %s",
+             hex_digest2, e_hex_digest2);
+    goto error_cleanup;
+  }
+
+  goto done;
+
+ error_cleanup:
+  tor_free(cons2_str); /* Sets it to NULL */
+
+ done:
+  if (cons2) {
+    smartlist_free(cons2);
+  }
+
+  return cons2_str;
+}
+
+/** Any consensus line longer than this means that the input is invalid. */
+#define CONSENSUS_LINE_MAX_LEN (1<<20)
+
+/**
+ * Helper: For every NL-terminated line in <b>s</b>, add a cdline referring to
+ * that line (without trailing newline) to <b>out</b>.  Return -1 if there are
+ * any non-NL terminated lines; 0 otherwise.
+ *
+ * Unlike tor_split_lines, this function avoids ambiguity on its
+ * handling of a final line that isn't NL-terminated.
+ *
+ * All cdline_t objects are allocated in the provided memarea.  Strings
+ * are not copied: if <b>s</b> changes or becomes invalid, then all
+ * generated cdlines will become invalid.
+ */
+STATIC int
+consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+{
+  const char *end_of_str = s + strlen(s);
+  tor_assert(*end_of_str == '\0');
+
+  while (*s) {
+    const char *eol = memchr(s, '\n', end_of_str - s);
+    if (!eol) {
+      /* File doesn't end with newline. */
+      return -1;
+    }
+    if (eol - s > CONSENSUS_LINE_MAX_LEN) {
+      /* Line is far too long. */
+      return -1;
+    }
+    cdline_t *line = memarea_alloc(area, sizeof(cdline_t));
+    line->s = s;
+    line->len = (uint32_t)(eol - s);
+    smartlist_add(out, line);
+    s = eol+1;
+  }
+  return 0;
+}
+
+/** Given a list of cdline_t, return a newly allocated string containing
+ * all of the lines, terminated with NL, concatenated.
+ *
+ * Unlike smartlist_join_strings(), avoids lossy operations on empty
+ * lists.  */
+static char *
+consensus_join_lines(const smartlist_t *inp)
+{
+  size_t n = 0;
+  SMARTLIST_FOREACH(inp, const cdline_t *, cdline, n += cdline->len + 1);
+  n += 1;
+  char *result = tor_malloc(n);
+  char *out = result;
+  SMARTLIST_FOREACH_BEGIN(inp, const cdline_t *, cdline) {
+    memcpy(out, cdline->s, cdline->len);
+    out += cdline->len;
+    *out++ = '\n';
+  } SMARTLIST_FOREACH_END(cdline);
+  *out++ = '\0';
+  tor_assert(out == result+n);
+  return result;
+}
+
+/** Given two consensus documents, try to compute a diff between them.  On
+ * success, retun a newly allocated string containing that diff.  On failure,
+ * return NULL. */
+char *
+consensus_diff_generate(const char *cons1,
+                        const char *cons2)
+{
+  consensus_digest_t d1, d2;
+  smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
+  int r1, r2;
+  char *result = NULL;
+
+  r1 = consensus_compute_digest_as_signed(cons1, &d1);
+  r2 = consensus_compute_digest(cons2, &d2);
+  if (BUG(r1 < 0 || r2 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  memarea_t *area = memarea_new();
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, cons1, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, cons2, area) < 0)
+    goto done;
+
+  result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
+
+ done:
+  if (result_lines) {
+    result = consensus_join_lines(result_lines);
+    smartlist_free(result_lines);
+  }
+
+  memarea_drop_all(area);
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+
+  return result;
+}
+
+/** Given a consensus document and a diff, try to apply the diff to the
+ * consensus.  On success return a newly allocated string containing the new
+ * consensus.  On failure, return NULL. */
+char *
+consensus_diff_apply(const char *consensus,
+                     const char *diff)
+{
+  consensus_digest_t d1;
+  smartlist_t *lines1 = NULL, *lines2 = NULL;
+  int r1;
+  char *result = NULL;
+  memarea_t *area = memarea_new();
+
+  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  if (BUG(r1 < 0))
+    return NULL; // LCOV_EXCL_LINE
+
+  lines1 = smartlist_new();
+  lines2 = smartlist_new();
+  if (consensus_split_lines(lines1, consensus, area) < 0)
+    goto done;
+  if (consensus_split_lines(lines2, diff, area) < 0)
+    goto done;
+
+  result = consdiff_apply_diff(lines1, lines2, &d1);
+
+ done:
+  smartlist_free(lines1);
+  smartlist_free(lines2);
+  memarea_drop_all(area);
+
+  return result;
+}
+
+/** Return true iff, based on its header, <b>document</b> is likely
+ * to be a consensus diff. */
+int
+looks_like_a_consensus_diff(const char *document, size_t len)
+{
+  return (len >= strlen(ns_diff_version) &&
+          fast_memeq(document, ns_diff_version, strlen(ns_diff_version)));
+}
+
diff --git a/src/or/consdiff.h b/src/or/consdiff.h
new file mode 100644
index 0000000000..d05df74b75
--- /dev/null
+++ b/src/or/consdiff.h
@@ -0,0 +1,98 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFF_H
+#define TOR_CONSDIFF_H
+
+#include "or.h"
+
+char *consensus_diff_generate(const char *cons1,
+                              const char *cons2);
+char *consensus_diff_apply(const char *consensus,
+                           const char *diff);
+
+int looks_like_a_consensus_diff(const char *document, size_t len);
+
+#ifdef CONSDIFF_PRIVATE
+struct memarea_t;
+
+/** Line type used for constructing consensus diffs.  Each of these lines
+ * refers to a chunk of memory allocated elsewhere, and is not necessarily
+ * NUL-terminated: this helps us avoid copies and save memory. */
+typedef struct cdline_t {
+  const char *s;
+  uint32_t len;
+} cdline_t;
+
+typedef struct consensus_digest_t {
+  uint8_t sha3_256[DIGEST256_LEN];
+} consensus_digest_t;
+
+STATIC smartlist_t *consdiff_gen_diff(const smartlist_t *cons1,
+                                      const smartlist_t *cons2,
+                                      const consensus_digest_t *digests1,
+                                      const consensus_digest_t *digests2,
+                                      struct memarea_t *area);
+STATIC char *consdiff_apply_diff(const smartlist_t *cons1,
+                                 const smartlist_t *diff,
+                                 const consensus_digest_t *digests1);
+STATIC int consdiff_get_digests(const smartlist_t *diff,
+                                char *digest1_out,
+                                char *digest2_out);
+
+/** Data structure to define a slice of a smarltist. */
+typedef struct smartlist_slice_t {
+  /**
+   * Smartlist that this slice is made from.
+   * References the whole original smartlist that the slice was made out of.
+   * */
+  const smartlist_t *list;
+  /** Starting position of the slice in the smartlist. */
+  int offset;
+  /** Length of the slice, i.e. the number of elements it holds. */
+  int len;
+} smartlist_slice_t;
+STATIC smartlist_t *gen_ed_diff(const smartlist_t *cons1,
+                                const smartlist_t *cons2,
+                                struct memarea_t *area);
+STATIC smartlist_t *apply_ed_diff(const smartlist_t *cons1,
+                                  const smartlist_t *diff,
+                                  int start_line);
+STATIC void calc_changes(smartlist_slice_t *slice1, smartlist_slice_t *slice2,
+                         bitarray_t *changed1, bitarray_t *changed2);
+STATIC smartlist_slice_t *smartlist_slice(const smartlist_t *list,
+                                          int start, int end);
+STATIC int next_router(const smartlist_t *cons, int cur);
+STATIC int *lcs_lengths(const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2,
+                        int direction);
+STATIC void trim_slices(smartlist_slice_t *slice1, smartlist_slice_t *slice2);
+STATIC int base64cmp(const cdline_t *hash1, const cdline_t *hash2);
+STATIC int get_id_hash(const cdline_t *line, cdline_t *hash_out);
+STATIC int is_valid_router_entry(const cdline_t *line);
+STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
+                                      const cdline_t *string);
+STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
+                        const smartlist_slice_t *slice1,
+                        const smartlist_slice_t *slice2);
+STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+                                 struct memarea_t *area);
+STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
+                                  const char *s);
+STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
+STATIC int line_str_eq(const cdline_t *a, const char *b);
+
+MOCK_DECL(STATIC int,
+          consensus_compute_digest,(const char *cons,
+                                    consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_compute_digest_as_signed,(const char *cons,
+                                              consensus_digest_t *digest_out));
+MOCK_DECL(STATIC int,
+          consensus_digest_eq,(const uint8_t *d1,
+                               const uint8_t *d2));
+#endif
+
+#endif
+
diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c
new file mode 100644
index 0000000000..a4ff9f2dad
--- /dev/null
+++ b/src/or/consdiffmgr.c
@@ -0,0 +1,1900 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file consdiffmsr.c
+ *
+ * \brief consensus diff manager functions
+ *
+ * This module is run by directory authorities and caches in order
+ * to remember a number of past consensus documents, and to generate
+ * and serve the diffs from those documents to the latest consensus.
+ */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "consdiff.h"
+#include "consdiffmgr.h"
+#include "cpuworker.h"
+#include "networkstatus.h"
+#include "routerparse.h"
+#include "workqueue.h"
+
+/**
+ * Labels to apply to items in the conscache object.
+ *
+ * @{
+ */
+/* One of DOCTYPE_CONSENSUS or DOCTYPE_CONSENSUS_DIFF */
+#define LABEL_DOCTYPE "document-type"
+/* The valid-after time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_AFTER "consensus-valid-after"
+/* The fresh-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_FRESH_UNTIL "consensus-fresh-until"
+/* The valid-until time for a consensus (or for the target consensus of a
+ * diff), encoded as ISO UTC. */
+#define LABEL_VALID_UNTIL "consensus-valid-until"
+/* Comma-separated list of hex-encoded identity digests for the voting
+ * authorities. */
+#define LABEL_SIGNATORIES "consensus-signatories"
+/* A hex encoded SHA3 digest of the object, as compressed (if any) */
+#define LABEL_SHA3_DIGEST "sha3-digest"
+/* A hex encoded SHA3 digest of the object before compression. */
+#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"
+/* A hex encoded SHA3 digest-as-signed of a consensus */
+#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"
+/* The flavor of the consensus or consensuses diff */
+#define LABEL_FLAVOR "consensus-flavor"
+/* Diff only: the SHA3 digest-as-signed of the source consensus. */
+#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"
+/* Diff only: the SHA3 digest-in-full of the target consensus. */
+#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"
+/* Diff only: the valid-after date of the source consensus. */
+#define LABEL_FROM_VALID_AFTER "from-valid-after"
+/* What kind of compression was used? */
+#define LABEL_COMPRESSION_TYPE "compression"
+/** @} */
+
+#define DOCTYPE_CONSENSUS "consensus"
+#define DOCTYPE_CONSENSUS_DIFF "consensus-diff"
+
+/**
+ * Underlying directory that stores consensuses and consensus diffs.  Don't
+ * use this directly: use cdm_cache_get() instead.
+ */
+static consensus_cache_t *cons_diff_cache = NULL;
+/**
+ * If true, we have learned at least one new consensus since the
+ * consensus cache was last up-to-date.
+ */
+static int cdm_cache_dirty = 0;
+/**
+ * If true, we have scanned the cache to update our hashtable of diffs.
+ */
+static int cdm_cache_loaded = 0;
+
+/**
+ * Possible status values for cdm_diff_t.cdm_diff_status
+ **/
+typedef enum cdm_diff_status_t {
+  CDM_DIFF_PRESENT=1,
+  CDM_DIFF_IN_PROGRESS=2,
+  CDM_DIFF_ERROR=3,
+} cdm_diff_status_t;
+
+/** Which methods do we use for precompressing diffs? */
+static const compress_method_t compress_diffs_with[] = {
+  NO_METHOD,
+  GZIP_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_diff_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_diffs_with);
+}
+
+/** Which methods do we use for precompressing consensuses? */
+static const compress_method_t compress_consensus_with[] = {
+  ZLIB_METHOD,
+#ifdef HAVE_LZMA
+  LZMA_METHOD,
+#endif
+#ifdef HAVE_ZSTD
+  ZSTD_METHOD,
+#endif
+};
+
+/** How many different methods will we try to use for diff compression? */
+STATIC unsigned
+n_consensus_compression_methods(void)
+{
+  return ARRAY_LENGTH(compress_consensus_with);
+}
+
+/** For which compression method do we retain old consensuses?  There's no
+ * need to keep all of them, since we won't be serving them.  We'll
+ * go with ZLIB_METHOD because it's pretty fast and everyone has it.
+ */
+#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD
+
+/** Handles pointing to the latest consensus entries as compressed and
+ * stored. */
+static consensus_cache_entry_handle_t *
+                  latest_consensus[N_CONSENSUS_FLAVORS]
+                                  [ARRAY_LENGTH(compress_consensus_with)];
+
+/** Hashtable node used to remember the current status of the diff
+ * from a given sha3 digest to the current consensus.  */
+typedef struct cdm_diff_t {
+  HT_ENTRY(cdm_diff_t) node;
+
+  /** Consensus flavor for this diff (part of ht key) */
+  consensus_flavor_t flavor;
+  /** SHA3-256 digest of the consensus that this diff is _from_. (part of the
+   * ht key) */
+  uint8_t from_sha3[DIGEST256_LEN];
+  /** Method by which the diff is compressed. (part of the ht key */
+  compress_method_t compress_method;
+
+  /** One of the CDM_DIFF_* values, depending on whether this diff
+   * is available, in progress, or impossible to compute. */
+  cdm_diff_status_t cdm_diff_status;
+  /** SHA3-256 digest of the consensus that this diff is _to. */
+  uint8_t target_sha3[DIGEST256_LEN];
+
+  /** Handle to the cache entry for this diff, if any.  We use a handle here
+   * to avoid thinking too hard about cache entry lifetime issues. */
+  consensus_cache_entry_handle_t *entry;
+} cdm_diff_t;
+
+/** Hashtable mapping flavor and source consensus digest to status. */
+static HT_HEAD(cdm_diff_ht, cdm_diff_t) cdm_diff_ht = HT_INITIALIZER();
+
+/**
+ * Configuration for this module
+ */
+static consdiff_cfg_t consdiff_cfg = {
+  // XXXX I'd like to make this number bigger, but it interferes with the
+  // XXXX seccomp2 syscall filter, which tops out at BPF_MAXINS (4096)
+  // XXXX rules.
+  /* .cache_max_num = */ 128
+};
+
+static int consdiffmgr_ensure_space_for_files(int n);
+static int consensus_queue_compression_work(const char *consensus,
+                                            const networkstatus_t *as_parsed);
+static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                                          consensus_cache_entry_t *diff_to);
+static void consdiffmgr_set_cache_flags(void);
+
+/* =====
+ * Hashtable setup
+ * ===== */
+
+/** Helper: hash the key of a cdm_diff_t. */
+static unsigned
+cdm_diff_hash(const cdm_diff_t *diff)
+{
+  uint8_t tmp[DIGEST256_LEN + 2];
+  memcpy(tmp, diff->from_sha3, DIGEST256_LEN);
+  tmp[DIGEST256_LEN] = (uint8_t) diff->flavor;
+  tmp[DIGEST256_LEN+1] = (uint8_t) diff->compress_method;
+  return (unsigned) siphash24g(tmp, sizeof(tmp));
+}
+/** Helper: compare two cdm_diff_t objects for key equality */
+static int
+cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
+{
+  return fast_memeq(diff1->from_sha3, diff2->from_sha3, DIGEST256_LEN) &&
+    diff1->flavor == diff2->flavor &&
+    diff1->compress_method == diff2->compress_method;
+}
+
+HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
+HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
+             0.6, tor_reallocarray, tor_free_)
+
+/** Release all storage held in <b>diff</b>. */
+static void
+cdm_diff_free(cdm_diff_t *diff)
+{
+  if (!diff)
+    return;
+  consensus_cache_entry_handle_free(diff->entry);
+  tor_free(diff);
+}
+
+/** Create and return a new cdm_diff_t with the given values.  Does not
+ * add it to the hashtable. */
+static cdm_diff_t *
+cdm_diff_new(consensus_flavor_t flav,
+             const uint8_t *from_sha3,
+             const uint8_t *target_sha3,
+             compress_method_t method)
+{
+  cdm_diff_t *ent;
+  ent = tor_malloc_zero(sizeof(cdm_diff_t));
+  ent->flavor = flav;
+  memcpy(ent->from_sha3, from_sha3, DIGEST256_LEN);
+  memcpy(ent->target_sha3, target_sha3, DIGEST256_LEN);
+  ent->compress_method = method;
+  return ent;
+}
+
+/**
+ * Examine the diff hashtable to see whether we know anything about computing
+ * a diff of type <b>flav</b> between consensuses with the two provided
+ * SHA3-256 digests.  If a computation is in progress, or if the computation
+ * has already been tried and failed, return 1.  Otherwise, note the
+ * computation as "in progress" so that we don't reattempt it later, and
+ * return 0.
+ */
+static int
+cdm_diff_ht_check_and_note_pending(consensus_flavor_t flav,
+                                   const uint8_t *from_sha3,
+                                   const uint8_t *target_sha3)
+{
+  struct cdm_diff_t search, *ent;
+  unsigned u;
+  int result = 0;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    memset(&search, 0, sizeof(cdm_diff_t));
+    search.flavor = flav;
+    search.compress_method = method;
+    memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+    ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+    if (ent) {
+      tor_assert_nonfatal(ent->cdm_diff_status != CDM_DIFF_PRESENT);
+      result = 1;
+      continue;
+    }
+    ent = cdm_diff_new(flav, from_sha3, target_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  }
+  return result;
+}
+
+/**
+ * Update the status of the diff of type <b>flav</b> between consensuses with
+ * the two provided SHA3-256 digests, so that its status becomes
+ * <b>status</b>, and its value becomes the <b>handle</b>.  If <b>handle</b>
+ * is NULL, then the old handle (if any) is freed, and replaced with NULL.
+ */
+static void
+cdm_diff_ht_set_status(consensus_flavor_t flav,
+                       const uint8_t *from_sha3,
+                       const uint8_t *to_sha3,
+                       compress_method_t method,
+                       int status,
+                       consensus_cache_entry_handle_t *handle)
+{
+  if (handle == NULL) {
+    tor_assert_nonfatal(status != CDM_DIFF_PRESENT);
+  }
+
+  struct cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(cdm_diff_t));
+  search.flavor = flav;
+  search.compress_method = method,
+  memcpy(search.from_sha3, from_sha3, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+  if (!ent) {
+    ent = cdm_diff_new(flav, from_sha3, to_sha3, method);
+    ent->cdm_diff_status = CDM_DIFF_IN_PROGRESS;
+    HT_INSERT(cdm_diff_ht, &cdm_diff_ht, ent);
+  } else if (fast_memneq(ent->target_sha3, to_sha3, DIGEST256_LEN)) {
+    // This can happen under certain really pathological conditions
+    // if we decide we don't care about a diff before it is actually
+    // done computing.
+    return;
+  }
+
+  tor_assert_nonfatal(ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS);
+
+  ent->cdm_diff_status = status;
+  consensus_cache_entry_handle_free(ent->entry);
+  ent->entry = handle;
+}
+
+/**
+ * Helper: Remove from the hash table every present (actually computed) diff
+ * of type <b>flav</b> whose target digest does not match
+ * <b>unless_target_sha3_matches</b>.
+ *
+ * This function is used for the hash table to throw away references to diffs
+ * that do not lead to the most given consensus of a given flavor.
+ */
+static void
+cdm_diff_ht_purge(consensus_flavor_t flav,
+                  const uint8_t *unless_target_sha3_matches)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+
+    if ((*diff)->cdm_diff_status == CDM_DIFF_PRESENT &&
+        flav == (*diff)->flavor) {
+
+      if (BUG((*diff)->entry == NULL) ||
+          consensus_cache_entry_handle_get((*diff)->entry) == NULL) {
+        /* the underlying entry has gone away; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+
+      if (unless_target_sha3_matches &&
+          fast_memneq(unless_target_sha3_matches, (*diff)->target_sha3,
+                      DIGEST256_LEN)) {
+        /* target hash doesn't match; drop this. */
+        next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+        cdm_diff_free(this);
+        continue;
+      }
+    }
+    next = HT_NEXT(cdm_diff_ht, &cdm_diff_ht, diff);
+  }
+}
+
+/**
+ * Helper: initialize <b>cons_diff_cache</b>.
+ */
+static void
+cdm_cache_init(void)
+{
+  unsigned n_entries = consdiff_cfg.cache_max_num * 2;
+
+  tor_assert(cons_diff_cache == NULL);
+  cons_diff_cache = consensus_cache_open("diff-cache", n_entries);
+  if (cons_diff_cache == NULL) {
+    // LCOV_EXCL_START
+    log_err(LD_FS, "Error: Couldn't open storage for consensus diffs.");
+    tor_assert_unreached();
+    // LCOV_EXCL_STOP
+  } else {
+    consdiffmgr_set_cache_flags();
+  }
+  cdm_cache_dirty = 1;
+  cdm_cache_loaded = 0;
+}
+
+/**
+ * Helper: return the consensus_cache_t * that backs this manager,
+ * initializing it if needed.
+ */
+STATIC consensus_cache_t *
+cdm_cache_get(void)
+{
+  if (PREDICT_UNLIKELY(cons_diff_cache == NULL)) {
+    cdm_cache_init();
+  }
+  return cons_diff_cache;
+}
+
+/**
+ * Helper: given a list of labels, prepend the hex-encoded SHA3 digest
+ * of the <b>bodylen</b>-byte object at <b>body</b> to those labels,
+ * with <b>label</b> as its label.
+ */
+static void
+cdm_labels_prepend_sha3(config_line_t **labels,
+                        const char *label,
+                        const uint8_t *body,
+                        size_t bodylen)
+{
+  uint8_t sha3_digest[DIGEST256_LEN];
+  char hexdigest[HEX_DIGEST256_LEN+1];
+  crypto_digest256((char *)sha3_digest,
+                   (const char *)body, bodylen, DIGEST_SHA3_256);
+  base16_encode(hexdigest, sizeof(hexdigest),
+                (const char *)sha3_digest, sizeof(sha3_digest));
+
+  config_line_prepend(labels, label, hexdigest);
+}
+
+/** Helper: if there is a sha3-256 hex-encoded digest in <b>ent</b> with the
+ * given label, set <b>digest_out</b> to that value (decoded), and return 0.
+ *
+ * Return -1 if there is no such label, and -2 if it is badly formatted. */
+STATIC int
+cdm_entry_get_sha3_value(uint8_t *digest_out,
+                         consensus_cache_entry_t *ent,
+                         const char *label)
+{
+  if (ent == NULL)
+    return -1;
+
+  const char *hex = consensus_cache_entry_get_value(ent, label);
+  if (hex == NULL)
+    return -1;
+
+  int n = base16_decode((char*)digest_out, DIGEST256_LEN, hex, strlen(hex));
+  if (n != DIGEST256_LEN)
+    return -2;
+  else
+    return 0;
+}
+
+/**
+ * Helper: look for a consensus with the given <b>flavor</b> and
+ * <b>valid_after</b> time in the cache. Return that consensus if it's
+ * present, or NULL if it's missing.
+ */
+STATIC consensus_cache_entry_t *
+cdm_cache_lookup_consensus(consensus_flavor_t flavor, time_t valid_after)
+{
+  char formatted_time[ISO_TIME_LEN+1];
+  format_iso_time_nospace(formatted_time, valid_after);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  /* We'll filter by valid-after time first, since that should
+   * match the fewest documents. */
+  /* We could add an extra hashtable here, but since we only do this scan
+   * when adding a new consensus, it probably doesn't matter much. */
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_VALID_AFTER, formatted_time);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  consensus_cache_entry_t *result = NULL;
+  if (smartlist_len(matches)) {
+    result = smartlist_get(matches, 0);
+  }
+  smartlist_free(matches);
+
+  return result;
+}
+
+/** Return the maximum age (in seconds) of consensuses that we should consider
+ * storing. The available space in the directory may impose additional limits
+ * on how much we store. */
+static int32_t
+get_max_age_to_cache(void)
+{
+  const int32_t DEFAULT_MAX_AGE_TO_CACHE = 8192;
+  const int32_t MIN_MAX_AGE_TO_CACHE = 0;
+  const int32_t MAX_MAX_AGE_TO_CACHE = 8192;
+  const char MAX_AGE_TO_CACHE_NAME[] = "max-consensus-age-to-cache-for-diff";
+
+  const or_options_t *options = get_options();
+
+  if (options->MaxConsensusAgeForDiffs) {
+    const int v = options->MaxConsensusAgeForDiffs;
+    if (v >= MAX_MAX_AGE_TO_CACHE * 3600)
+      return MAX_MAX_AGE_TO_CACHE;
+    else
+      return v;
+  }
+
+  /* The parameter is in hours, so we multiply */
+  return 3600 * networkstatus_get_param(NULL,
+                                        MAX_AGE_TO_CACHE_NAME,
+                                        DEFAULT_MAX_AGE_TO_CACHE,
+                                        MIN_MAX_AGE_TO_CACHE,
+                                        MAX_MAX_AGE_TO_CACHE);
+}
+
+/**
+ * Given a string containing a networkstatus consensus, and the results of
+ * having parsed that consensus, add that consensus to the cache if it is not
+ * already present and not too old.  Create new consensus diffs from or to
+ * that consensus as appropriate.
+ *
+ * Return 0 on success and -1 on failure.
+ */
+int
+consdiffmgr_add_consensus(const char *consensus,
+                          const networkstatus_t *as_parsed)
+{
+  if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
+    return -1; // LCOV_EXCL_LINE
+  if (BUG(as_parsed->type != NS_TYPE_CONSENSUS))
+    return -1; // LCOV_EXCL_LINE
+
+  const consensus_flavor_t flavor = as_parsed->flavor;
+  const time_t valid_after = as_parsed->valid_after;
+
+  if (valid_after < approx_time() - get_max_age_to_cache()) {
+    log_info(LD_DIRSERV, "We don't care about this consensus document; it's "
+             "too old.");
+    return -1;
+  }
+
+  /* Do we already have this one? */
+  consensus_cache_entry_t *entry =
+    cdm_cache_lookup_consensus(flavor, valid_after);
+  if (entry) {
+    log_info(LD_DIRSERV, "We already have a copy of that consensus");
+    return -1;
+  }
+
+  /* We don't have it. Add it to the cache. */
+  return consensus_queue_compression_work(consensus, as_parsed);
+}
+
+/**
+ * Helper: used to sort two smartlists of consensus_cache_entry_t by their
+ * LABEL_VALID_AFTER labels.
+ */
+static int
+compare_by_valid_after_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  /* We're in luck here: sorting UTC iso-encoded values lexically will work
+   * fine (until 9999). */
+  return strcmp_opt(consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER),
+                    consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER));
+}
+
+/**
+ * Helper: Sort <b>lst</b> by LABEL_VALID_AFTER and return the most recent
+ * entry.
+ */
+static consensus_cache_entry_t *
+sort_and_find_most_recent(smartlist_t *lst)
+{
+  smartlist_sort(lst, compare_by_valid_after_);
+  if (smartlist_len(lst)) {
+    return smartlist_get(lst, smartlist_len(lst) - 1);
+  } else {
+    return NULL;
+  }
+}
+
+/** Return i such that compress_consensus_with[i] == method. Return
+ * -1 if no such i exists. */
+static int
+consensus_compression_method_pos(compress_method_t method)
+{
+  unsigned i;
+  for (i = 0; i < n_consensus_compression_methods(); ++i) {
+    if (compress_consensus_with[i] == method) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+/**
+ * If we know a consensus with the flavor <b>flavor</b> compressed with
+ * <b>method</b>, set *<b>entry_out</b> to that value.  Return values are as
+ * for consdiffmgr_find_diff_from().
+ */
+consdiff_status_t
+consdiffmgr_find_consensus(struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           compress_method_t method)
+{
+  tor_assert(entry_out);
+  tor_assert((int)flavor < N_CONSENSUS_FLAVORS);
+
+  int pos = consensus_compression_method_pos(method);
+  if (pos < 0) {
+     // We don't compress consensuses with this method.
+    return CONSDIFF_NOT_FOUND;
+  }
+  consensus_cache_entry_handle_t *handle = latest_consensus[flavor][pos];
+  if (!handle)
+    return CONSDIFF_NOT_FOUND;
+  *entry_out = consensus_cache_entry_handle_get(handle);
+  if (*entry_out)
+    return CONSDIFF_AVAILABLE;
+  else
+    return CONSDIFF_NOT_FOUND;
+}
+
+/**
+ * Look up consensus_cache_entry_t for the consensus of type <b>flavor</b>,
+ * from the source consensus with the specified digest (which must be SHA3).
+ *
+ * If the diff is present, store it into *<b>entry_out</b> and return
+ * CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or
+ * CONSDIFF_IN_PROGRESS.
+ */
+consdiff_status_t
+consdiffmgr_find_diff_from(consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           compress_method_t method)
+{
+  if (BUG(digest_type != DIGEST_SHA3_256) ||
+      BUG(digestlen != DIGEST256_LEN)) {
+    return CONSDIFF_NOT_FOUND; // LCOV_EXCL_LINE
+  }
+
+  // Try to look up the entry in the hashtable.
+  cdm_diff_t search, *ent;
+  memset(&search, 0, sizeof(search));
+  search.flavor = flavor;
+  search.compress_method = method;
+  memcpy(search.from_sha3, digest, DIGEST256_LEN);
+  ent = HT_FIND(cdm_diff_ht, &cdm_diff_ht, &search);
+
+  if (ent == NULL ||
+      ent->cdm_diff_status == CDM_DIFF_ERROR) {
+    return CONSDIFF_NOT_FOUND;
+  } else if (ent->cdm_diff_status == CDM_DIFF_IN_PROGRESS) {
+    return CONSDIFF_IN_PROGRESS;
+  } else if (BUG(ent->cdm_diff_status != CDM_DIFF_PRESENT)) {
+    return CONSDIFF_IN_PROGRESS;
+  }
+
+  if (BUG(ent->entry == NULL)) {
+    return CONSDIFF_NOT_FOUND;
+  }
+  *entry_out = consensus_cache_entry_handle_get(ent->entry);
+  return (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+
+#if 0
+  // XXXX Remove this.  I'm keeping it around for now in case we need to
+  // XXXX debug issues in the hashtable.
+  char hex[HEX_DIGEST256_LEN+1];
+  base16_encode(hex, sizeof(hex), (const char *)digest, digestlen);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  smartlist_t *matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FROM_SHA3_DIGEST, hex);
+  consensus_cache_filter_list(matches, LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+
+  *entry_out = sort_and_find_most_recent(matches);
+  consdiff_status_t result =
+    (*entry_out) ? CONSDIFF_AVAILABLE : CONSDIFF_NOT_FOUND;
+  smartlist_free(matches);
+
+  return result;
+#endif
+}
+
+/**
+ * Perform periodic cleanup tasks on the consensus diff cache.  Return
+ * the number of objects marked for deletion.
+ */
+int
+consdiffmgr_cleanup(void)
+{
+  smartlist_t *objects = smartlist_new();
+  smartlist_t *consensuses = smartlist_new();
+  smartlist_t *diffs = smartlist_new();
+  int n_to_delete = 0;
+
+  log_debug(LD_DIRSERV, "Looking for consdiffmgr entries to remove");
+
+  // 1. Delete any consensus or diff or anything whose valid_after is too old.
+  const time_t valid_after_cutoff = approx_time() - get_max_age_to_cache();
+
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    const char *lv_valid_after =
+      consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (! lv_valid_after) {
+      log_debug(LD_DIRSERV, "Ignoring entry because it had no %s label",
+                LABEL_VALID_AFTER);
+      continue;
+    }
+    time_t valid_after = 0;
+    if (parse_iso_time_nospace(lv_valid_after, &valid_after) < 0) {
+      log_debug(LD_DIRSERV, "Ignoring entry because its %s value (%s) was "
+                "unparseable", LABEL_VALID_AFTER, escaped(lv_valid_after));
+      continue;
+    }
+    if (valid_after < valid_after_cutoff) {
+      log_debug(LD_DIRSERV, "Deleting entry because its %s value (%s) was "
+                "too old", LABEL_VALID_AFTER, lv_valid_after);
+      consensus_cache_entry_mark_for_removal(ent);
+      ++n_to_delete;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+
+  // 2. Delete all diffs that lead to a consensus whose valid-after is not the
+  // latest.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+
+    /* consider all such-flavored diffs, and look to see if they match. */
+    consensus_cache_find_all(diffs, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+    consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+    SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+      const char *this_diff_target_sha3 =
+        consensus_cache_entry_get_value(diff, LABEL_TARGET_SHA3_DIGEST);
+      if (!this_diff_target_sha3)
+        continue;
+      if (strcmp(this_diff_target_sha3, most_recent_sha3)) {
+        consensus_cache_entry_mark_for_removal(diff);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(diff);
+    smartlist_clear(consensuses);
+    smartlist_clear(diffs);
+  }
+
+  // 3. Delete all consensuses except the most recent that are compressed with
+  // an un-preferred method.
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    /* Determine the most recent consensus of this flavor */
+    consensus_cache_find_all(consensuses, cdm_cache_get(),
+                             LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_filter_list(consensuses, LABEL_FLAVOR, flavname);
+    consensus_cache_entry_t *most_recent =
+      sort_and_find_most_recent(consensuses);
+    if (most_recent == NULL)
+      continue;
+    const char *most_recent_sha3_uncompressed =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    const char *retain_methodname = compression_method_get_name(
+                               RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+    if (BUG(most_recent_sha3_uncompressed == NULL))
+      continue;
+    SMARTLIST_FOREACH_BEGIN(consensuses, consensus_cache_entry_t *, ent) {
+      const char *lv_sha3_uncompressed =
+        consensus_cache_entry_get_value(ent, LABEL_SHA3_DIGEST_UNCOMPRESSED);
+      if (BUG(! lv_sha3_uncompressed))
+        continue;
+      if (!strcmp(lv_sha3_uncompressed, most_recent_sha3_uncompressed))
+        continue; // This _is_ the most recent.
+      const char *lv_methodname =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      if (! lv_methodname || strcmp(lv_methodname, retain_methodname)) {
+        consensus_cache_entry_mark_for_removal(ent);
+        ++n_to_delete;
+      }
+    } SMARTLIST_FOREACH_END(ent);
+  }
+
+  smartlist_free(objects);
+  smartlist_free(consensuses);
+  smartlist_free(diffs);
+
+  // Actually remove files, if they're not used.
+  consensus_cache_delete_pending(cdm_cache_get(), 0);
+  return n_to_delete;
+}
+
+/**
+ * Initialize the consensus diff manager and its cache, and configure
+ * its parameters based on the latest torrc and networkstatus parameters.
+ */
+void
+consdiffmgr_configure(const consdiff_cfg_t *cfg)
+{
+  if (cfg)
+    memcpy(&consdiff_cfg, cfg, sizeof(consdiff_cfg));
+
+  (void) cdm_cache_get();
+}
+
+/**
+ * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
+ * operations that the consensus diff manager will need.
+ */
+int
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
+{
+  return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
+}
+
+/**
+ * Scan the consensus diff manager's cache for any grossly malformed entries,
+ * and mark them as deletable.  Return 0 if no problems were found; 1
+ * if problems were found and fixed.
+ */
+int
+consdiffmgr_validate(void)
+{
+  /* Right now, we only check for entries that have bad sha3 values */
+  int problems = 0;
+
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(),
+                           NULL, NULL);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, obj) {
+    uint8_t sha3_expected[DIGEST256_LEN];
+    uint8_t sha3_received[DIGEST256_LEN];
+    int r = cdm_entry_get_sha3_value(sha3_expected, obj, LABEL_SHA3_DIGEST);
+    if (r == -1) {
+      /* digest isn't there; that's allowed */
+      continue;
+    } else if (r == -2) {
+      /* digest is malformed; that's not allowed */
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+    const uint8_t *body;
+    size_t bodylen;
+    consensus_cache_entry_incref(obj);
+    r = consensus_cache_entry_get_body(obj, &body, &bodylen);
+    if (r == 0) {
+      crypto_digest256((char *)sha3_received, (const char *)body, bodylen,
+                       DIGEST_SHA3_256);
+    }
+    consensus_cache_entry_decref(obj);
+    if (r < 0)
+      continue;
+
+    // Deconfuse coverity about the possibility of sha3_received being
+    // uninitialized
+    tor_assert(r <= 0);
+
+    if (fast_memneq(sha3_received, sha3_expected, DIGEST256_LEN)) {
+      problems = 1;
+      consensus_cache_entry_mark_for_removal(obj);
+      continue;
+    }
+
+  } SMARTLIST_FOREACH_END(obj);
+  smartlist_free(objects);
+  return problems;
+}
+
+/**
+ * Helper: build new diffs of <b>flavor</b> as needed
+ */
+static void
+consdiffmgr_rescan_flavor_(consensus_flavor_t flavor)
+{
+  smartlist_t *matches = NULL;
+  smartlist_t *diffs = NULL;
+  smartlist_t *compute_diffs_from = NULL;
+  strmap_t *have_diff_from = NULL;
+
+  // look for the most recent consensus, and for all previous in-range
+  // consensuses.  Do they all have diffs to it?
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  // 1. find the most recent consensus, and the ones that we might want
+  //    to diff to it.
+  const char *methodname = compression_method_get_name(
+                             RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD);
+
+  matches = smartlist_new();
+  consensus_cache_find_all(matches, cdm_cache_get(),
+                           LABEL_FLAVOR, flavname);
+  consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+  consensus_cache_filter_list(matches, LABEL_COMPRESSION_TYPE, methodname);
+  consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+  if (!most_recent) {
+    log_info(LD_DIRSERV, "No 'most recent' %s consensus found; "
+             "not making diffs", flavname);
+    goto done;
+  }
+  tor_assert(smartlist_len(matches));
+  smartlist_del(matches, smartlist_len(matches) - 1);
+
+  const char *most_recent_valid_after =
+    consensus_cache_entry_get_value(most_recent, LABEL_VALID_AFTER);
+  if (BUG(most_recent_valid_after == NULL))
+    goto done; //LCOV_EXCL_LINE
+  uint8_t most_recent_sha3[DIGEST256_LEN];
+  if (BUG(cdm_entry_get_sha3_value(most_recent_sha3, most_recent,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    goto done; //LCOV_EXCL_LINE
+
+  // 2. Find all the relevant diffs _to_ this consensus. These are ones
+  //    that we don't need to compute.
+  diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_VALID_AFTER, most_recent_valid_after);
+  consensus_cache_filter_list(diffs, LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  consensus_cache_filter_list(diffs, LABEL_FLAVOR, flavname);
+  have_diff_from = strmap_new();
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *va = consensus_cache_entry_get_value(diff,
+                                                     LABEL_FROM_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    strmap_set(have_diff_from, va, diff);
+  } SMARTLIST_FOREACH_END(diff);
+
+  // 3. See which consensuses in 'matches' don't have diffs yet.
+  smartlist_reverse(matches); // from newest to oldest.
+  compute_diffs_from = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+    const char *va = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+    if (BUG(va == NULL))
+      continue; // LCOV_EXCL_LINE
+    if (strmap_get(have_diff_from, va) != NULL)
+      continue; /* we already have this one. */
+    smartlist_add(compute_diffs_from, ent);
+    /* Since we are not going to serve this as the most recent consensus
+     * any more, we should stop keeping it mmap'd when it's not in use.
+     */
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+
+  log_info(LD_DIRSERV,
+           "The most recent %s consensus is valid-after %s. We have diffs to "
+           "this consensus for %d/%d older %s consensuses. Generating diffs "
+           "for the other %d.",
+           flavname,
+           most_recent_valid_after,
+           smartlist_len(matches) - smartlist_len(compute_diffs_from),
+           smartlist_len(matches),
+           flavname,
+           smartlist_len(compute_diffs_from));
+
+  // 4. Update the hashtable; remove entries in this flavor to other
+  //    target consensuses.
+  cdm_diff_ht_purge(flavor, most_recent_sha3);
+
+  // 5. Actually launch the requests.
+  SMARTLIST_FOREACH_BEGIN(compute_diffs_from, consensus_cache_entry_t *, c) {
+    if (BUG(c == most_recent))
+      continue; // LCOV_EXCL_LINE
+
+    uint8_t this_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(this_sha3, c,
+                                 LABEL_SHA3_DIGEST_AS_SIGNED)<0) {
+      // Not actually a bug, since we might be running with a directory
+      // with stale files from before the #22143 fixes.
+      continue;
+    }
+    if (cdm_diff_ht_check_and_note_pending(flavor,
+                                           this_sha3, most_recent_sha3)) {
+      // This is already pending, or we encountered an error.
+      continue;
+    }
+    consensus_diff_queue_diff_work(c, most_recent);
+  } SMARTLIST_FOREACH_END(c);
+
+ done:
+  smartlist_free(matches);
+  smartlist_free(diffs);
+  smartlist_free(compute_diffs_from);
+  strmap_free(have_diff_from, NULL);
+}
+
+/**
+ * Scan the cache for the latest consensuses and add their handles to
+ * latest_consensus
+ */
+static void
+consdiffmgr_consensus_load(void)
+{
+  smartlist_t *matches = smartlist_new();
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    const char *flavname = networkstatus_get_flavor_name(flav);
+    smartlist_clear(matches);
+    consensus_cache_find_all(matches, cdm_cache_get(),
+                             LABEL_FLAVOR, flavname);
+    consensus_cache_filter_list(matches, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+    consensus_cache_entry_t *most_recent = sort_and_find_most_recent(matches);
+    if (! most_recent)
+      continue; // no consensuses.
+    const char *most_recent_sha3 =
+      consensus_cache_entry_get_value(most_recent,
+                                      LABEL_SHA3_DIGEST_UNCOMPRESSED);
+    if (BUG(most_recent_sha3 == NULL))
+      continue; // LCOV_EXCL_LINE
+    consensus_cache_filter_list(matches, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                                most_recent_sha3);
+
+    // Everything that remains matches the most recent consensus of this
+    // flavor.
+    SMARTLIST_FOREACH_BEGIN(matches, consensus_cache_entry_t *, ent) {
+      const char *lv_compression =
+        consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+      compress_method_t method =
+        compression_method_get_by_name(lv_compression);
+      int pos = consensus_compression_method_pos(method);
+      if (pos < 0)
+        continue;
+      consensus_cache_entry_handle_free(latest_consensus[flav][pos]);
+      latest_consensus[flav][pos] = consensus_cache_entry_handle_new(ent);
+    } SMARTLIST_FOREACH_END(ent);
+  }
+  smartlist_free(matches);
+}
+
+/**
+ * Scan the cache for diffs, and add them to the hashtable.
+ */
+static void
+consdiffmgr_diffs_load(void)
+{
+  smartlist_t *diffs = smartlist_new();
+  consensus_cache_find_all(diffs, cdm_cache_get(),
+                           LABEL_DOCTYPE, DOCTYPE_CONSENSUS_DIFF);
+  SMARTLIST_FOREACH_BEGIN(diffs, consensus_cache_entry_t *, diff) {
+    const char *lv_flavor =
+      consensus_cache_entry_get_value(diff, LABEL_FLAVOR);
+    if (!lv_flavor)
+      continue;
+    int flavor = networkstatus_parse_flavor_name(lv_flavor);
+    if (flavor < 0)
+      continue;
+    const char *lv_compression =
+      consensus_cache_entry_get_value(diff, LABEL_COMPRESSION_TYPE);
+    compress_method_t method = NO_METHOD;
+    if (lv_compression) {
+      method = compression_method_get_by_name(lv_compression);
+      if (method == UNKNOWN_METHOD) {
+        continue;
+      }
+    }
+
+    uint8_t from_sha3[DIGEST256_LEN];
+    uint8_t to_sha3[DIGEST256_LEN];
+    if (cdm_entry_get_sha3_value(from_sha3, diff, LABEL_FROM_SHA3_DIGEST)<0)
+      continue;
+    if (cdm_entry_get_sha3_value(to_sha3, diff, LABEL_TARGET_SHA3_DIGEST)<0)
+      continue;
+
+    cdm_diff_ht_set_status(flavor, from_sha3, to_sha3,
+                           method,
+                           CDM_DIFF_PRESENT,
+                           consensus_cache_entry_handle_new(diff));
+  } SMARTLIST_FOREACH_END(diff);
+  smartlist_free(diffs);
+}
+
+/**
+ * Build new diffs as needed.
+ */
+void
+consdiffmgr_rescan(void)
+{
+  if (cdm_cache_dirty == 0)
+    return;
+
+  // Clean up here to make room for new diffs, and to ensure that older
+  // consensuses do not have any entries.
+  consdiffmgr_cleanup();
+
+  if (cdm_cache_loaded == 0) {
+    consdiffmgr_diffs_load();
+    consdiffmgr_consensus_load();
+    cdm_cache_loaded = 1;
+  }
+
+  for (int flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
+    consdiffmgr_rescan_flavor_((consensus_flavor_t) flav);
+  }
+
+  cdm_cache_dirty = 0;
+}
+
+/**
+ * Helper: compare two files by their from-valid-after and valid-after labels,
+ * trying to sort in ascending order by from-valid-after (when present) and
+ * valid-after (when not).  Place everything that has neither label first in
+ * the list.
+ */
+static int
+compare_by_staleness_(const void **a, const void **b)
+{
+  const consensus_cache_entry_t *e1 = *a;
+  const consensus_cache_entry_t *e2 = *b;
+  const char *va1, *fva1, *va2, *fva2;
+  va1 = consensus_cache_entry_get_value(e1, LABEL_VALID_AFTER);
+  va2 = consensus_cache_entry_get_value(e2, LABEL_VALID_AFTER);
+  fva1 = consensus_cache_entry_get_value(e1, LABEL_FROM_VALID_AFTER);
+  fva2 = consensus_cache_entry_get_value(e2, LABEL_FROM_VALID_AFTER);
+
+  if (fva1)
+    va1 = fva1;
+  if (fva2)
+    va2 = fva2;
+
+  /* See note about iso-encoded values in compare_by_valid_after_.  Also note
+   * that missing dates will get placed first. */
+  return strcmp_opt(va1, va2);
+}
+
+/** If there are not enough unused filenames to store <b>n</b> files, then
+ * delete old consensuses until there are.  (We have to keep track of the
+ * number of filenames because of the way that the seccomp2 cache works.)
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+consdiffmgr_ensure_space_for_files(int n)
+{
+  consensus_cache_t *cache = cdm_cache_get();
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // there are already enough unused filenames.
+    return 0;
+  }
+  // Try a cheap deletion of stuff that's waiting to get deleted.
+  consensus_cache_delete_pending(cache, 0);
+  if (consensus_cache_get_n_filenames_available(cache) >= n) {
+    // okay, _that_ made enough filenames available.
+    return 0;
+  }
+  // Let's get more assertive: clean out unused stuff, and force-remove
+  // the files that we can.
+  consdiffmgr_cleanup();
+  consensus_cache_delete_pending(cache, 1);
+  const int n_to_remove = n - consensus_cache_get_n_filenames_available(cache);
+  if (n_to_remove <= 0) {
+    // okay, finally!
+    return 0;
+  }
+
+  // At this point, we're going to have to throw out objects that will be
+  // missed.  Too bad!
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cache, NULL, NULL);
+  smartlist_sort(objects, compare_by_staleness_);
+  int n_marked = 0;
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_removal(ent);
+    if (++n_marked >= n_to_remove)
+      break;
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+
+  consensus_cache_delete_pending(cache, 1);
+
+  if (consensus_cache_may_overallocate(cache)) {
+    /* If we're allowed to throw extra files into the cache, let's do so
+     * rather getting upset.
+     */
+    return 0;
+  }
+
+  if (BUG(n_marked < n_to_remove))
+    return -1;
+  else
+    return 0;
+}
+
+/**
+ * Set consensus cache flags on the objects in this consdiffmgr.
+ */
+static void
+consdiffmgr_set_cache_flags(void)
+{
+  /* Right now, we just mark the consensus objects for aggressive release,
+   * so that they get mmapped for as little time as possible. */
+  smartlist_t *objects = smartlist_new();
+  consensus_cache_find_all(objects, cdm_cache_get(), LABEL_DOCTYPE,
+                           DOCTYPE_CONSENSUS);
+  SMARTLIST_FOREACH_BEGIN(objects, consensus_cache_entry_t *, ent) {
+    consensus_cache_entry_mark_for_aggressive_release(ent);
+  } SMARTLIST_FOREACH_END(ent);
+  smartlist_free(objects);
+}
+
+/**
+ * Called before shutdown: drop all storage held by the consdiffmgr.c module.
+ */
+void
+consdiffmgr_free_all(void)
+{
+  cdm_diff_t **diff, **next;
+  for (diff = HT_START(cdm_diff_ht, &cdm_diff_ht); diff; diff = next) {
+    cdm_diff_t *this = *diff;
+    next = HT_NEXT_RMV(cdm_diff_ht, &cdm_diff_ht, diff);
+    cdm_diff_free(this);
+  }
+  int i;
+  unsigned j;
+  for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) {
+    for (j = 0; j < n_consensus_compression_methods(); ++j) {
+      consensus_cache_entry_handle_free(latest_consensus[i][j]);
+    }
+  }
+  memset(latest_consensus, 0, sizeof(latest_consensus));
+  consensus_cache_free(cons_diff_cache);
+  cons_diff_cache = NULL;
+}
+
+/* =====
+   Thread workers
+   =====*/
+
+typedef struct compressed_result_t {
+  config_line_t *labels;
+  /**
+   * Output: Body of the diff, as compressed.
+   */
+  uint8_t *body;
+  /**
+   * Output: length of body_out
+   */
+  size_t bodylen;
+} compressed_result_t;
+
+/**
+ * Compress the bytestring <b>input</b> of length <b>len</b> using the
+ * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
+ *
+ * For each successful compression, set the fields in the <b>results_out</b>
+ * array in the position corresponding to the compression method. Use
+ * <b>labels_in</b> as a basis for the labels of the result.
+ *
+ * Return 0 if all compression succeeded; -1 if any failed.
+ */
+static int
+compress_multiple(compressed_result_t *results_out, int n_methods,
+                  const compress_method_t *methods,
+                  const uint8_t *input, size_t len,
+                  const config_line_t *labels_in)
+{
+  int rv = 0;
+  int i;
+  for (i = 0; i < n_methods; ++i) {
+    compress_method_t method = methods[i];
+    const char *methodname = compression_method_get_name(method);
+    char *result;
+    size_t sz;
+    if (0 == tor_compress(&result, &sz, (const char*)input, len, method)) {
+      results_out[i].body = (uint8_t*)result;
+      results_out[i].bodylen = sz;
+      results_out[i].labels = config_lines_dup(labels_in);
+      cdm_labels_prepend_sha3(&results_out[i].labels, LABEL_SHA3_DIGEST,
+                              results_out[i].body,
+                              results_out[i].bodylen);
+      config_line_prepend(&results_out[i].labels,
+                          LABEL_COMPRESSION_TYPE,
+                          methodname);
+    } else {
+      rv = -1;
+    }
+  }
+  return rv;
+}
+
+/**
+ * Given an array of <b>n</b> compressed_result_t in <b>results</b>,
+ * as produced by compress_multiple, store them all into the
+ * consdiffmgr, and store handles to them in the <b>handles_out</b>
+ * array.
+ *
+ * Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none
+ * was stored.
+ */
+static cdm_diff_status_t
+store_multiple(consensus_cache_entry_handle_t **handles_out,
+               int n,
+               const compress_method_t *methods,
+               const compressed_result_t *results,
+               const char *description)
+{
+  cdm_diff_status_t status = CDM_DIFF_ERROR;
+  consdiffmgr_ensure_space_for_files(n);
+
+  int i;
+  for (i = 0; i < n; ++i) {
+    compress_method_t method = methods[i];
+    uint8_t *body_out = results[i].body;
+    size_t bodylen_out = results[i].bodylen;
+    config_line_t *labels = results[i].labels;
+    const char *methodname = compression_method_get_name(method);
+    if (body_out && bodylen_out && labels) {
+      /* Success! Store the results */
+      log_info(LD_DIRSERV, "Adding %s, compressed with %s",
+               description, methodname);
+
+      consensus_cache_entry_t *ent =
+        consensus_cache_add(cdm_cache_get(),
+                            labels,
+                            body_out,
+                            bodylen_out);
+      if (ent == NULL) {
+        static ratelim_t cant_store_ratelim = RATELIM_INIT(5*60);
+        log_fn_ratelim(&cant_store_ratelim, LOG_WARN, LD_FS,
+                       "Unable to store object %s compressed with %s.",
+                       description, methodname);
+        continue;
+      }
+
+      status = CDM_DIFF_PRESENT;
+      handles_out[i] = consensus_cache_entry_handle_new(ent);
+      consensus_cache_entry_decref(ent);
+    }
+  }
+  return status;
+}
+
+/**
+ * An object passed to a worker thread that will try to produce a consensus
+ * diff.
+ */
+typedef struct consensus_diff_worker_job_t {
+  /**
+   * Input: The consensus to compute the diff from.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_from;
+  /**
+   * Input: The consensus to compute the diff to.  Holds a reference to the
+   * cache entry, which must not be released until the job is passed back to
+   * the main thread. The body must be mapped into memory in the main thread.
+   */
+  consensus_cache_entry_t *diff_to;
+
+  /** Output: labels and bodies */
+  compressed_result_t out[ARRAY_LENGTH(compress_diffs_with)];
+} consensus_diff_worker_job_t;
+
+/** Given a consensus_cache_entry_t, check whether it has a label claiming
+ * that it was compressed.  If so, uncompress its contents into <b>out</b> and
+ * set <b>outlen</b> to hold their size.  If not, just copy the body into
+ * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
+ * -1 on failure.
+ *
+ * In all cases, the output is nul-terminated. */
+STATIC int
+uncompress_or_copy(char **out, size_t *outlen,
+                   consensus_cache_entry_t *ent)
+{
+  const uint8_t *body;
+  size_t bodylen;
+
+  if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
+    return -1;
+
+  const char *lv_compression =
+    consensus_cache_entry_get_value(ent, LABEL_COMPRESSION_TYPE);
+  compress_method_t method = NO_METHOD;
+
+  if (lv_compression)
+    method = compression_method_get_by_name(lv_compression);
+
+  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+                        method, 1, LOG_WARN);
+}
+
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_diff_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_diff_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_diff_worker_job_t *job = work_;
+  const uint8_t *diff_from, *diff_to;
+  size_t len_from, len_to;
+  int r;
+  /* We need to have the body already mapped into RAM here.
+   */
+  r = consensus_cache_entry_get_body(job->diff_from, &diff_from, &len_from);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  r = consensus_cache_entry_get_body(job->diff_to, &diff_to, &len_to);
+  if (BUG(r < 0))
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+
+  const char *lv_to_valid_after =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_AFTER);
+  const char *lv_to_fresh_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FRESH_UNTIL);
+  const char *lv_to_valid_until =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_VALID_UNTIL);
+  const char *lv_to_signatories =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_SIGNATORIES);
+  const char *lv_from_valid_after =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_VALID_AFTER);
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_from_flavor =
+    consensus_cache_entry_get_value(job->diff_from, LABEL_FLAVOR);
+  const char *lv_to_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+
+  if (! lv_from_digest) {
+    /* This isn't a bug right now, since it can happen if you're migrating
+     * from an older version of master to a newer one.  The older ones didn't
+     * annotate their stored consensus objects with sha3-digest-as-signed.
+    */
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  /* All these values are mandatory on the input */
+  if (BUG(!lv_to_valid_after) ||
+      BUG(!lv_from_valid_after) ||
+      BUG(!lv_from_flavor) ||
+      BUG(!lv_to_flavor)) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+  /* The flavors need to match */
+  if (BUG(strcmp(lv_from_flavor, lv_to_flavor))) {
+    return WQ_RPL_REPLY; // LCOV_EXCL_LINE
+  }
+
+  char *consensus_diff;
+  {
+    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    size_t diff_from_nt_len, diff_to_nt_len;
+
+    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
+                           job->diff_from) < 0) {
+      return WQ_RPL_REPLY;
+    }
+    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
+                           job->diff_to) < 0) {
+      tor_free(diff_from_nt);
+      return WQ_RPL_REPLY;
+    }
+    tor_assert(diff_from_nt);
+    tor_assert(diff_to_nt);
+
+    // XXXX ugh; this is going to calculate the SHA3 of both its
+    // XXXX inputs again, even though we already have that. Maybe it's time
+    // XXXX to change the API here?
+    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
+    tor_free(diff_from_nt);
+    tor_free(diff_to_nt);
+  }
+  if (!consensus_diff) {
+    /* Couldn't generate consensus; we'll leave the reply blank. */
+    return WQ_RPL_REPLY;
+  }
+
+  /* Compress the results and send the reply */
+  tor_assert(compress_diffs_with[0] == NO_METHOD);
+  size_t difflen = strlen(consensus_diff);
+  job->out[0].body = (uint8_t *) consensus_diff;
+  job->out[0].bodylen = difflen;
+
+  config_line_t *common_labels = NULL;
+  if (lv_to_valid_until)
+    config_line_prepend(&common_labels, LABEL_VALID_UNTIL, lv_to_valid_until);
+  if (lv_to_fresh_until)
+    config_line_prepend(&common_labels, LABEL_FRESH_UNTIL, lv_to_fresh_until);
+  if (lv_to_signatories)
+    config_line_prepend(&common_labels, LABEL_SIGNATORIES, lv_to_signatories);
+  cdm_labels_prepend_sha3(&common_labels,
+                          LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+  config_line_prepend(&common_labels, LABEL_FROM_VALID_AFTER,
+                      lv_from_valid_after);
+  config_line_prepend(&common_labels, LABEL_VALID_AFTER,
+                      lv_to_valid_after);
+  config_line_prepend(&common_labels, LABEL_FLAVOR, lv_from_flavor);
+  config_line_prepend(&common_labels, LABEL_FROM_SHA3_DIGEST,
+                      lv_from_digest);
+  config_line_prepend(&common_labels, LABEL_TARGET_SHA3_DIGEST,
+                      lv_to_digest);
+  config_line_prepend(&common_labels, LABEL_DOCTYPE,
+                      DOCTYPE_CONSENSUS_DIFF);
+
+  job->out[0].labels = config_lines_dup(common_labels);
+  cdm_labels_prepend_sha3(&job->out[0].labels,
+                          LABEL_SHA3_DIGEST,
+                          job->out[0].body,
+                          job->out[0].bodylen);
+
+  compress_multiple(job->out+1,
+                    n_diff_compression_methods()-1,
+                    compress_diffs_with+1,
+                    (const uint8_t*)consensus_diff, difflen, common_labels);
+
+  config_free_lines(common_labels);
+  return WQ_RPL_REPLY;
+}
+
+/**
+ * Helper: release all storage held in <b>job</b>.
+ */
+static void
+consensus_diff_worker_job_free(consensus_diff_worker_job_t *job)
+{
+  if (!job)
+    return;
+  unsigned u;
+  for (u = 0; u < n_diff_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  consensus_cache_entry_decref(job->diff_from);
+  consensus_cache_entry_decref(job->diff_to);
+  tor_free(job);
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_worker_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_diff_worker_replyfn(void *work_)
+{
+  tor_assert(in_main_thread());
+  tor_assert(work_);
+
+  consensus_diff_worker_job_t *job = work_;
+
+  const char *lv_from_digest =
+    consensus_cache_entry_get_value(job->diff_from,
+                                    LABEL_SHA3_DIGEST_AS_SIGNED);
+  const char *lv_to_digest =
+    consensus_cache_entry_get_value(job->diff_to,
+                                    LABEL_SHA3_DIGEST_UNCOMPRESSED);
+  const char *lv_flavor =
+    consensus_cache_entry_get_value(job->diff_to, LABEL_FLAVOR);
+  if (BUG(lv_from_digest == NULL))
+    lv_from_digest = "???"; // LCOV_EXCL_LINE
+  if (BUG(lv_to_digest == NULL))
+    lv_to_digest = "???"; // LCOV_EXCL_LINE
+
+  uint8_t from_sha3[DIGEST256_LEN];
+  uint8_t to_sha3[DIGEST256_LEN];
+  int flav = -1;
+  int cache = 1;
+  if (BUG(cdm_entry_get_sha3_value(from_sha3, job->diff_from,
+                                   LABEL_SHA3_DIGEST_AS_SIGNED) < 0))
+    cache = 0;
+  if (BUG(cdm_entry_get_sha3_value(to_sha3, job->diff_to,
+                                   LABEL_SHA3_DIGEST_UNCOMPRESSED) < 0))
+    cache = 0;
+  if (BUG(lv_flavor == NULL)) {
+    cache = 0;
+  } else if ((flav = networkstatus_parse_flavor_name(lv_flavor)) < 0) {
+    cache = 0;
+  }
+
+  consensus_cache_entry_handle_t *handles[ARRAY_LENGTH(compress_diffs_with)];
+  memset(handles, 0, sizeof(handles));
+
+  char description[128];
+  tor_snprintf(description, sizeof(description),
+               "consensus diff from %s to %s",
+               lv_from_digest, lv_to_digest);
+
+  int status = store_multiple(handles,
+                              n_diff_compression_methods(),
+                              compress_diffs_with,
+                              job->out,
+                              description);
+
+  if (status != CDM_DIFF_PRESENT) {
+    /* Failure! Nothing to do but complain */
+    log_warn(LD_DIRSERV,
+             "Worker was unable to compute consensus diff "
+             "from %s to %s", lv_from_digest, lv_to_digest);
+    /* Cache this error so we don't try to compute this one again. */
+    status = CDM_DIFF_ERROR;
+  }
+
+  unsigned u;
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    compress_method_t method = compress_diffs_with[u];
+    if (cache) {
+      consensus_cache_entry_handle_t *h = handles[u];
+      int this_status = status;
+      if (h == NULL) {
+        this_status = CDM_DIFF_ERROR;
+      }
+      tor_assert_nonfatal(h != NULL || this_status == CDM_DIFF_ERROR);
+      cdm_diff_ht_set_status(flav, from_sha3, to_sha3, method, this_status, h);
+    } else {
+      consensus_cache_entry_handle_free(handles[u]);
+    }
+  }
+
+  consensus_diff_worker_job_free(job);
+}
+
+/**
+ * Queue the job of computing the diff from <b>diff_from</b> to <b>diff_to</b>
+ * in a worker thread.
+ */
+static int
+consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
+                               consensus_cache_entry_t *diff_to)
+{
+  tor_assert(in_main_thread());
+
+  consensus_cache_entry_incref(diff_from);
+  consensus_cache_entry_incref(diff_to);
+
+  consensus_diff_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->diff_from = diff_from;
+  job->diff_to = diff_to;
+
+  /* Make sure body is mapped. */
+  const uint8_t *body;
+  size_t bodylen;
+  int r1 = consensus_cache_entry_get_body(diff_from, &body, &bodylen);
+  int r2 = consensus_cache_entry_get_body(diff_to, &body, &bodylen);
+  if (r1 < 0 || r2 < 0)
+    goto err;
+
+  workqueue_entry_t *work;
+  work = cpuworker_queue_work(WQ_PRI_LOW,
+                              consensus_diff_worker_threadfn,
+                              consensus_diff_worker_replyfn,
+                              job);
+  if (!work)
+    goto err;
+
+  return 0;
+ err:
+  consensus_diff_worker_job_free(job); // includes decrefs.
+  return -1;
+}
+
+/**
+ * Holds requests and replies for consensus_compress_workers.
+ */
+typedef struct consensus_compress_worker_job_t {
+  char *consensus;
+  size_t consensus_len;
+  consensus_flavor_t flavor;
+  config_line_t *labels_in;
+  compressed_result_t out[ARRAY_LENGTH(compress_consensus_with)];
+} consensus_compress_worker_job_t;
+
+/**
+ * Free all resources held in <b>job</b>
+ */
+static void
+consensus_compress_worker_job_free(consensus_compress_worker_job_t *job)
+{
+  if (!job)
+    return;
+  tor_free(job->consensus);
+  config_free_lines(job->labels_in);
+  unsigned u;
+  for (u = 0; u < n_consensus_compression_methods(); ++u) {
+    config_free_lines(job->out[u].labels);
+    tor_free(job->out[u].body);
+  }
+  tor_free(job);
+}
+/**
+ * Worker function. This function runs inside a worker thread and receives
+ * a consensus_compress_worker_job_t as its input.
+ */
+static workqueue_reply_t
+consensus_compress_worker_threadfn(void *state_, void *work_)
+{
+  (void)state_;
+  consensus_compress_worker_job_t *job = work_;
+  consensus_flavor_t flavor = job->flavor;
+  const char *consensus = job->consensus;
+  size_t bodylen = job->consensus_len;
+
+  config_line_t *labels = config_lines_dup(job->labels_in);
+  const char *flavname = networkstatus_get_flavor_name(flavor);
+
+  cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_UNCOMPRESSED,
+                          (const uint8_t *)consensus, bodylen);
+  {
+    const char *start, *end;
+    if (router_get_networkstatus_v3_signed_boundaries(consensus,
+                                                        &start, &end) < 0) {
+      start = consensus;
+      end = consensus+bodylen;
+    }
+    cdm_labels_prepend_sha3(&labels, LABEL_SHA3_DIGEST_AS_SIGNED,
+                            (const uint8_t *)start,
+                            end - start);
+  }
+  config_line_prepend(&labels, LABEL_FLAVOR, flavname);
+  config_line_prepend(&labels, LABEL_DOCTYPE, DOCTYPE_CONSENSUS);
+
+  compress_multiple(job->out,
+                    n_consensus_compression_methods(),
+                    compress_consensus_with,
+                    (const uint8_t*)consensus, bodylen, labels);
+  config_free_lines(labels);
+  return WQ_RPL_REPLY;
+}
+
+/**
+ * Worker function: This function runs in the main thread, and receives
+ * a consensus_diff_compress_job_t that the worker thread has already
+ * processed.
+ */
+static void
+consensus_compress_worker_replyfn(void *work_)
+{
+  consensus_compress_worker_job_t *job = work_;
+
+  consensus_cache_entry_handle_t *handles[
+                               ARRAY_LENGTH(compress_consensus_with)];
+  memset(handles, 0, sizeof(handles));
+
+  store_multiple(handles,
+                 n_consensus_compression_methods(),
+                 compress_consensus_with,
+                 job->out,
+                 "consensus");
+  cdm_cache_dirty = 1;
+
+  unsigned u;
+  consensus_flavor_t f = job->flavor;
+  tor_assert((int)f < N_CONSENSUS_FLAVORS);
+  for (u = 0; u < ARRAY_LENGTH(handles); ++u) {
+    if (handles[u] == NULL)
+      continue;
+    consensus_cache_entry_handle_free(latest_consensus[f][u]);
+    latest_consensus[f][u] = handles[u];
+  }
+
+  consensus_compress_worker_job_free(job);
+}
+
+/**
+ * If true, we compress in worker threads.
+ */
+static int background_compression = 0;
+
+/**
+ * Queue a job to compress <b>consensus</b> and store its compressed
+ * text in the cache.
+ */
+static int
+consensus_queue_compression_work(const char *consensus,
+                                 const networkstatus_t *as_parsed)
+{
+  tor_assert(consensus);
+  tor_assert(as_parsed);
+
+  consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
+  job->consensus = tor_strdup(consensus);
+  job->consensus_len = strlen(consensus);
+  job->flavor = as_parsed->flavor;
+
+  char va_str[ISO_TIME_LEN+1];
+  char vu_str[ISO_TIME_LEN+1];
+  char fu_str[ISO_TIME_LEN+1];
+  format_iso_time_nospace(va_str, as_parsed->valid_after);
+  format_iso_time_nospace(fu_str, as_parsed->fresh_until);
+  format_iso_time_nospace(vu_str, as_parsed->valid_until);
+  config_line_append(&job->labels_in, LABEL_VALID_AFTER, va_str);
+  config_line_append(&job->labels_in, LABEL_FRESH_UNTIL, fu_str);
+  config_line_append(&job->labels_in, LABEL_VALID_UNTIL, vu_str);
+  if (as_parsed->voters) {
+    smartlist_t *hexvoters = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(as_parsed->voters,
+                            networkstatus_voter_info_t *, vi) {
+      if (smartlist_len(vi->sigs) == 0)
+        continue; // didn't sign.
+      char d[HEX_DIGEST_LEN+1];
+      base16_encode(d, sizeof(d), vi->identity_digest, DIGEST_LEN);
+      smartlist_add_strdup(hexvoters, d);
+    } SMARTLIST_FOREACH_END(vi);
+    char *signers = smartlist_join_strings(hexvoters, ",", 0, NULL);
+    config_line_prepend(&job->labels_in, LABEL_SIGNATORIES, signers);
+    tor_free(signers);
+    SMARTLIST_FOREACH(hexvoters, char *, cp, tor_free(cp));
+    smartlist_free(hexvoters);
+  }
+
+  if (background_compression) {
+    workqueue_entry_t *work;
+    work = cpuworker_queue_work(WQ_PRI_LOW,
+                                consensus_compress_worker_threadfn,
+                                consensus_compress_worker_replyfn,
+                                job);
+    if (!work) {
+      consensus_compress_worker_job_free(job);
+      return -1;
+    }
+
+    return 0;
+  } else {
+    consensus_compress_worker_threadfn(NULL, job);
+    consensus_compress_worker_replyfn(job);
+    return 0;
+  }
+}
+
+/**
+ * Tell the consdiffmgr backend to compress consensuses in worker threads.
+ */
+void
+consdiffmgr_enable_background_compression(void)
+{
+  // This isn't the default behavior because it would break unit tests.
+  background_compression = 1;
+}
+
+/** Read the set of voters from the cached object <b>ent</b> into
+ * <b>out</b>, as a list of hex-encoded digests. Return 0 on success,
+ * -1 if no signatories were recorded. */
+int
+consensus_cache_entry_get_voter_id_digests(const consensus_cache_entry_t *ent,
+                                           smartlist_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_SIGNATORIES);
+  if (s == NULL)
+    return -1;
+  smartlist_split_string(out, s, ",", SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE, 0);
+  return 0;
+}
+
+/** Read the fresh-until time of cached object <b>ent</b> into *<b>out</b>
+ * and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_fresh_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_FRESH_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid until timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_until(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_UNTIL);
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
+/** Read the valid after timestamp from the cached object <b>ent</b> into
+ * *<b>out</b> and return 0, or return -1 if no such time was recorded. */
+int
+consensus_cache_entry_get_valid_after(const consensus_cache_entry_t *ent,
+                                      time_t *out)
+{
+  tor_assert(ent);
+  tor_assert(out);
+
+  const char *s;
+  s = consensus_cache_entry_get_value(ent, LABEL_VALID_AFTER);
+
+  if (s == NULL || parse_iso_time_nospace(s, out) < 0)
+    return -1;
+  else
+    return 0;
+}
+
diff --git a/src/or/consdiffmgr.h b/src/or/consdiffmgr.h
new file mode 100644
index 0000000000..079f9fe2d2
--- /dev/null
+++ b/src/or/consdiffmgr.h
@@ -0,0 +1,74 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CONSDIFFMGR_H
+#define TOR_CONSDIFFMGR_H
+
+/**
+ * Possible outcomes from trying to look up a given consensus diff.
+ */
+typedef enum consdiff_status_t {
+  CONSDIFF_AVAILABLE,
+  CONSDIFF_NOT_FOUND,
+  CONSDIFF_IN_PROGRESS,
+} consdiff_status_t;
+
+typedef struct consdiff_cfg_t {
+  int32_t cache_max_num;
+} consdiff_cfg_t;
+
+struct consensus_cache_entry_t; // from conscache.h
+
+int consdiffmgr_add_consensus(const char *consensus,
+                              const networkstatus_t *as_parsed);
+
+consdiff_status_t consdiffmgr_find_consensus(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           compress_method_t method);
+
+consdiff_status_t consdiffmgr_find_diff_from(
+                           struct consensus_cache_entry_t **entry_out,
+                           consensus_flavor_t flavor,
+                           int digest_type,
+                           const uint8_t *digest,
+                           size_t digestlen,
+                           compress_method_t method);
+
+int consensus_cache_entry_get_voter_id_digests(
+                                  const struct consensus_cache_entry_t *ent,
+                                  smartlist_t *out);
+int consensus_cache_entry_get_fresh_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_until(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+int consensus_cache_entry_get_valid_after(
+                                  const struct consensus_cache_entry_t *ent,
+                                  time_t *out);
+
+void consdiffmgr_rescan(void);
+int consdiffmgr_cleanup(void);
+void consdiffmgr_enable_background_compression(void);
+void consdiffmgr_configure(const consdiff_cfg_t *cfg);
+struct sandbox_cfg_elem;
+int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);
+void consdiffmgr_free_all(void);
+int consdiffmgr_validate(void);
+
+#ifdef CONSDIFFMGR_PRIVATE
+STATIC unsigned n_diff_compression_methods(void);
+STATIC unsigned n_consensus_compression_methods(void);
+STATIC consensus_cache_t *cdm_cache_get(void);
+STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
+                          consensus_flavor_t flavor, time_t valid_after);
+STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
+                                    consensus_cache_entry_t *ent,
+                                    const char *label);
+STATIC int uncompress_or_copy(char **out, size_t *outlen,
+                              consensus_cache_entry_t *ent);
+#endif
+
+#endif
+
diff --git a/src/or/control.c b/src/or/control.c
index 03d9fcee2a..1837d49025 100644
--- a/src/or/control.c
+++ b/src/or/control.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,6 +36,7 @@
 
 #include "or.h"
 #include "addressmap.h"
+#include "bridges.h"
 #include "buffers.h"
 #include "channel.h"
 #include "channeltls.h"
@@ -57,6 +58,7 @@
 #include "entrynodes.h"
 #include "geoip.h"
 #include "hibernate.h"
+#include "hs_common.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -69,6 +71,7 @@
 #include "router.h"
 #include "routerlist.h"
 #include "routerparse.h"
+#include "shared_random.h"
 
 #ifndef _WIN32
 #include <pwd.h>
@@ -942,7 +945,7 @@ control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
       ++body;
   }
 
-  smartlist_add(entries, tor_strdup(""));
+  smartlist_add_strdup(entries, "");
   config = smartlist_join_strings(entries, "\n", 0, NULL);
   SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
   smartlist_free(entries);
@@ -1459,8 +1462,10 @@ handle_control_saveconf(control_connection_t *conn, uint32_t len,
                         const char *body)
 {
   (void) len;
-  (void) body;
-  if (options_save_current()<0) {
+
+  int force = !strcmpstart(body, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
     connection_write_str_to_buf(
       "551 Unable to write configuration to disk.\r\n", conn);
   } else {
@@ -1674,6 +1679,8 @@ getinfo_helper_misc(control_connection_t *conn, const char *question,
       *answer = tor_strdup(a);
   } else if (!strcmp(question, "config-text")) {
     *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
   } else if (!strcmp(question, "info/names")) {
     *answer = list_getinfo_options();
   } else if (!strcmp(question, "dormant")) {
@@ -1870,7 +1877,7 @@ getinfo_helper_listeners(control_connection_t *control_conn,
 
 /** Implementation helper for GETINFO: knows the answers for questions about
  * directory information. */
-static int
+STATIC int
 getinfo_helper_dir(control_connection_t *control_conn,
                    const char *question, char **answer,
                    const char **errmsg)
@@ -2028,7 +2035,7 @@ getinfo_helper_dir(control_connection_t *control_conn,
   } else if (!strcmpstart(question, "dir/status/")) {
     *answer = tor_strdup("");
   } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (directory_caches_dir_info(get_options())) {
+    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
       const cached_dir_t *consensus = dirserv_get_consensus("ns");
       if (consensus)
         *answer = tor_strdup(consensus->dir);
@@ -2044,6 +2051,12 @@ getinfo_helper_dir(control_connection_t *control_conn,
       }
     }
   } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
     routerlist_t *routerlist = router_get_routerlist();
     if (!routerlist || !routerlist->routers ||
         list_server_status_v1(routerlist->routers, answer, 1) < 0) {
@@ -2539,7 +2552,7 @@ circuit_describe_status_for_controller(origin_circuit_t *circ)
 
   if (circ->rend_data != NULL) {
     smartlist_add_asprintf(descparts, "REND_QUERY=%s",
-                 circ->rend_data->onion_address);
+                           rend_data_get_address(circ->rend_data));
   }
 
   {
@@ -2594,6 +2607,8 @@ getinfo_helper_events(control_connection_t *control_conn,
 
       if (circ->base_.state == CIRCUIT_STATE_OPEN)
         state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
       else if (circ->cpath)
         state = "EXTENDED";
       else
@@ -2819,12 +2834,13 @@ getinfo_helper_events(control_connection_t *control_conn,
 
 /** Implementation helper for GETINFO: knows how to enumerate hidden services
  * created via the control port. */
-static int
+STATIC int
 getinfo_helper_onions(control_connection_t *control_conn,
                       const char *question, char **answer,
                       const char **errmsg)
 {
   smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
 
   if (control_conn && !strcmp(question, "onions/current")) {
     onion_list = control_conn->ephemeral_onion_services;
@@ -2834,13 +2850,13 @@ getinfo_helper_onions(control_connection_t *control_conn,
     return 0;
   }
   if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (errmsg) {
-        *errmsg = "No onion services of the specified type.";
+    if (answer) {
+      *answer = tor_strdup("");
     }
-    return -1;
-  }
-  if (answer) {
+  } else {
+    if (answer) {
       *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
   }
 
   return 0;
@@ -2866,6 +2882,26 @@ getinfo_helper_liveness(control_connection_t *control_conn,
   return 0;
 }
 
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
 /** Callback function for GETINFO: on a given control connection, try to
  * answer the question <b>q</b> and store the newly-allocated answer in
  * *<b>a</b>. If an internal error occurs, return -1 and optionally set
@@ -2899,6 +2935,8 @@ static const getinfo_item_t getinfo_items[] = {
   ITEM("config-defaults-file", misc, "Current location of the defaults file."),
   ITEM("config-text", misc,
        "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
   ITEM("accounting/bytes", accounting,
        "Number of bytes read/written so far in the accounting interval."),
   ITEM("accounting/bytes-left", accounting,
@@ -3058,6 +3096,8 @@ static const getinfo_item_t getinfo_items[] = {
        "Onion services owned by the current control connection."),
   ITEM("onions/detached", onions,
        "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
   { NULL, NULL, NULL, 0 }
 };
 
@@ -3139,7 +3179,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
     if (!ans) {
       smartlist_add(unrecognized, (char*)q);
     } else {
-      smartlist_add(answers, tor_strdup(q));
+      smartlist_add_strdup(answers, q);
       smartlist_add(answers, ans);
     }
   } SMARTLIST_FOREACH_END(q);
@@ -3350,7 +3390,8 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
   SMARTLIST_FOREACH(nodes, const node_t *, node,
   {
     extend_info_t *info = extend_info_from_node(node, first_node);
-    if (first_node && !info) {
+    if (!info) {
+      tor_assert_nonfatal(first_node);
       log_warn(LD_CONTROL,
                "controller tried to connect to a node that doesn't have any "
                "addresses that are allowed by the firewall configuration; "
@@ -3358,10 +3399,6 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
       circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
       connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
       goto done;
-    } else {
-      /* True, since node_has_descriptor(node) == true and we are extending
-       * to the node's primary address */
-      tor_assert(info);
     }
     circuit_append_new_exit(circ, info);
     extend_info_free(info);
@@ -3377,7 +3414,8 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
       goto done;
     }
   } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN) {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
       int err_reason = 0;
       circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
       if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
@@ -3519,24 +3557,9 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len,
   }
   /* Is this a single hop circuit? */
   if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    const node_t *node = NULL;
-    char *exit_digest = NULL;
-    if (circ->build_state &&
-        circ->build_state->chosen_exit &&
-        !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) {
-      exit_digest = circ->build_state->chosen_exit->identity_digest;
-      node = node_get_by_id(exit_digest);
-    }
-    /* Do both the client and relay allow one-hop exit circuits? */
-    if (!node ||
-        !node_allows_single_hop_exits(node) ||
-        !get_options()->AllowSingleHopCircuits) {
-      connection_write_str_to_buf(
-      "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-      return 0;
-    }
-    tor_assert(exit_digest);
-    ap_conn->chosen_exit_name = tor_strdup(hex_str(exit_digest, DIGEST_LEN));
+    connection_write_str_to_buf(
+               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
+    return 0;
   }
 
   if (circ && hop>0) {
@@ -4039,6 +4062,14 @@ handle_control_dropguards(control_connection_t *conn,
   smartlist_split_string(args, body, " ",
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
 
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
   if (smartlist_len(args)) {
     connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
   } else {
@@ -4084,7 +4115,7 @@ handle_control_hsfetch(control_connection_t *conn, uint32_t len,
      * of the id. */
     desc_id = digest;
   } else {
-    connection_printf_to_buf(conn, "513 Unrecognized \"%s\"\r\n",
+    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
                              arg1);
     goto done;
   }
@@ -6053,9 +6084,9 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses,
     return 0;
 
   strs = smartlist_new();
-  smartlist_add(strs, tor_strdup("650+"));
-  smartlist_add(strs, tor_strdup(event_string));
-  smartlist_add(strs, tor_strdup("\r\n"));
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
   SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
     {
       s = networkstatus_getinfo_helper_single(rs);
@@ -6483,7 +6514,7 @@ monitor_owning_controller_process(const char *process_spec)
             msg);
     owning_controller_process_spec = NULL;
     tor_cleanup();
-    exit(0);
+    exit(1);
   }
 }
 
@@ -6686,8 +6717,8 @@ control_event_bootstrap(bootstrap_status_t status, int progress)
  * is the connection that caused this problem.
  */
 MOCK_IMPL(void,
-          control_event_bootstrap_problem, (const char *warn, int reason,
-                                            or_connection_t *or_conn))
+control_event_bootstrap_problem, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
 {
   int status = bootstrap_percent;
   const char *tag = "", *summary = "";
@@ -6864,8 +6895,10 @@ control_event_hs_descriptor_requested(const rend_data_t *rend_query,
 
   send_control_event(EVENT_HS_DESC,
                      "650 HS_DESC REQUESTED %s %s %s %s\r\n",
-                     rend_hsaddress_str_or_unknown(rend_query->onion_address),
-                     rend_auth_type_to_string(rend_query->auth_type),
+                     rend_hsaddress_str_or_unknown(
+                          rend_data_get_address(rend_query)),
+                     rend_auth_type_to_string(
+                          TO_REND_DATA_V2(rend_query)->auth_type),
                      node_describe_longname_by_id(id_digest),
                      desc_id_base32);
 }
@@ -6881,19 +6914,25 @@ get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
 {
   int replica;
   const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
 
   /* Possible if the fetch was done using a descriptor ID. This means that
    * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
-    desc_id = rend_data->desc_id_fetch;
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
     goto end;
   }
 
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
   /* OK, we have an onion address so now let's find which descriptor ID
    * is the one associated with the HSDir fingerprint. */
   for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
        replica++) {
-    const char *digest = rend_data->descriptor_id[replica];
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
 
     SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
       if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
@@ -6978,10 +7017,9 @@ control_event_hs_descriptor_receive_end(const char *action,
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
   const char *desc_id = NULL;
 
-  if (!action || !id_digest || !rend_data || !onion_address) {
-    log_warn(LD_BUG, "Called with action==%p, id_digest==%p, "
-             "rend_data==%p, onion_address==%p", action, id_digest,
-             rend_data, onion_address);
+  if (!action || !rend_data || !onion_address) {
+    log_warn(LD_BUG, "Called with action==%p, rend_data==%p, "
+                     "onion_address==%p", action, rend_data, onion_address);
     return;
   }
 
@@ -7002,8 +7040,10 @@ control_event_hs_descriptor_receive_end(const char *action,
                      "650 HS_DESC %s %s %s %s%s%s\r\n",
                      action,
                      rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(rend_data->auth_type),
-                     node_describe_longname_by_id(id_digest),
+                     rend_auth_type_to_string(
+                          TO_REND_DATA_V2(rend_data)->auth_type),
+                     id_digest ?
+                        node_describe_longname_by_id(id_digest) : "UNKNOWN",
                      desc_id_field ? desc_id_field : "",
                      reason_field ? reason_field : "");
 
@@ -7083,28 +7123,30 @@ control_event_hs_descriptor_uploaded(const char *id_digest,
                                          id_digest, NULL);
 }
 
-/** Send HS_DESC event to inform controller that query <b>rend_query</b>
- * failed to retrieve hidden service descriptor identified by
- * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON=
- * field.
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
  */
 void
 control_event_hs_descriptor_failed(const rend_data_t *rend_data,
                                    const char *id_digest,
                                    const char *reason)
 {
-  if (!rend_data || !id_digest) {
-    log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p",
-             rend_data, id_digest);
+  if (!rend_data) {
+    log_warn(LD_BUG, "Called with rend_data==%p", rend_data);
     return;
   }
   control_event_hs_descriptor_receive_end("FAILED",
-                                          rend_data->onion_address,
+                                          rend_data_get_address(rend_data),
                                           rend_data, id_digest, reason);
 }
 
-/** send HS_DESC_CONTENT event after completion of a successful fetch from
- * hs directory. */
+/** Send HS_DESC_CONTENT event after completion of a successful fetch from hs
+ * directory. If <b>hsdir_id_digest</b> is NULL, it is replaced by "UNKNOWN".
+ * If <b>content</b> is NULL, it is replaced by an empty string. The
+ * <b>onion_address</b> or <b>desc_id</b> set to NULL will no trigger the
+ * control event. */
 void
 control_event_hs_descriptor_content(const char *onion_address,
                                     const char *desc_id,
@@ -7114,9 +7156,9 @@ control_event_hs_descriptor_content(const char *onion_address,
   static const char *event_name = "HS_DESC_CONTENT";
   char *esc_content = NULL;
 
-  if (!onion_address || !desc_id || !hsdir_id_digest) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, "
-             "hsdir_id_digest==%p", onion_address, desc_id, hsdir_id_digest);
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
     return;
   }
 
@@ -7131,7 +7173,9 @@ control_event_hs_descriptor_content(const char *onion_address,
                      event_name,
                      rend_hsaddress_str_or_unknown(onion_address),
                      desc_id,
-                     node_describe_longname_by_id(hsdir_id_digest),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
                      esc_content);
   tor_free(esc_content);
 }
diff --git a/src/or/control.h b/src/or/control.h
index 6330c85571..41a194bfcb 100644
--- a/src/or/control.h
+++ b/src/or/control.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -262,6 +262,11 @@ STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk,
 STATIC rend_authorized_client_t *
 add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
 
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
 STATIC void getinfo_helper_downloads_networkstatus(
     const char *flavor,
     download_status_t **dl_to_emit,
@@ -285,6 +290,10 @@ STATIC int getinfo_helper_downloads(
     control_connection_t *control_conn,
     const char *question, char **answer,
     const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
 
 #endif
 
diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c
index fd6de6ea7c..f5fff2b331 100644
--- a/src/or/cpuworker.c
+++ b/src/or/cpuworker.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,8 +11,11 @@
  * The multithreading backend for this module is in workqueue.c; this module
  * specializes workqueue.c.
  *
- * Right now, we only use this for processing onionskins, and invoke it mostly
- * from onion.c.
+ * Right now, we use this infrastructure
+ *  <ul><li>for processing onionskins in onion.c
+ *      <li>for compressing consensuses in consdiffmgr.c,
+ *      <li>and for calculating diffs and compressing them in consdiffmgr.c.
+ *  </ul>
  **/
 #include "or.h"
 #include "channel.h"
@@ -89,7 +92,14 @@ cpu_init(void)
     event_add(reply_event, NULL);
   }
   if (!threadpool) {
-    threadpool = threadpool_new(get_num_cpus(get_options()),
+    /*
+      In our threadpool implementation, half the threads are permissive and
+      half are strict (when it comes to running lower-priority tasks). So we
+      always make sure we have at least two threads, so that there will be at
+      least one thread of each kind.
+    */
+    const int n_threads = get_num_cpus(get_options()) + 1;
+    threadpool = threadpool_new(n_threads,
                                 replyqueue,
                                 worker_state_new,
                                 worker_state_free,
@@ -475,10 +485,26 @@ queue_pending_tasks(void)
       return;
 
     if (assign_onionskin_to_cpuworker(circ, onionskin))
-      log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring.");
+      log_info(LD_OR,"assign_to_cpuworker failed. Ignoring.");
   }
 }
 
+/** DOCDOC */
+MOCK_IMPL(workqueue_entry_t *,
+cpuworker_queue_work,(workqueue_priority_t priority,
+                      workqueue_reply_t (*fn)(void *, void *),
+                      void (*reply_fn)(void *),
+                      void *arg))
+{
+  tor_assert(threadpool);
+
+  return threadpool_queue_work_priority(threadpool,
+                                        priority,
+                                        fn,
+                                        reply_fn,
+                                        arg);
+}
+
 /** Try to tell a cpuworker to perform the public key operations necessary to
  * respond to <b>onionskin</b> for the circuit <b>circ</b>.
  *
@@ -531,7 +557,8 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ,
   memwipe(&req, 0, sizeof(req));
 
   ++total_pending_tasks;
-  queue_entry = threadpool_queue_work(threadpool,
+  queue_entry = threadpool_queue_work_priority(threadpool,
+                                      WQ_PRI_HIGH,
                                       cpuworker_onion_handshake_threadfn,
                                       cpuworker_onion_handshake_replyfn,
                                       job);
diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h
index 62cf0eb164..320de9532f 100644
--- a/src/or/cpuworker.h
+++ b/src/or/cpuworker.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,14 @@
 
 void cpu_init(void);
 void cpuworkers_rotate_keyinfo(void);
+struct workqueue_entry_s;
+enum workqueue_reply_t;
+enum workqueue_priority_t;
+MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, (
+                    enum workqueue_priority_t priority,
+                    enum workqueue_reply_t (*fn)(void *, void *),
+                    void (*reply_fn)(void *),
+                    void *arg));
 
 struct create_cell_t;
 int assign_onionskin_to_cpuworker(or_circuit_t *circ,
diff --git a/src/or/dircollate.c b/src/or/dircollate.c
index 033a7afe0f..172364c5f5 100644
--- a/src/or/dircollate.c
+++ b/src/or/dircollate.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/dircollate.h b/src/or/dircollate.h
index 358c730cbb..52214282b9 100644
--- a/src/or/dircollate.h
+++ b/src/or/dircollate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/directory.c b/src/or/directory.c
index f285e4c6ed..f52354356d 100644
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@ -1,21 +1,31 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define DIRECTORY_PRIVATE
+
 #include "or.h"
 #include "backtrace.h"
+#include "bridges.h"
 #include "buffers.h"
 #include "circuitbuild.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
+#include "conscache.h"
+#include "consdiff.h"
+#include "consdiffmgr.h"
 #include "control.h"
+#include "compat.h"
+#define DIRECTORY_PRIVATE
 #include "directory.h"
 #include "dirserv.h"
 #include "dirvote.h"
 #include "entrynodes.h"
 #include "geoip.h"
+#include "hs_cache.h"
+#include "hs_common.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
@@ -33,16 +43,45 @@
 #include "shared_random.h"
 
 #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
-#ifndef OPENBSD
+#if !defined(OpenBSD)
 #include <malloc.h>
 #endif
 #endif
 
 /**
  * \file directory.c
- * \brief Code to send and fetch directories and router
- * descriptors via HTTP.  Directories use dirserv.c to generate the
- * results; clients use routers.c to parse them.
+ * \brief Code to send and fetch information from directory authorities and
+ * caches via HTTP.
+ *
+ * Directory caches and authorities use dirserv.c to generate the results of a
+ * query and stream them to the connection; clients use routerparse.c to parse
+ * them.
+ *
+ * Every directory request has a dir_connection_t on the client side and on
+ * the server side.  In most cases, the dir_connection_t object is a linked
+ * connection, tunneled through an edge_connection_t so that it can be a
+ * stream on the Tor network.  The only non-tunneled connections are those
+ * that are used to upload material (descriptors and votes) to authorities.
+ * Among tunneled connections, some use one-hop circuits, and others use
+ * multi-hop circuits for anonymity.
+ *
+ * Directory requests are launched by calling
+ * directory_initiate_request(). This
+ * launch the connection, will construct an HTTP request with
+ * directory_send_command(), send the and wait for a response.  The client
+ * later handles the response with connection_dir_client_reached_eof(),
+ * which passes the information received to another part of Tor.
+ *
+ * On the server side, requests are read in directory_handle_command(),
+ * which dispatches first on the request type (GET or POST), and then on
+ * the URL requested. GET requests are processed with a table-based
+ * dispatcher in url_table[].  The process of handling larger GET requests
+ * is complicated because we need to avoid allocating a copy of all the
+ * data to be sent to the client in one huge buffer.  Instead, we spool the
+ * data into the buffer using logic in connection_dirserv_flushed_some() in
+ * dirserv.c.  (TODO: If we extended buf.c to have a zero-copy
+ * reference-based buffer type, we could remove most of that code, at the
+ * cost of a bit more reference counting.)
  **/
 
 /* In-points to directory.c:
@@ -62,10 +101,8 @@
  *   connection_finished_connecting() in connection.c
  */
 static void directory_send_command(dir_connection_t *conn,
-                             int purpose, int direct, const char *resource,
-                             const char *payload, size_t payload_len,
-                             time_t if_modified_since);
-static int directory_handle_command(dir_connection_t *conn);
+                                   int direct,
+                                   const directory_request_t *request);
 static int body_is_plausible(const char *body, size_t body_len, int purpose);
 static char *http_get_header(const char *headers, const char *which);
 static void http_set_address_origin(const char *headers, connection_t *conn);
@@ -80,21 +117,10 @@ static void dir_routerdesc_download_failed(smartlist_t *failed,
                                            int was_extrainfo,
                                            int was_descriptor_digests);
 static void dir_microdesc_download_failed(smartlist_t *failed,
-                                          int status_code);
-static int client_likes_consensus(networkstatus_t *v, const char *want_url);
-
-static void directory_initiate_command_rend(
-                                          const tor_addr_port_t *or_addr_port,
-                                          const tor_addr_port_t *dir_addr_port,
-                                          const char *digest,
-                                          uint8_t dir_purpose,
-                                          uint8_t router_purpose,
-                                          dir_indirection_t indirection,
-                                          const char *resource,
-                                          const char *payload,
-                                          size_t payload_len,
-                                          time_t if_modified_since,
-                                          const rend_data_t *rend_query);
+                                          int status_code,
+                                          const char *dir_id);
+static int client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                                  const char *want_url);
 
 static void connection_dir_close_consensus_fetches(
                    dir_connection_t *except_this_one, const char *resource);
@@ -106,6 +132,7 @@ static void connection_dir_close_consensus_fetches(
 #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
 
 #define X_ADDRESS_HEADER "X-Your-Address-Is: "
+#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
 
 /** HTTP cache control: how long do we tell proxies they can cache each
  * kind of document we serve? */
@@ -120,29 +147,55 @@ static void connection_dir_close_consensus_fetches(
 
 /********* END VARIABLES ************/
 
-/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's
- * fetching descriptors, it's fetching them for <b>router_purpose</b>)
- * must use an anonymous connection to a directory. */
+/** Return false if the directory purpose <b>dir_purpose</b>
+ * does not require an anonymous (three-hop) connection.
+ *
+ * Return true 1) by default, 2) if all directory actions have
+ * specifically been configured to be over an anonymous connection,
+ * or 3) if the router is a bridge */
 int
-purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
+purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                        const char *resource)
 {
   if (get_options()->AllDirActionsPrivate)
     return 1;
-  if (router_purpose == ROUTER_PURPOSE_BRIDGE)
+
+  if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+    if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
+        && resource && !strcmp(resource, "authority.z")) {
+      /* We are asking a bridge for its own descriptor. That doesn't need
+         anonymity. */
+      return 0;
+    }
+    /* Assume all other bridge stuff needs anonymity. */
     return 1; /* if no circuits yet, this might break bootstrapping, but it's
                * needed to be safe. */
-  if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
-      dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
-      dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
-      dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
-      dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
-      dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
-      dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
-      dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
-      dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
-    return 0;
-  return 1;
+  }
+
+  switch (dir_purpose)
+  {
+    case DIR_PURPOSE_UPLOAD_DIR:
+    case DIR_PURPOSE_UPLOAD_VOTE:
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      return 0;
+    case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      return 1;
+    case DIR_PURPOSE_SERVER:
+    default:
+      log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
+               dir_purpose, router_purpose);
+      tor_assert_nonfatal_unreached();
+      return 1; /* Assume it needs anonymity; better safe than sorry. */
+  }
 }
 
 /** Return a newly allocated string describing <b>auth</b>. Only describes
@@ -347,7 +400,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
         log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
                  (int) extrainfo_len);
       }
-      if (purpose_needs_anonymity(dir_purpose, router_purpose)) {
+      if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) {
         indirection = DIRIND_ANONYMOUS;
       } else if (!fascist_firewall_allows_dir_server(ds,
                                                      FIREWALL_DIR_CONNECTION,
@@ -359,10 +412,14 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
       } else {
         indirection = DIRIND_DIRECT_CONN;
       }
-      directory_initiate_command_routerstatus(rs, dir_purpose,
-                                              router_purpose,
-                                              indirection,
-                                              NULL, payload, upload_len, 0);
+
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_indirection(req, indirection);
+      directory_request_set_payload(req, payload, upload_len);
+      directory_initiate_request(req);
+      directory_request_free(req);
   } SMARTLIST_FOREACH_END(ds);
   if (!found) {
     char *s = authdir_type_to_string(type);
@@ -380,10 +437,9 @@ should_use_directory_guards(const or_options_t *options)
   /* Public (non-bridge) servers never use directory guards. */
   if (public_server_mode(options))
     return 0;
-  /* If guards are disabled, or directory guards are disabled, we can't
-   * use directory guards.
+  /* If guards are disabled, we can't use directory guards.
    */
-  if (!options->UseEntryGuards || !options->UseEntryGuardsAsDirGuards)
+  if (!options->UseEntryGuards)
     return 0;
   /* If we're configured to fetch directory info aggressively or of a
    * nonstandard type, don't use directory guards. */
@@ -398,7 +454,8 @@ should_use_directory_guards(const or_options_t *options)
  * information of type <b>type</b>, and return its routerstatus. */
 static const routerstatus_t *
 directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
-                                 uint8_t dir_purpose)
+                                 uint8_t dir_purpose,
+                                 circuit_guard_state_t **guard_state_out)
 {
   const routerstatus_t *rs = NULL;
   const or_options_t *options = get_options();
@@ -407,7 +464,7 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
     log_warn(LD_BUG, "Called when we have UseBridges set.");
 
   if (should_use_directory_guards(options)) {
-    const node_t *node = choose_random_dirguard(type);
+    const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out);
     if (node)
       rs = node->rs;
   } else {
@@ -423,13 +480,94 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
   return rs;
 }
 
+/**
+ * Set the extra fields in <b>req</b> that are used when requesting a
+ * consensus of type <b>resource</b>.
+ *
+ * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
+ */
+static void
+dir_consensus_request_set_additional_headers(directory_request_t *req,
+                                             const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+  int or_diff_from_is_set = 0;
+
+  /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
+   * period of 1 hour.
+   */
+  const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180;
+  const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72;
+  const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0;
+  const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192;
+  const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] =
+    "try-diff-for-consensus-newer-than";
+
+  int flav = FLAV_NS;
+  if (resource)
+    flav = networkstatus_parse_flavor_name(resource);
+
+  int32_t max_age_for_diff = 3600 *
+    networkstatus_get_param(NULL,
+                            TRY_DIFF_FOR_CONSENSUS_NEWER_NAME,
+                            DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MIN_TRY_DIFF_FOR_CONSENSUS_NEWER,
+                            MAX_TRY_DIFF_FOR_CONSENSUS_NEWER);
+
+  if (flav != -1) {
+    /* IF we have a parsed consensus of this type, we can do an
+     * if-modified-time based on it. */
+    networkstatus_t *v;
+    v = networkstatus_get_latest_consensus_by_flavor(flav);
+    if (v) {
+      /* In networks with particularly short V3AuthVotingIntervals,
+       * ask for the consensus if it's been modified since half the
+       * V3AuthVotingInterval of the most recent consensus. */
+      time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (v->fresh_until > v->valid_after
+          && ims_delay > (v->fresh_until - v->valid_after)/2) {
+        ims_delay = (v->fresh_until - v->valid_after)/2;
+      }
+      if_modified_since = v->valid_after + ims_delay;
+      if (v->valid_after >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  } else {
+    /* Otherwise it might be a consensus we don't parse, but which we
+     * do cache.  Look at the cached copy, perhaps. */
+    cached_dir_t *cd = dirserv_get_consensus(resource);
+    /* We have no method of determining the voting interval from an
+     * unparsed consensus, so we use the default. */
+    if (cd) {
+      if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
+      if (cd->published >= approx_time() - max_age_for_diff) {
+        memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN);
+        or_diff_from_is_set = 1;
+      }
+    }
+  }
+
+  if (if_modified_since > 0)
+    directory_request_set_if_modified_since(req, if_modified_since);
+  if (or_diff_from_is_set) {
+    char hex[HEX_DIGEST256_LEN + 1];
+    base16_encode(hex, sizeof(hex),
+                  (const char*)or_diff_from, sizeof(or_diff_from));
+    directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
+  }
+}
+
 /** Start a connection to a random running directory server, using
  * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
  * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
  * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
  * or router_pick_trusteddirserver().
  */
-MOCK_IMPL(void, directory_get_from_dirserver, (
+MOCK_IMPL(void,
+directory_get_from_dirserver,(
                             uint8_t dir_purpose,
                             uint8_t router_purpose,
                             const char *resource,
@@ -441,52 +579,17 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
   int prefer_authority = (directory_fetches_from_authorities(options)
                           || want_authority == DL_WANT_AUTHORITY);
   int require_authority = 0;
-  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
+  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
+                                            resource);
   dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
-  time_t if_modified_since = 0;
 
   if (type == NO_DIRINFO)
     return;
 
-  if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    int flav = FLAV_NS;
-    networkstatus_t *v;
-    if (resource)
-      flav = networkstatus_parse_flavor_name(resource);
-
-    /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
-     * period of 1 hour.
-     */
-#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180)
-    if (flav != -1) {
-      /* IF we have a parsed consensus of this type, we can do an
-       * if-modified-time based on it. */
-      v = networkstatus_get_latest_consensus_by_flavor(flav);
-      if (v) {
-        /* In networks with particularly short V3AuthVotingIntervals,
-         * ask for the consensus if it's been modified since half the
-         * V3AuthVotingInterval of the most recent consensus. */
-        time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
-        if (v->fresh_until > v->valid_after
-            && ims_delay > (v->fresh_until - v->valid_after)/2) {
-          ims_delay = (v->fresh_until - v->valid_after)/2;
-        }
-        if_modified_since = v->valid_after + ims_delay;
-      }
-    } else {
-      /* Otherwise it might be a consensus we don't parse, but which we
-       * do cache.  Look at the cached copy, perhaps. */
-      cached_dir_t *cd = dirserv_get_consensus(resource);
-      /* We have no method of determining the voting interval from an
-       * unparsed consensus, so we use the default. */
-      if (cd)
-        if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
-    }
-  }
-
   if (!options->FetchServerDescriptors)
     return;
 
+  circuit_guard_state_t *guard_state = NULL;
   if (!get_via_tor) {
     if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
       /* We want to ask a running bridge for which we have a descriptor.
@@ -495,25 +598,34 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
        * sort of dir fetch we'll be doing, so it won't return a bridge
        * that can't answer our question.
        */
-      const node_t *node = choose_random_dirguard(type);
+      const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state);
       if (node && node->ri) {
         /* every bridge has a routerinfo. */
         routerinfo_t *ri = node->ri;
         /* clients always make OR connections to bridges */
         tor_addr_port_t or_ap;
+        directory_request_t *req = directory_request_new(dir_purpose);
         /* we are willing to use a non-preferred address if we need to */
         fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
                                              &or_ap);
-        directory_initiate_command(&or_ap.addr, or_ap.port,
-                                   NULL, 0, /*no dirport*/
-                                   ri->cache_info.identity_digest,
-                                   dir_purpose,
-                                   router_purpose,
-                                   DIRIND_ONEHOP,
-                                   resource, NULL, 0, if_modified_since);
-      } else
+        directory_request_set_or_addr_port(req, &or_ap);
+        directory_request_set_directory_id_digest(req,
+                                            ri->cache_info.identity_digest);
+        directory_request_set_router_purpose(req, router_purpose);
+        directory_request_set_resource(req, resource);
+        if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+          dir_consensus_request_set_additional_headers(req, resource);
+        directory_request_set_guard_state(req, guard_state);
+        directory_initiate_request(req);
+        directory_request_free(req);
+      } else {
+        if (guard_state) {
+          entry_guard_cancel(&guard_state);
+        }
         log_notice(LD_DIR, "Ignoring directory request, since no bridge "
                            "nodes are available yet.");
+      }
+
       return;
     } else {
       if (prefer_authority || (type & BRIDGE_DIRINFO)) {
@@ -544,9 +656,9 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
         }
       }
       if (!rs && !(type & BRIDGE_DIRINFO)) {
-        /* */
         rs = directory_pick_generic_dirserver(type, pds_flags,
-                                              dir_purpose);
+                                              dir_purpose,
+                                              &guard_state);
         if (!rs)
           get_via_tor = 1; /* last resort: try routing it via Tor */
       }
@@ -565,17 +677,23 @@ MOCK_IMPL(void, directory_get_from_dirserver, (
   if (rs) {
     const dir_indirection_t indirection =
       get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
-    directory_initiate_command_routerstatus(rs, dir_purpose,
-                                            router_purpose,
-                                            indirection,
-                                            resource, NULL, 0,
-                                            if_modified_since);
+    directory_request_t *req = directory_request_new(dir_purpose);
+    directory_request_set_routerstatus(req, rs);
+    directory_request_set_router_purpose(req, router_purpose);
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
+      dir_consensus_request_set_additional_headers(req, resource);
+    if (guard_state)
+      directory_request_set_guard_state(req, guard_state);
+    directory_initiate_request(req);
+    directory_request_free(req);
   } else {
     log_notice(LD_DIR,
                "While fetching directory info, "
                "no running dirservers known. Will try again later. "
                "(purpose %d)", dir_purpose);
-    if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
+    if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
       /* remember we tried them all and failed. */
       directory_all_unreachable(time(NULL));
     }
@@ -595,15 +713,17 @@ directory_get_from_all_authorities(uint8_t dir_purpose,
 
   SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
                           dir_server_t *, ds) {
-      routerstatus_t *rs;
       if (router_digest_is_me(ds->digest))
         continue;
       if (!(ds->type & V3_DIRINFO))
         continue;
-      rs = &ds->fake_status;
-      directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
-                                              DIRIND_ONEHOP, resource, NULL,
-                                              0, 0);
+      const routerstatus_t *rs = &ds->fake_status;
+      directory_request_t *req = directory_request_new(dir_purpose);
+      directory_request_set_routerstatus(req, rs);
+      directory_request_set_router_purpose(req, router_purpose);
+      directory_request_set_resource(req, resource);
+      directory_initiate_request(req);
+      directory_request_free(req);
   } SMARTLIST_FOREACH_END(ds);
 }
 
@@ -703,106 +823,6 @@ directory_choose_address_routerstatus(const routerstatus_t *status,
   return 0;
 }
 
-/** Same as directory_initiate_command_routerstatus(), but accepts
- * rendezvous data to fetch a hidden service descriptor. */
-void
-directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
-                                             uint8_t dir_purpose,
-                                             uint8_t router_purpose,
-                                             dir_indirection_t indirection,
-                                             const char *resource,
-                                             const char *payload,
-                                             size_t payload_len,
-                                             time_t if_modified_since,
-                                             const rend_data_t *rend_query)
-{
-  const or_options_t *options = get_options();
-  const node_t *node;
-  tor_addr_port_t use_or_ap, use_dir_ap;
-  const int anonymized_connection = dirind_is_anon(indirection);
-
-  tor_assert(status != NULL);
-
-  node = node_get_by_id(status->identity_digest);
-
-  /* XXX The below check is wrong: !node means it's not in the consensus,
-   * but we haven't checked if we have a descriptor for it -- and also,
-   * we only care about the descriptor if it's a begindir-style anonymized
-   * connection. */
-  if (!node && anonymized_connection) {
-    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
-             "don't have its router descriptor.",
-             routerstatus_describe(status));
-    return;
-  }
-
-  if (options->ExcludeNodes && options->StrictNodes &&
-      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
-    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
-             "it's in our ExcludedNodes list and StrictNodes is set. "
-             "Skipping. This choice might make your Tor not work.",
-             routerstatus_describe(status),
-             dir_conn_purpose_to_string(dir_purpose));
-    return;
-  }
-
-  /* At this point, if we are a client making a direct connection to a
-   * directory server, we have selected a server that has at least one address
-   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
-   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
-   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
-   * by default.)
-   *
-   * Now choose an address that we can use to connect to the directory server.
-   */
-  if (directory_choose_address_routerstatus(status, indirection, &use_or_ap,
-                                            &use_dir_ap) < 0) {
-    return;
-  }
-
-  /* We don't retry the alternate OR/Dir address for the same directory if
-   * the address we choose fails (#6772).
-   * Instead, we'll retry another directory on failure. */
-
-  directory_initiate_command_rend(&use_or_ap, &use_dir_ap,
-                                  status->identity_digest,
-                                  dir_purpose, router_purpose,
-                                  indirection, resource,
-                                  payload, payload_len, if_modified_since,
-                                  rend_query);
-}
-
-/** Launch a new connection to the directory server <b>status</b> to
- * upload or download a server or rendezvous
- * descriptor. <b>dir_purpose</b> determines what
- * kind of directory connection we're launching, and must be one of
- * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b>
- * specifies the descriptor purposes we have in mind (currently only
- * used for FETCH_DIR).
- *
- * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
- * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
- *
- * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
- * want to fetch.
- */
-MOCK_IMPL(void, directory_initiate_command_routerstatus,
-                (const routerstatus_t *status,
-                 uint8_t dir_purpose,
-                 uint8_t router_purpose,
-                 dir_indirection_t indirection,
-                 const char *resource,
-                 const char *payload,
-                 size_t payload_len,
-                 time_t if_modified_since))
-{
-  directory_initiate_command_routerstatus_rend(status, dir_purpose,
-                                          router_purpose,
-                                          indirection, resource,
-                                          payload, payload_len,
-                                          if_modified_since, NULL);
-}
-
 /** Return true iff <b>conn</b> is the client side of a directory connection
  * we launched to ourself in order to determine the reachability of our
  * dir_port. */
@@ -828,6 +848,11 @@ directory_conn_is_self_reachability_test(dir_connection_t *conn)
 static void
 connection_dir_request_failed(dir_connection_t *conn)
 {
+  if (conn->guard_state) {
+    /* We haven't seen a success on this guard state, so consider it to have
+     * failed. */
+    entry_guard_failed(&conn->guard_state);
+  }
   if (directory_conn_is_self_reachability_test(conn)) {
     return; /* this was a test fetch. don't retry. */
   }
@@ -981,8 +1006,50 @@ directory_must_use_begindir(const or_options_t *options)
   return !public_server_mode(options);
 }
 
+struct directory_request_t {
+  /**
+   * These fields specify which directory we're contacting.  Routerstatus,
+   * if present, overrides the other fields.
+   *
+   * @{ */
+  tor_addr_port_t or_addr_port;
+  tor_addr_port_t dir_addr_port;
+  char digest[DIGEST_LEN];
+
+  const routerstatus_t *routerstatus;
+  /** @} */
+  /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
+   * kind of operation we'll be doing (upload/download), and of what kind
+   * of document. */
+  uint8_t dir_purpose;
+  /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
+   * and extrainfo docs.  */
+  uint8_t router_purpose;
+  /** Enum: determines whether to anonymize, and whether to use dirport or
+   * orport. */
+  dir_indirection_t indirection;
+  /** Alias to the variable part of the URL for this request */
+  const char *resource;
+  /** Alias to the payload to upload (if any) */
+  const char *payload;
+  /** Number of bytes to upload from payload</b> */
+  size_t payload_len;
+  /** Value to send in an if-modified-since header, or 0 for none. */
+  time_t if_modified_since;
+  /** Hidden-service-specific information */
+  const rend_data_t *rend_query;
+  /** Extra headers to append to the request */
+  config_line_t *additional_headers;
+  /** */
+  /** Used internally to directory.c: gets informed when the attempt to
+   * connect to the directory succeeds or fails, if that attempt bears on the
+   * directory's usability as a directory guard. */
+  circuit_guard_state_t *guard_state;
+};
+
 /** Evaluate the situation and decide if we should use an encrypted
  * "begindir-style" connection for this directory request.
+ * 0) If there is no DirPort, yes.
  * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
  *    or we're a dir mirror, no.
  * 2) If we prefer to avoid begindir conns, and we're not fetching or
@@ -993,15 +1060,24 @@ directory_must_use_begindir(const or_options_t *options)
  */
 static int
 directory_command_should_use_begindir(const or_options_t *options,
-                                      const tor_addr_t *addr,
-                                      int or_port, uint8_t router_purpose,
-                                      dir_indirection_t indirection,
+                                      const directory_request_t *req,
                                       const char **reason)
 {
-  (void) router_purpose;
+  const tor_addr_t *or_addr = &req->or_addr_port.addr;
+  //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
+  const int or_port = req->or_addr_port.port;
+  const int dir_port = req->dir_addr_port.port;
+
+  const dir_indirection_t indirection = req->indirection;
+
   tor_assert(reason);
   *reason = NULL;
 
+  /* Reasons why we must use begindir */
+  if (!dir_port) {
+    *reason = "(using begindir - directory with no DirPort)";
+    return 1; /* We don't know a DirPort -- must begindir. */
+  }
   /* Reasons why we can't possibly use begindir */
   if (!or_port) {
     *reason = "directory with unknown ORPort";
@@ -1014,7 +1090,7 @@ directory_command_should_use_begindir(const or_options_t *options,
   }
   if (indirection == DIRIND_ONEHOP) {
     /* We're firewalled and want a direct OR connection */
-    if (!fascist_firewall_allows_address_addr(addr, or_port,
+    if (!fascist_firewall_allows_address_addr(or_addr, or_port,
                                               FIREWALL_OR_CONNECTION, 0, 0)) {
       *reason = "ORPort not reachable";
       return 0;
@@ -1033,79 +1109,290 @@ directory_command_should_use_begindir(const or_options_t *options,
   return 1;
 }
 
-/** Helper for directory_initiate_command_rend: send the
- * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>,
- * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose
- * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and
- * <b>router_purpose</b>, making an (in)direct connection as specified in
- * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of
- * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>.
+/**
+ * Create and return a new directory_request_t with purpose
+ * <b>dir_purpose</b>.
+ */
+directory_request_t *
+directory_request_new(uint8_t dir_purpose)
+{
+  tor_assert(dir_purpose >= DIR_PURPOSE_MIN_);
+  tor_assert(dir_purpose <= DIR_PURPOSE_MAX_);
+  tor_assert(dir_purpose != DIR_PURPOSE_SERVER);
+  tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2);
+
+  directory_request_t *result = tor_malloc_zero(sizeof(*result));
+  tor_addr_make_null(&result->or_addr_port.addr, AF_INET);
+  result->or_addr_port.port = 0;
+  tor_addr_make_null(&result->dir_addr_port.addr, AF_INET);
+  result->dir_addr_port.port = 0;
+  result->dir_purpose = dir_purpose;
+  result->router_purpose = ROUTER_PURPOSE_GENERAL;
+  result->indirection = DIRIND_ONEHOP;
+  return result;
+}
+/**
+ * Release all resources held by <b>req</b>.
+ */
+void
+directory_request_free(directory_request_t *req)
+{
+  if (req == NULL)
+    return;
+  config_free_lines(req->additional_headers);
+  tor_free(req);
+}
+/**
+ * Set the address and OR port to use for this directory request.  If there is
+ * no OR port, we'll have to connect over the dirport.  (If there are both,
+ * the indirection setting determins which to use.)
+ */
+void
+directory_request_set_or_addr_port(directory_request_t *req,
+                                   const tor_addr_port_t *p)
+{
+  memcpy(&req->or_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the address and dirport to use for this directory request.  If there
+ * is no dirport, we'll have to connect over the OR port.  (If there are both,
+ * the indirection setting determins which to use.)
  */
 void
-directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
-                           const tor_addr_t *dir_addr, uint16_t dir_port,
-                           const char *digest,
-                           uint8_t dir_purpose, uint8_t router_purpose,
-                           dir_indirection_t indirection, const char *resource,
-                           const char *payload, size_t payload_len,
-                           time_t if_modified_since)
+directory_request_set_dir_addr_port(directory_request_t *req,
+                                    const tor_addr_port_t *p)
 {
-  tor_addr_port_t or_ap, dir_ap;
+  memcpy(&req->dir_addr_port, p, sizeof(*p));
+}
+/**
+ * Set the RSA identity digest of the directory to use for this directory
+ * request.
+ */
+void
+directory_request_set_directory_id_digest(directory_request_t *req,
+                                          const char *digest)
+{
+  memcpy(req->digest, digest, DIGEST_LEN);
+}
+/**
+ * Set the router purpose associated with uploaded and downloaded router
+ * descriptors and extrainfo documents in this directory request.  The purpose
+ * must be one of ROUTER_PURPOSE_GENERAL (the default) or
+ * ROUTER_PURPOSE_BRIDGE.
+ */
+void
+directory_request_set_router_purpose(directory_request_t *req,
+                                     uint8_t router_purpose)
+{
+  tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL ||
+             router_purpose == ROUTER_PURPOSE_BRIDGE);
+  // assert that it actually makes sense to set this purpose, given
+  // the dir_purpose.
+  req->router_purpose = router_purpose;
+}
+/**
+ * Set the indirection to be used for the directory request.  The indirection
+ * parameter configures whether to connect to a DirPort or ORPort, and whether
+ * to anonymize the connection.  DIRIND_ONEHOP (use ORPort, don't anonymize)
+ * is the default.  See dir_indirection_t for more information.
+ */
+void
+directory_request_set_indirection(directory_request_t *req,
+                                  dir_indirection_t indirection)
+{
+  req->indirection = indirection;
+}
 
-  /* Use the null tor_addr and 0 port if the address or port isn't valid. */
-  if (tor_addr_port_is_valid(or_addr, or_port, 0)) {
-    tor_addr_copy(&or_ap.addr, or_addr);
-    or_ap.port = or_port;
-  } else {
-    /* the family doesn't matter here, so make it IPv4 */
-    tor_addr_make_null(&or_ap.addr, AF_INET);
-    or_ap.port = or_port = 0;
-  }
+/**
+ * Set a pointer to the resource to request from a directory.  Different
+ * request types use resources to indicate different components of their URL.
+ * Note that only an alias to <b>resource</b> is stored, so the
+ * <b>resource</b> must outlive the request.
+ */
+void
+directory_request_set_resource(directory_request_t *req,
+                               const char *resource)
+{
+  req->resource = resource;
+}
+/**
+ * Set a pointer to the payload to include with this directory request, along
+ * with its length.  Note that only an alias to <b>payload</b> is stored, so
+ * the <b>payload</b> must outlive the request.
+ */
+void
+directory_request_set_payload(directory_request_t *req,
+                              const char *payload,
+                              size_t payload_len)
+{
+  tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose));
 
-  if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) {
-    tor_addr_copy(&dir_ap.addr, dir_addr);
-    dir_ap.port = dir_port;
-  } else {
-    /* the family doesn't matter here, so make it IPv4 */
-    tor_addr_make_null(&dir_ap.addr, AF_INET);
-    dir_ap.port = dir_port = 0;
+  req->payload = payload;
+  req->payload_len = payload_len;
+}
+/**
+ * Set an if-modified-since date to send along with the request.  The
+ * default is 0 (meaning, send no if-modified-since header).
+ */
+void
+directory_request_set_if_modified_since(directory_request_t *req,
+                                        time_t if_modified_since)
+{
+  req->if_modified_since = if_modified_since;
+}
+
+/** Include a header of name <b>key</b> with content <b>val</b> in the
+ * request. Neither may include newlines or other odd characters. Their
+ * ordering is not currently guaranteed.
+ *
+ * Note that, as elsewhere in this module, header keys include a trailing
+ * colon and space.
+ */
+void
+directory_request_add_header(directory_request_t *req,
+                             const char *key,
+                             const char *val)
+{
+  config_line_prepend(&req->additional_headers, key, val);
+}
+/**
+ * Set an object containing HS data to be associated with this request.  Note
+ * that only an alias to <b>query</b> is stored, so the <b>query</b> object
+ * must outlive the request.
+ */
+void
+directory_request_set_rend_query(directory_request_t *req,
+                                 const rend_data_t *query)
+{
+  if (query) {
+    tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
+               req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
   }
+  req->rend_query = query;
+}
+/** Set a static circuit_guard_state_t object to affliate with the request in
+ * <b>req</b>.  This object will receive notification when the attempt to
+ * connect to the guard either succeeds or fails. */
+void
+directory_request_set_guard_state(directory_request_t *req,
+                                  circuit_guard_state_t *state)
+{
+  req->guard_state = state;
+}
 
-  directory_initiate_command_rend(&or_ap, &dir_ap,
-                             digest, dir_purpose,
-                             router_purpose, indirection,
-                             resource, payload, payload_len,
-                             if_modified_since, NULL);
+/**
+ * Internal: Return true if any information for contacting the directory in
+ * <b>req</b> has been set, other than by the routerstatus. */
+static int
+directory_request_dir_contact_info_specified(const directory_request_t *req)
+{
+  /* We only check for ports here, since we don't use an addr unless the port
+   * is set */
+  return (req->or_addr_port.port ||
+          req->dir_addr_port.port ||
+          ! tor_digest_is_zero(req->digest));
 }
 
-/** Return non-zero iff a directory connection with purpose
- * <b>dir_purpose</b> reveals sensitive information about a Tor
- * instance's client activities.  (Such connections must be performed
- * through normal three-hop Tor circuits.) */
-int
-is_sensitive_dir_purpose(uint8_t dir_purpose)
+/**
+ * Set the routerstatus to use for the directory associated with this
+ * request.  If this option is set, then no other function to set the
+ * directory's address or identity should be called.
+ */
+void
+directory_request_set_routerstatus(directory_request_t *req,
+                                   const routerstatus_t *status)
 {
-  return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) ||
-          (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) ||
-          (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2));
+  req->routerstatus = status;
 }
+/**
+ * Helper: update the addresses, ports, and identities in <b>req</b>
+ * from the routerstatus object in <b>req</b>.  Return 0 on success.
+ * On failure, warn and return -1.
+ */
+static int
+directory_request_set_dir_from_routerstatus(directory_request_t *req)
 
-/** Same as directory_initiate_command(), but accepts rendezvous data to
- * fetch a hidden service descriptor, and takes its address & port arguments
- * as tor_addr_port_t. */
-static void
-directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
-                                const tor_addr_port_t *dir_addr_port,
-                                const char *digest,
-                                uint8_t dir_purpose, uint8_t router_purpose,
-                                dir_indirection_t indirection,
-                                const char *resource,
-                                const char *payload, size_t payload_len,
-                                time_t if_modified_since,
-                                const rend_data_t *rend_query)
 {
-  tor_assert(or_addr_port);
-  tor_assert(dir_addr_port);
+  const routerstatus_t *status = req->routerstatus;
+  if (BUG(status == NULL))
+    return -1;
+  const or_options_t *options = get_options();
+  const node_t *node;
+  tor_addr_port_t use_or_ap, use_dir_ap;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
+  tor_assert(status != NULL);
+
+  node = node_get_by_id(status->identity_digest);
+
+  /* XXX The below check is wrong: !node means it's not in the consensus,
+   * but we haven't checked if we have a descriptor for it -- and also,
+   * we only care about the descriptor if it's a begindir-style anonymized
+   * connection. */
+  if (!node && anonymized_connection) {
+    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
+             "don't have its router descriptor.",
+             routerstatus_describe(status));
+    return -1;
+  }
+
+  if (options->ExcludeNodes && options->StrictNodes &&
+      routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
+    log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
+             "it's in our ExcludedNodes list and StrictNodes is set. "
+             "Skipping. This choice might make your Tor not work.",
+             routerstatus_describe(status),
+             dir_conn_purpose_to_string(req->dir_purpose));
+    return -1;
+  }
+
+    /* At this point, if we are a client making a direct connection to a
+   * directory server, we have selected a server that has at least one address
+   * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
+   * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
+   * possible. (If UseBridges is set, clients always use IPv6, and prefer it
+   * by default.)
+   *
+   * Now choose an address that we can use to connect to the directory server.
+   */
+  if (directory_choose_address_routerstatus(status,
+                                            req->indirection, &use_or_ap,
+                                            &use_dir_ap) < 0) {
+    return -1;
+  }
+
+  directory_request_set_or_addr_port(req, &use_or_ap);
+  directory_request_set_dir_addr_port(req, &use_dir_ap);
+  directory_request_set_directory_id_digest(req, status->identity_digest);
+  return 0;
+}
+
+/**
+ * Launch the provided directory request, configured in <b>request</b>.
+ * After this function is called, you can free <b>request</b>.
+ */
+MOCK_IMPL(void,
+directory_initiate_request,(directory_request_t *request))
+{
+  tor_assert(request);
+  if (request->routerstatus) {
+    tor_assert_nonfatal(
+               ! directory_request_dir_contact_info_specified(request));
+    if (directory_request_set_dir_from_routerstatus(request) < 0) {
+      return;
+    }
+  }
+
+  const tor_addr_port_t *or_addr_port = &request->or_addr_port;
+  const tor_addr_port_t *dir_addr_port = &request->dir_addr_port;
+  const char *digest = request->digest;
+  const uint8_t dir_purpose = request->dir_purpose;
+  const uint8_t router_purpose = request->router_purpose;
+  const dir_indirection_t indirection = request->indirection;
+  const char *resource = request->resource;
+  const rend_data_t *rend_query = request->rend_query;
+  circuit_guard_state_t *guard_state = request->guard_state;
+
   tor_assert(or_addr_port->port || dir_addr_port->port);
   tor_assert(digest);
 
@@ -1115,10 +1402,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
   const char *begindir_reason = NULL;
   /* Should the connection be to a relay's OR port (and inside that we will
    * send our directory request)? */
-  const int use_begindir = directory_command_should_use_begindir(options,
-                                     &or_addr_port->addr, or_addr_port->port,
-                                     router_purpose, indirection,
-                                     &begindir_reason);
+  const int use_begindir =
+    directory_command_should_use_begindir(options, request, &begindir_reason);
+
   /* Will the connection go via a three-hop Tor circuit? Note that this
    * is separate from whether it will use_begindir. */
   const int anonymized_connection = dirind_is_anon(indirection);
@@ -1137,7 +1423,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
 
   log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
 
-  if (is_sensitive_dir_purpose(dir_purpose)) {
+  if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
     tor_assert(anonymized_connection ||
                rend_non_anonymous_mode_enabled(options));
   }
@@ -1163,9 +1449,9 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
   if (!port || tor_addr_is_null(&addr)) {
     static int logged_backtrace = 0;
     log_warn(LD_DIR,
-             "Cannot make an outgoing %sconnection without %sPort.",
+             "Cannot make an outgoing %sconnection without a remote %sPort.",
              use_begindir ? "begindir " : "",
-             use_begindir ? "an OR" : "a Dir");
+             use_begindir ? "OR" : "Dir");
     if (!logged_backtrace) {
       log_backtrace(LOG_INFO, LD_BUG, "Address came from");
       logged_backtrace = 1;
@@ -1203,6 +1489,11 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
       port = options->HTTPProxyPort;
     }
 
+    // In this case we should not have picked a directory guard.
+    if (BUG(guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
     switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
                                port, &socket_error)) {
       case -1:
@@ -1214,9 +1505,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
         /* fall through */
       case 0:
         /* queue the command on the outbuf */
-        directory_send_command(conn, dir_purpose, 1, resource,
-                               payload, payload_len,
-                               if_modified_since);
+        directory_send_command(conn, 1, request);
         connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
         /* writable indicates finish, readable indicates broken link,
            error indicates broken link in windowsland. */
@@ -1239,6 +1528,14 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
     else if (anonymized_connection && !use_begindir)
       rep_hist_note_used_port(time(NULL), conn->base_.port);
 
+    // In this case we should not have a directory guard; we'll
+    // get a regular guard later when we build the circuit.
+    if (BUG(anonymized_connection && guard_state)) {
+      entry_guard_cancel(&guard_state);
+    }
+
+    conn->guard_state = guard_state;
+
     /* make an AP connection
      * populate it and add it at the right state
      * hook up both sides
@@ -1262,9 +1559,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
     }
     conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
     /* queue the command on the outbuf */
-    directory_send_command(conn, dir_purpose, 0, resource,
-                           payload, payload_len,
-                           if_modified_since);
+    directory_send_command(conn, 0, request);
 
     connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
     connection_start_reading(ENTRY_TO_CONN(linked_conn));
@@ -1274,7 +1569,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port,
 /** Return true iff anything we say on <b>conn</b> is being encrypted before
  * we send it to the client/server. */
 int
-connection_dir_is_encrypted(dir_connection_t *conn)
+connection_dir_is_encrypted(const dir_connection_t *conn)
 {
   /* Right now it's sufficient to see if conn is or has been linked, since
    * the only thing it could be linked to is an edge connection on a
@@ -1367,15 +1662,23 @@ copy_ipv6_address(char* destination, const char* source, size_t len,
   }
 }
 
-/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
- * are as in directory_initiate_command().
+/** Queue an appropriate HTTP command for <b>request</b> on
+ * <b>conn</b>-\>outbuf.  If <b>direct</b> is true, we're making a
+ * non-anonymized connection to the dirport.
  */
 static void
 directory_send_command(dir_connection_t *conn,
-                       int purpose, int direct, const char *resource,
-                       const char *payload, size_t payload_len,
-                       time_t if_modified_since)
+                       const int direct,
+                       const directory_request_t *req)
 {
+  tor_assert(req);
+  const int purpose = req->dir_purpose;
+  const char *resource = req->resource;
+  const char *payload = req->payload;
+  const size_t payload_len = req->payload_len;
+  const time_t if_modified_since = req->if_modified_since;
+  const int anonymized_connection = dirind_is_anon(req->indirection);
+
   char proxystring[256];
   char hoststring[128];
   /* NEEDS to be the same size hoststring.
@@ -1383,7 +1686,10 @@ directory_send_command(dir_connection_t *conn,
   char decorated_address[128];
   smartlist_t *headers = smartlist_new();
   char *url;
+  char *accept_encoding;
+  size_t url_len;
   char request[8192];
+  size_t request_len, total_request_len = 0;
   const char *httpcommand = NULL;
 
   tor_assert(conn);
@@ -1437,6 +1743,22 @@ directory_send_command(dir_connection_t *conn,
     proxystring[0] = 0;
   }
 
+  if (! anonymized_connection) {
+    /* Add Accept-Encoding. */
+    accept_encoding = accept_encoding_header();
+    smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n",
+                           accept_encoding);
+    tor_free(accept_encoding);
+  }
+
+  /* Add additional headers, if any */
+  {
+    config_line_t *h;
+    for (h = req->additional_headers; h; h = h->next) {
+      smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value);
+    }
+  }
+
   switch (purpose) {
     case DIR_PURPOSE_FETCH_CONSENSUS:
       /* resource is optional.  If present, it's a flavor name */
@@ -1529,8 +1851,14 @@ directory_send_command(dir_connection_t *conn,
   }
 
   tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
-  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
-  connection_write_to_buf(url, strlen(url), TO_CONN(conn));
+
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_write_to_buf(request, request_len, TO_CONN(conn));
+
+  url_len = strlen(url);
+  total_request_len += url_len;
+  connection_write_to_buf(url, url_len, TO_CONN(conn));
   tor_free(url);
 
   if (!strcmp(httpcommand, "POST") || payload) {
@@ -1545,15 +1873,27 @@ directory_send_command(dir_connection_t *conn,
     tor_free(header);
   }
 
-  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
+  request_len = strlen(request);
+  total_request_len += request_len;
+  connection_write_to_buf(request, request_len, TO_CONN(conn));
 
   if (payload) {
     /* then send the payload afterwards too */
     connection_write_to_buf(payload, payload_len, TO_CONN(conn));
+    total_request_len += payload_len;
   }
 
   SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
   smartlist_free(headers);
+
+  log_debug(LD_DIR,
+            "Sent request to directory server '%s:%d': "
+            "(purpose: %d, request size: " U64_FORMAT ", "
+            "payload size: " U64_FORMAT ")",
+            conn->base_.address, conn->base_.port,
+            conn->base_.purpose,
+            U64_PRINTF_ARG(total_request_len),
+            U64_PRINTF_ARG(payload ? payload_len : 0));
 }
 
 /** Parse an HTTP request string <b>headers</b> of the form
@@ -1738,16 +2078,15 @@ parse_http_response(const char *headers, int *code, time_t *date,
       if (!strcmpstart(s, "Content-Encoding: ")) {
         enc = s+18; break;
       });
-    if (!enc || !strcmp(enc, "identity")) {
+
+    if (enc == NULL)
       *compression = NO_METHOD;
-    } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
-      *compression = ZLIB_METHOD;
-    } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
-      *compression = GZIP_METHOD;
-    } else {
-      log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
-               escaped(enc));
-      *compression = UNKNOWN_METHOD;
+    else {
+      *compression = compression_method_get_by_name(enc);
+
+      if (*compression == UNKNOWN_METHOD)
+        log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
+                 escaped(enc));
     }
   }
   SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
@@ -1770,15 +2109,15 @@ body_is_plausible(const char *body, size_t len, int purpose)
   if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
     return (!strcmpstart(body,"onion-key"));
   }
-  if (1) {
-    if (!strcmpstart(body,"router") ||
-        !strcmpstart(body,"network-status"))
-      return 1;
-    for (i=0;i<32;++i) {
-      if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
-        return 0;
-    }
+
+  if (!strcmpstart(body,"router") ||
+      !strcmpstart(body,"network-status"))
+    return 1;
+  for (i=0;i<32;++i) {
+    if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
+      return 0;
   }
+
   return 1;
 }
 
@@ -1820,6 +2159,154 @@ load_downloaded_routers(const char *body, smartlist_t *which,
   return added;
 }
 
+/** A structure to hold arguments passed into each directory response
+ * handler */
+typedef struct response_handler_args_t {
+  int status_code;
+  const char *reason;
+  const char *body;
+  size_t body_len;
+  const char *headers;
+} response_handler_args_t;
+
+static int handle_response_fetch_consensus(dir_connection_t *,
+                                           const response_handler_args_t *);
+static int handle_response_fetch_certificate(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_status_vote(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_detached_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_desc(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_dir(dir_connection_t *,
+                                      const response_handler_args_t *);
+static int handle_response_upload_vote(dir_connection_t *,
+                                       const response_handler_args_t *);
+static int handle_response_upload_signatures(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_fetch_renddesc_v2(dir_connection_t *,
+                                             const response_handler_args_t *);
+static int handle_response_upload_renddesc_v2(dir_connection_t *,
+                                              const response_handler_args_t *);
+
+static int
+dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
+                                    dir_connection_t *conn,
+                                    compress_method_t compression,
+                                    int anonymized_connection)
+{
+  int rv = 0;
+  const char *body = *bodyp;
+  size_t body_len = *bodylenp;
+  int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
+                       conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+
+  int plausible = body_is_plausible(body, body_len, conn->base_.purpose);
+
+  if (plausible && compression == NO_METHOD) {
+    return 0;
+  }
+
+  int severity = LOG_DEBUG;
+  char *new_body = NULL;
+  size_t new_len = 0;
+  const char *description1, *description2;
+  int want_to_try_both = 0;
+  int tried_both = 0;
+  compress_method_t guessed = detect_compression_method(body, body_len);
+
+  description1 = compression_method_get_human_name(compression);
+
+  if (BUG(description1 == NULL))
+    description1 = compression_method_get_human_name(UNKNOWN_METHOD);
+
+  if (guessed == UNKNOWN_METHOD && !plausible)
+    description2 = "confusing binary junk";
+  else
+    description2 = compression_method_get_human_name(guessed);
+
+  /* Tell the user if we don't believe what we're told about compression.*/
+  want_to_try_both = (compression == UNKNOWN_METHOD ||
+                      guessed != compression);
+  if (want_to_try_both) {
+    severity = LOG_PROTOCOL_WARN;
+  }
+
+  tor_log(severity, LD_HTTP,
+          "HTTP body from server '%s:%d' was labeled as %s, "
+          "%s it seems to be %s.%s",
+          conn->base_.address, conn->base_.port, description1,
+          guessed != compression?"but":"and",
+          description2,
+          (compression>0 && guessed>0 && want_to_try_both)?
+          "  Trying both.":"");
+
+  /* Try declared compression first if we can.
+   * tor_compress_supports_method() also returns true for NO_METHOD.
+   * Ensure that the server is not sending us data compressed using a
+   * compression method that is not allowed for anonymous connections. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(compression)) {
+    warn_disallowed_anonymous_compression_method(compression);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(compression)) {
+    tor_uncompress(&new_body, &new_len, body, body_len, compression,
+                   !allow_partial, LOG_PROTOCOL_WARN);
+    if (new_body) {
+      /* We succeeded with the declared compression method. Great! */
+      rv = 0;
+      goto done;
+    }
+  }
+
+  /* Okay, if that didn't work, and we think that it was compressed
+   * differently, try that. */
+  if (anonymized_connection &&
+      ! allowed_anonymous_connection_compression_method(guessed)) {
+    warn_disallowed_anonymous_compression_method(guessed);
+    rv = -1;
+    goto done;
+  }
+
+  if (tor_compress_supports_method(guessed) &&
+      compression != guessed) {
+    tor_uncompress(&new_body, &new_len, body, body_len, guessed,
+                   !allow_partial, LOG_INFO);
+    tried_both = 1;
+  }
+  /* If we're pretty sure that we have a compressed directory, and
+   * we didn't manage to uncompress it, then warn and bail. */
+  if (!plausible && !new_body) {
+    log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+           "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
+           description1,
+           tried_both?" and ":"",
+           tried_both?description2:"",
+           conn->base_.address, conn->base_.port);
+    rv = -1;
+    goto done;
+  }
+
+ done:
+  if (new_body) {
+    if (rv == 0) {
+      /* success! */
+      tor_free(*bodyp);
+      *bodyp = new_body;
+      *bodylenp = new_len;
+    } else {
+      tor_free(new_body);
+    }
+  }
+
+  return rv;
+}
+
 /** We are a client, and we've finished reading the server's
  * response. Parse it and act appropriately.
  *
@@ -1832,21 +2319,26 @@ load_downloaded_routers(const char *body, smartlist_t *which,
 static int
 connection_dir_client_reached_eof(dir_connection_t *conn)
 {
-  char *body;
-  char *headers;
+  char *body = NULL;
+  char *headers = NULL;
   char *reason = NULL;
   size_t body_len = 0;
   int status_code;
   time_t date_header = 0;
   long apparent_skew;
   compress_method_t compression;
-  int plausible;
   int skewed = 0;
+  int rv;
   int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
                        conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
                        conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
-  time_t now = time(NULL);
-  int src_code;
+  size_t received_bytes;
+  const int anonymized_connection =
+    purpose_needs_anonymity(conn->base_.purpose,
+                            conn->router_purpose,
+                            conn->requested_resource);
+
+  received_bytes = connection_get_inbuf_len(TO_CONN(conn));
 
   switch (connection_fetch_from_buf_http(TO_CONN(conn),
                               &headers, MAX_HEADERS_SIZE,
@@ -1868,17 +2360,39 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
                           &compression, &reason) < 0) {
     log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
              conn->base_.address, conn->base_.port);
-    tor_free(body); tor_free(headers);
-    return -1;
+
+    rv = -1;
+    goto done;
   }
   if (!reason) reason = tor_strdup("[no reason given]");
 
-  log_debug(LD_DIR,
+  tor_log(LOG_DEBUG, LD_DIR,
             "Received response from directory server '%s:%d': %d %s "
-            "(purpose: %d)",
+            "(purpose: %d, response size: " U64_FORMAT
+#ifdef MEASUREMENTS_21206
+            ", data cells received: %d, data cells sent: %d"
+#endif
+            ", compression: %d)",
             conn->base_.address, conn->base_.port, status_code,
-            escaped(reason),
-            conn->base_.purpose);
+            escaped(reason), conn->base_.purpose,
+            U64_PRINTF_ARG(received_bytes),
+#ifdef MEASUREMENTS_21206
+            conn->data_cells_received, conn->data_cells_sent,
+#endif
+            compression);
+
+  if (conn->guard_state) {
+    /* we count the connection as successful once we can read from it.  We do
+     * not, however, delay use of the circuit here, since it's just for a
+     * one-hop directory request. */
+    /* XXXXprop271 note that this will not do the right thing for other
+     * waiting circuits that would be triggered by this circuit becoming
+     * complete/usable. But that's ok, I think.
+     */
+    entry_guard_succeeded(&conn->guard_state);
+    circuit_guard_state_free(conn->guard_state);
+    conn->guard_state = NULL;
+  }
 
   /* now check if it's got any hints for us about our IP address. */
   if (conn->dirconn_direct) {
@@ -1916,540 +2430,754 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
              "'%s:%d'. I'll try again soon.",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
+    time_t now = approx_time();
     if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
       rs->last_dir_503_at = now;
     if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
       ds->fake_status.last_dir_503_at = now;
 
-    tor_free(body); tor_free(headers); tor_free(reason);
-    return -1;
+    rv = -1;
+    goto done;
   }
 
-  plausible = body_is_plausible(body, body_len, conn->base_.purpose);
-  if (compression != NO_METHOD || !plausible) {
-    char *new_body = NULL;
-    size_t new_len = 0;
-    compress_method_t guessed = detect_compression_method(body, body_len);
-    if (compression == UNKNOWN_METHOD || guessed != compression) {
-      /* Tell the user if we don't believe what we're told about compression.*/
-      const char *description1, *description2;
-      if (compression == ZLIB_METHOD)
-        description1 = "as deflated";
-      else if (compression == GZIP_METHOD)
-        description1 = "as gzipped";
-      else if (compression == NO_METHOD)
-        description1 = "as uncompressed";
-      else
-        description1 = "with an unknown Content-Encoding";
-      if (guessed == ZLIB_METHOD)
-        description2 = "deflated";
-      else if (guessed == GZIP_METHOD)
-        description2 = "gzipped";
-      else if (!plausible)
-        description2 = "confusing binary junk";
-      else
-        description2 = "uncompressed";
+  if (dir_client_decompress_response_body(&body, &body_len,
+                             conn, compression, anonymized_connection) < 0) {
+    rv = -1;
+    goto done;
+  }
 
-      log_info(LD_HTTP, "HTTP body from server '%s:%d' was labeled %s, "
-               "but it seems to be %s.%s",
-               conn->base_.address, conn->base_.port, description1,
-               description2,
-               (compression>0 && guessed>0)?"  Trying both.":"");
-    }
-    /* Try declared compression first if we can. */
-    if (compression == GZIP_METHOD  || compression == ZLIB_METHOD)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression,
-                          !allow_partial, LOG_PROTOCOL_WARN);
-    /* Okay, if that didn't work, and we think that it was compressed
-     * differently, try that. */
-    if (!new_body &&
-        (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) &&
-        compression != guessed)
-      tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed,
-                          !allow_partial, LOG_PROTOCOL_WARN);
-    /* If we're pretty sure that we have a compressed directory, and
-     * we didn't manage to uncompress it, then warn and bail. */
-    if (!plausible && !new_body) {
-      log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
-             "Unable to decompress HTTP body (server '%s:%d').",
-             conn->base_.address, conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    if (new_body) {
-      tor_free(body);
-      body = new_body;
-      body_len = new_len;
-    }
+  response_handler_args_t args;
+  memset(&args, 0, sizeof(args));
+  args.status_code = status_code;
+  args.reason = reason;
+  args.body = body;
+  args.body_len = body_len;
+  args.headers = headers;
+
+  switch (conn->base_.purpose) {
+    case DIR_PURPOSE_FETCH_CONSENSUS:
+      rv = handle_response_fetch_consensus(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_CERTIFICATE:
+      rv = handle_response_fetch_certificate(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_STATUS_VOTE:
+      rv = handle_response_fetch_status_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+      rv = handle_response_fetch_detached_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_SERVERDESC:
+    case DIR_PURPOSE_FETCH_EXTRAINFO:
+      rv = handle_response_fetch_desc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_MICRODESC:
+      rv = handle_response_fetch_microdesc(conn, &args);
+      break;
+    case DIR_PURPOSE_FETCH_RENDDESC_V2:
+      rv = handle_response_fetch_renddesc_v2(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_DIR:
+      rv = handle_response_upload_dir(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_SIGNATURES:
+      rv = handle_response_upload_signatures(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_VOTE:
+      rv = handle_response_upload_vote(conn, &args);
+      break;
+    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+      rv = handle_response_upload_renddesc_v2(conn, &args);
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      rv = -1;
+      break;
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
-    int r;
-    const char *flavname = conn->requested_resource;
-    if (status_code != 200) {
-      int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
-      tor_log(severity, LD_DIR,
-          "Received http status code %d (%s) from server "
-          "'%s:%d' while fetching consensus directory.",
-           status_code, escaped(reason), conn->base_.address,
-           conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      networkstatus_consensus_download_failed(status_code, flavname);
-      return -1;
+ done:
+  tor_free(body);
+  tor_free(headers);
+  tor_free(reason);
+  return rv;
+}
+
+/**
+ * Handler function: processes a response to a request for a networkstatus
+ * consensus document by checking the consensus, storing it, and marking
+ * router requests as reachable.
+ **/
+static int
+handle_response_fetch_consensus(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS);
+  const int status_code = args->status_code;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+  const char *reason = args->reason;
+  const time_t now = approx_time();
+
+  const char *consensus;
+  char *new_consensus = NULL;
+  const char *sourcename;
+
+  int r;
+  const char *flavname = conn->requested_resource;
+  if (status_code != 200) {
+    int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
+    tor_log(severity, LD_DIR,
+            "Received http status code %d (%s) from server "
+            "'%s:%d' while fetching consensus directory.",
+            status_code, escaped(reason), conn->base_.address,
+            conn->base_.port);
+    networkstatus_consensus_download_failed(status_code, flavname);
+    return -1;
+  }
+
+  if (looks_like_a_consensus_diff(body, body_len)) {
+    /* First find our previous consensus. Maybe it's in ram, maybe not. */
+    cached_dir_t *cd = dirserv_get_consensus(flavname);
+    const char *consensus_body;
+    char *owned_consensus = NULL;
+    if (cd) {
+      consensus_body = cd->dir;
+    } else {
+      owned_consensus = networkstatus_read_cached_consensus(flavname);
+      consensus_body = owned_consensus;
     }
-    log_info(LD_DIR,"Received consensus directory (size %d) from server "
-             "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
-    if ((r=networkstatus_set_current_consensus(body, flavname, 0,
-                                               conn->identity_digest))<0) {
-      log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
-             "Unable to load %s consensus directory downloaded from "
-             "server '%s:%d'. I'll try again soon.",
-             flavname, conn->base_.address, conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
+    if (!consensus_body) {
+      log_warn(LD_DIR, "Received a consensus diff, but we can't find "
+               "any %s-flavored consensus in our current cache.",flavname);
       networkstatus_consensus_download_failed(0, flavname);
+      // XXXX if this happens too much, see below
       return -1;
     }
 
-    /* If we launched other fetches for this consensus, cancel them. */
-    connection_dir_close_consensus_fetches(conn, flavname);
-
-    /* launches router downloads as needed */
-    routers_update_all_from_networkstatus(now, 3);
-    update_microdescs_from_networkstatus(now);
-    update_microdesc_downloads(now);
-    directory_info_has_arrived(now, 0, 0);
-    if (authdir_mode_v3(get_options())) {
-      sr_act_post_consensus(
-                   networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
-    }
-    log_info(LD_DIR, "Successfully loaded consensus.");
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
-    if (status_code != 200) {
-      log_warn(LD_DIR,
-          "Received http status code %d (%s) from server "
-          "'%s:%d' while fetching \"/tor/keys/%s\".",
-           status_code, escaped(reason), conn->base_.address,
-           conn->base_.port, conn->requested_resource);
-      connection_dir_download_cert_failed(conn, status_code);
-      tor_free(body); tor_free(headers); tor_free(reason);
+    new_consensus = consensus_diff_apply(consensus_body, body);
+    tor_free(owned_consensus);
+    if (new_consensus == NULL) {
+      log_warn(LD_DIR, "Could not apply consensus diff received from server "
+               "'%s:%d'", conn->base_.address, conn->base_.port);
+      // XXXX If this happens too many times, we should maybe not use
+      // XXXX this directory for diffs any more?
+      networkstatus_consensus_download_failed(0, flavname);
       return -1;
     }
-    log_info(LD_DIR,"Received authority certificates (size %d) from server "
+    log_info(LD_DIR, "Applied consensus diff (size %d) from server "
+             "'%s:%d', resulting in a new consensus document (size %d).",
+             (int)body_len, conn->base_.address, conn->base_.port,
+             (int)strlen(new_consensus));
+    consensus = new_consensus;
+    sourcename = "generated based on a diff";
+  } else {
+    log_info(LD_DIR,"Received consensus directory (body size %d) from server "
              "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
+    consensus = body;
+    sourcename = "downloaded";
+  }
 
-    /*
-     * Tell trusted_dirs_load_certs_from_string() whether it was by fp
-     * or fp-sk pair.
-     */
-    src_code = -1;
-    if (!strcmpstart(conn->requested_resource, "fp/")) {
-      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
-    } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
-      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
-    }
+  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+                                             conn->identity_digest))<0) {
+    log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
+           "Unable to load %s consensus directory %s from "
+           "server '%s:%d'. I'll try again soon.",
+           flavname, sourcename, conn->base_.address, conn->base_.port);
+    networkstatus_consensus_download_failed(0, flavname);
+    tor_free(new_consensus);
+    return -1;
+  }
 
-    if (src_code != -1) {
-      if (trusted_dirs_load_certs_from_string(body, src_code, 1,
-                                              conn->identity_digest)<0) {
-        log_warn(LD_DIR, "Unable to parse fetched certificates");
-        /* if we fetched more than one and only some failed, the successful
-         * ones got flushed to disk so it's safe to call this on them */
-        connection_dir_download_cert_failed(conn, status_code);
-      } else {
-        directory_info_has_arrived(now, 0, 0);
-        log_info(LD_DIR, "Successfully loaded certificates from fetch.");
-      }
-    } else {
-      log_warn(LD_DIR,
-               "Couldn't figure out what to do with fetched certificates for "
-               "unknown resource %s",
-               conn->requested_resource);
+  /* If we launched other fetches for this consensus, cancel them. */
+  connection_dir_close_consensus_fetches(conn, flavname);
+
+  /* update the list of routers and directory guards */
+  routers_update_all_from_networkstatus(now, 3);
+  update_microdescs_from_networkstatus(now);
+  directory_info_has_arrived(now, 0, 0);
+
+  if (authdir_mode_v3(get_options())) {
+    sr_act_post_consensus(
+                     networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
+  }
+  log_info(LD_DIR, "Successfully loaded consensus.");
+
+  tor_free(new_consensus);
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for one or more
+ * authority certificates
+ **/
+static int
+handle_response_fetch_certificate(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  if (status_code != 200) {
+    log_warn(LD_DIR,
+             "Received http status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/keys/%s\".",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port, conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
+    return -1;
+  }
+  log_info(LD_DIR,"Received authority certificates (body size %d) from "
+           "server '%s:%d'",
+           (int)body_len, conn->base_.address, conn->base_.port);
+
+  /*
+   * Tell trusted_dirs_load_certs_from_string() whether it was by fp
+   * or fp-sk pair.
+   */
+  int src_code = -1;
+  if (!strcmpstart(conn->requested_resource, "fp/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
+  } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
+    src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
+  }
+
+  if (src_code != -1) {
+    if (trusted_dirs_load_certs_from_string(body, src_code, 1,
+                                            conn->identity_digest)<0) {
+      log_warn(LD_DIR, "Unable to parse fetched certificates");
+      /* if we fetched more than one and only some failed, the successful
+       * ones got flushed to disk so it's safe to call this on them */
       connection_dir_download_cert_failed(conn, status_code);
+    } else {
+      time_t now = approx_time();
+      directory_info_has_arrived(now, 0, 0);
+      log_info(LD_DIR, "Successfully loaded certificates from fetch.");
     }
+  } else {
+    log_warn(LD_DIR,
+             "Couldn't figure out what to do with fetched certificates for "
+             "unknown resource %s",
+             conn->requested_resource);
+    connection_dir_download_cert_failed(conn, status_code);
   }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
-    const char *msg;
-    int st;
-    log_info(LD_DIR,"Got votes (size %d) from server %s:%d",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (status_code != 200) {
-      log_warn(LD_DIR,
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for an authority's
+ * current networkstatus vote.
+ **/
+static int
+handle_response_fetch_status_vote(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg;
+  int st;
+  log_info(LD_DIR,"Got votes (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
              "Received http status code %d (%s) from server "
              "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port, conn->requested_resource);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    dirvote_add_vote(body, &msg, &st);
-    if (st > 299) {
-      log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
-    } else {
-      log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
-    }
+    return -1;
   }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
-    const char *msg = NULL;
-    log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (status_code != 200) {
-      log_warn(LD_DIR,
+  dirvote_add_vote(body, &msg, &st);
+  if (st > 299) {
+    log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
+  } else {
+    log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
+  }
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for the signatures
+ * that an authority knows about on a given consensus.
+ **/
+static int
+handle_response_fetch_detached_signatures(dir_connection_t *conn,
+                                          const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  const char *msg = NULL;
+  log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (status_code != 200) {
+    log_warn(LD_DIR,
         "Received http status code %d (%s) from server '%s:%d' while fetching "
         "\"/tor/status-vote/next/consensus-signatures.z\".",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return -1;
-    }
-    if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
-      log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
-               conn->base_.address, conn->base_.port, msg?msg:"???");
-    }
+        status_code, escaped(reason), conn->base_.address,
+        conn->base_.port);
+    return -1;
+  }
+  if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
+    log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
+             conn->base_.address, conn->base_.port, msg?msg:"???");
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
-      conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
-    int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
-    smartlist_t *which = NULL;
-    int n_asked_for = 0;
-    int descriptor_digests = conn->requested_resource &&
-                             !strcmpstart(conn->requested_resource,"d/");
-    log_info(LD_DIR,"Received %s (size %d) from server '%s:%d'",
-             was_ei ? "extra server info" : "server info",
-             (int)body_len, conn->base_.address, conn->base_.port);
-    if (conn->requested_resource &&
-        (!strcmpstart(conn->requested_resource,"d/") ||
-         !strcmpstart(conn->requested_resource,"fp/"))) {
-      which = smartlist_new();
-      dir_split_resource_into_fingerprints(conn->requested_resource +
-                                             (descriptor_digests ? 2 : 3),
-                                           which, NULL, 0);
-      n_asked_for = smartlist_len(which);
-    }
-    if (status_code != 200) {
-      int dir_okay = status_code == 404 ||
-        (status_code == 400 && !strcmp(reason, "Servers unavailable."));
-      /* 404 means that it didn't have them; no big deal.
-       * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
-      log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
-             "Received http status code %d (%s) from server '%s:%d' "
-             "while fetching \"/tor/server/%s\". I'll try again soon.",
-             status_code, escaped(reason), conn->base_.address,
-             conn->base_.port, conn->requested_resource);
-      if (!which) {
-        connection_dir_download_routerdesc_failed(conn);
-      } else {
-        dir_routerdesc_download_failed(which, status_code,
-                                       conn->router_purpose,
-                                       was_ei, descriptor_digests);
-        SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-        smartlist_free(which);
-      }
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return dir_okay ? 0 : -1;
-    }
-    /* Learn the routers, assuming we requested by fingerprint or "all"
-     * or "authority".
-     *
-     * We use "authority" to fetch our own descriptor for
-     * testing, and to fetch bridge descriptors for bootstrapping. Ignore
-     * the output of "authority" requests unless we are using bridges,
-     * since otherwise they'll be the response from reachability tests,
-     * and we don't really want to add that to our routerlist. */
-    if (which || (conn->requested_resource &&
-                  (!strcmpstart(conn->requested_resource, "all") ||
-                   (!strcmpstart(conn->requested_resource, "authority") &&
-                    get_options()->UseBridges)))) {
-      /* as we learn from them, we remove them from 'which' */
-      if (was_ei) {
-        router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
-                                          descriptor_digests);
-      } else {
-        //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
-        //                       descriptor_digests, conn->router_purpose);
-        if (load_downloaded_routers(body, which, descriptor_digests,
-                                conn->router_purpose,
-                                conn->base_.address))
-          directory_info_has_arrived(now, 0, 0);
-      }
-    }
-    if (which) { /* mark remaining ones as failed */
-      log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
-               n_asked_for-smartlist_len(which), n_asked_for,
-               was_ei ? "extra-info documents" : "router descriptors",
-               conn->base_.address, (int)conn->base_.port);
-      if (smartlist_len(which)) {
-        dir_routerdesc_download_failed(which, status_code,
-                                       conn->router_purpose,
-                                       was_ei, descriptor_digests);
-      }
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-    }
-    if (directory_conn_is_self_reachability_test(conn))
-      router_dirport_found_reachable();
-  }
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
-    smartlist_t *which = NULL;
-    log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
-             "size %d) from server '%s:%d'",
-             status_code, (int)body_len, conn->base_.address,
-             conn->base_.port);
-    tor_assert(conn->requested_resource &&
-               !strcmpstart(conn->requested_resource, "d/"));
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of server
+ * descriptors or an extrainfo documents.
+ **/
+static int
+handle_response_fetch_desc(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
+             conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
+  smartlist_t *which = NULL;
+  int n_asked_for = 0;
+  int descriptor_digests = conn->requested_resource &&
+    !strcmpstart(conn->requested_resource,"d/");
+  log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'",
+           was_ei ? "extra server info" : "server info",
+           (int)body_len, conn->base_.address, conn->base_.port);
+  if (conn->requested_resource &&
+      (!strcmpstart(conn->requested_resource,"d/") ||
+       !strcmpstart(conn->requested_resource,"fp/"))) {
     which = smartlist_new();
-    dir_split_resource_into_fingerprints(conn->requested_resource+2,
-                                         which, NULL,
-                                         DSR_DIGEST256|DSR_BASE64);
-    if (status_code != 200) {
-      log_info(LD_DIR, "Received status code %d (%s) from server "
-               "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
-               "soon.",
-               status_code, escaped(reason), conn->base_.address,
-               (int)conn->base_.port, conn->requested_resource);
-      dir_microdesc_download_failed(which, status_code);
+    dir_split_resource_into_fingerprints(conn->requested_resource +
+                                         (descriptor_digests ? 2 : 3),
+                                         which, NULL, 0);
+    n_asked_for = smartlist_len(which);
+  }
+  if (status_code != 200) {
+    int dir_okay = status_code == 404 ||
+      (status_code == 400 && !strcmp(reason, "Servers unavailable."));
+    /* 404 means that it didn't have them; no big deal.
+     * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
+    log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
+           "Received http status code %d (%s) from server '%s:%d' "
+           "while fetching \"/tor/server/%s\". I'll try again soon.",
+           status_code, escaped(reason), conn->base_.address,
+           conn->base_.port, conn->requested_resource);
+    if (!which) {
+      connection_dir_download_routerdesc_failed(conn);
+    } else {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
       SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
       smartlist_free(which);
-      tor_free(body); tor_free(headers); tor_free(reason);
-      return 0;
+    }
+    return dir_okay ? 0 : -1;
+  }
+  /* Learn the routers, assuming we requested by fingerprint or "all"
+   * or "authority".
+   *
+   * We use "authority" to fetch our own descriptor for
+   * testing, and to fetch bridge descriptors for bootstrapping. Ignore
+   * the output of "authority" requests unless we are using bridges,
+   * since otherwise they'll be the response from reachability tests,
+   * and we don't really want to add that to our routerlist. */
+  if (which || (conn->requested_resource &&
+                (!strcmpstart(conn->requested_resource, "all") ||
+                 (!strcmpstart(conn->requested_resource, "authority") &&
+                  get_options()->UseBridges)))) {
+    /* as we learn from them, we remove them from 'which' */
+    if (was_ei) {
+      router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
+                                        descriptor_digests);
     } else {
-      smartlist_t *mds;
-      mds = microdescs_add_to_cache(get_microdesc_cache(),
-                                    body, body+body_len, SAVED_NOWHERE, 0,
-                                    now, which);
-      if (smartlist_len(which)) {
-        /* Mark remaining ones as failed. */
-        dir_microdesc_download_failed(which, status_code);
-      }
-      if (mds && smartlist_len(mds)) {
-        control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
-                                count_loading_descriptors_progress());
-        directory_info_has_arrived(now, 0, 1);
+      //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
+      //                       descriptor_digests, conn->router_purpose);
+      if (load_downloaded_routers(body, which, descriptor_digests,
+                                  conn->router_purpose,
+                                  conn->base_.address)) {
+        time_t now = approx_time();
+        directory_info_has_arrived(now, 0, 0);
       }
-      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
-      smartlist_free(which);
-      smartlist_free(mds);
     }
   }
+  if (which) { /* mark remaining ones as failed */
+    log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
+             n_asked_for-smartlist_len(which), n_asked_for,
+             was_ei ? "extra-info documents" : "router descriptors",
+             conn->base_.address, (int)conn->base_.port);
+    if (smartlist_len(which)) {
+      dir_routerdesc_download_failed(which, status_code,
+                                     conn->router_purpose,
+                                     was_ei, descriptor_digests);
+    }
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+  }
+  if (directory_conn_is_self_reachability_test(conn))
+    router_dirport_found_reachable();
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR) {
-    switch (status_code) {
-      case 200: {
-          dir_server_t *ds =
-            router_get_trusteddirserver_by_digest(conn->identity_digest);
-          char *rejected_hdr = http_get_header(headers,
-                                               "X-Descriptor-Not-New: ");
-          if (rejected_hdr) {
-            if (!strcmp(rejected_hdr, "Yes")) {
-              log_info(LD_GENERAL,
-                       "Authority '%s' declined our descriptor (not new)",
-                       ds->nickname);
-              /* XXXX use this information; be sure to upload next one
-               * sooner. -NM */
-              /* XXXX++ On further thought, the task above implies that we're
-               * basing our regenerate-descriptor time on when we uploaded the
-               * last descriptor, not on the published time of the last
-               * descriptor.  If those are different, that's a bad thing to
-               * do. -NM */
-            }
-            tor_free(rejected_hdr);
-          }
-          log_info(LD_GENERAL,"eof (status 200) after uploading server "
-                   "descriptor: finished.");
-          control_event_server_status(
-                      LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
-                      conn->base_.address, conn->base_.port);
-
-          ds->has_accepted_serverdesc = 1;
-          if (directories_have_accepted_server_descriptor())
-            control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
-        }
-        break;
-      case 400:
-        log_warn(LD_GENERAL,"http status 400 (%s) response from "
-                 "dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        control_event_server_status(LOG_WARN,
-                      "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
-                      conn->base_.address, conn->base_.port, escaped(reason));
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "descriptor to server '%s:%d').",
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a group of
+ * microdescriptors
+ **/
+STATIC int
+handle_response_fetch_microdesc(dir_connection_t *conn,
+                                const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+  smartlist_t *which = NULL;
+  log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
+           "body size %d) from server '%s:%d'",
+           status_code, (int)body_len, conn->base_.address,
+           conn->base_.port);
+  tor_assert(conn->requested_resource &&
+             !strcmpstart(conn->requested_resource, "d/"));
+  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  which = smartlist_new();
+  dir_split_resource_into_fingerprints(conn->requested_resource+2,
+                                       which, NULL,
+                                       DSR_DIGEST256|DSR_BASE64);
+  if (status_code != 200) {
+    log_info(LD_DIR, "Received status code %d (%s) from server "
+             "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
+             "soon.",
              status_code, escaped(reason), conn->base_.address,
-             conn->base_.port);
-        break;
+             (int)conn->base_.port, conn->requested_resource);
+    dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    return 0;
+  } else {
+    smartlist_t *mds;
+    time_t now = approx_time();
+    mds = microdescs_add_to_cache(get_microdesc_cache(),
+                                  body, body+body_len, SAVED_NOWHERE, 0,
+                                  now, which);
+    if (smartlist_len(which)) {
+      /* Mark remaining ones as failed. */
+      dir_microdesc_download_failed(which, status_code, conn->identity_digest);
+    }
+    if (mds && smartlist_len(mds)) {
+      control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
+                              count_loading_descriptors_progress());
+      directory_info_has_arrived(now, 0, 1);
     }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
+    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
+    smartlist_free(which);
+    smartlist_free(mds);
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE) {
-    switch (status_code) {
-      case 200: {
-        log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload our
+ * router descriptor.
+ **/
+static int
+handle_response_upload_dir(dir_connection_t *conn,
+                           const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *headers = args->headers;
+
+  switch (status_code) {
+  case 200: {
+    dir_server_t *ds =
+      router_get_trusteddirserver_by_digest(conn->identity_digest);
+    char *rejected_hdr = http_get_header(headers,
+                                         "X-Descriptor-Not-New: ");
+    if (rejected_hdr) {
+      if (!strcmp(rejected_hdr, "Yes")) {
+        log_info(LD_GENERAL,
+                 "Authority '%s' declined our descriptor (not new)",
+                 ds->nickname);
+        /* XXXX use this information; be sure to upload next one
+         * sooner. -NM */
+        /* XXXX++ On further thought, the task above implies that we're
+         * basing our regenerate-descriptor time on when we uploaded the
+         * last descriptor, not on the published time of the last
+         * descriptor.  If those are different, that's a bad thing to
+         * do. -NM */
+      }
+      tor_free(rejected_hdr);
+    }
+    log_info(LD_GENERAL,"eof (status 200) after uploading server "
+             "descriptor: finished.");
+    control_event_server_status(
+                   LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
                    conn->base_.address, conn->base_.port);
-        }
-        break;
-      case 400:
-        log_warn(LD_DIR,"http status 400 (%s) response after uploading "
-                 "vote to dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "vote to server '%s:%d').",
+
+    ds->has_accepted_serverdesc = 1;
+    if (directories_have_accepted_server_descriptor())
+      control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
+  }
+    break;
+  case 400:
+    log_warn(LD_GENERAL,"http status 400 (%s) response from "
+             "dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    control_event_server_status(LOG_WARN,
+                    "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
+                    conn->base_.address, conn->base_.port, escaped(reason));
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "descriptor to server '%s:%d'. Possibly the server is "
+             "misconfigured?",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
-        break;
-    }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
+    break;
   }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES) {
-    switch (status_code) {
-      case 200: {
-        log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
-                   conn->base_.address, conn->base_.port);
-        }
-        break;
-      case 400:
-        log_warn(LD_DIR,"http status 400 (%s) response after uploading "
-                 "signatures to dirserver '%s:%d'. Please correct.",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        break;
-      default:
-        log_warn(LD_GENERAL,
-             "http status %d (%s) reason unexpected while uploading "
-             "signatures to server '%s:%d').",
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * own networkstatus vote.
+ **/
+static int
+handle_response_upload_vote(dir_connection_t *conn,
+                            const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "vote to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "vote to server '%s:%d'.",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
-        break;
-    }
-    /* return 0 in all cases, since we don't want to mark any
-     * dirservers down just because they don't like us. */
-  }
-
-  if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) {
-    #define SEND_HS_DESC_FAILED_EVENT(reason) ( \
-      control_event_hs_descriptor_failed(conn->rend_data, \
-                                         conn->identity_digest, \
-                                         reason) )
-    #define SEND_HS_DESC_FAILED_CONTENT() ( \
-      control_event_hs_descriptor_content(conn->rend_data->onion_address, \
-                                          conn->requested_resource, \
-                                          conn->identity_digest, \
-                                          NULL) )
-    tor_assert(conn->rend_data);
-    log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d "
-             "(%s))",
-             (int)body_len, status_code, escaped(reason));
-    switch (status_code) {
-      case 200:
-      {
-        rend_cache_entry_t *entry = NULL;
-
-        if (rend_cache_store_v2_desc_as_client(body,
-            conn->requested_resource, conn->rend_data, &entry) < 0) {
-          log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
-                   "Retrying at another directory.");
-          /* We'll retry when connection_about_to_close_connection()
-           * cleans this dir conn up. */
-          SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
-          SEND_HS_DESC_FAILED_CONTENT();
-        } else {
-          char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
-          /* Should never be NULL here if we found the descriptor. */
-          tor_assert(entry);
-          rend_get_service_id(entry->parsed->pk, service_id);
-
-          /* success. notify pending connections about this. */
-          log_info(LD_REND, "Successfully fetched v2 rendezvous "
-                   "descriptor.");
-          control_event_hs_descriptor_received(service_id,
-                                               conn->rend_data,
-                                               conn->identity_digest);
-          control_event_hs_descriptor_content(service_id,
-                                              conn->requested_resource,
-                                              conn->identity_digest,
-                                              body);
-          conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
-          rend_client_desc_trynow(service_id);
-          memwipe(service_id, 0, sizeof(service_id));
-        }
-        break;
-      }
-      case 404:
-        /* Not there. We'll retry when
-         * connection_about_to_close_connection() cleans this conn up. */
-        log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
-                         "Retrying at another directory.");
-        SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-      case 400:
-        log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
-                 "http status 400 (%s). Dirserver didn't like our "
-                 "v2 rendezvous query? Retrying at another directory.",
-                 escaped(reason));
-        SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
-        SEND_HS_DESC_FAILED_CONTENT();
-        break;
-      default:
-        log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
-                 "http status %d (%s) response unexpected while "
-                 "fetching v2 hidden service descriptor (server '%s:%d'). "
-                 "Retrying at another directory.",
-                 status_code, escaped(reason), conn->base_.address,
-                 conn->base_.port);
-        SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to POST request to upload our
+ * view of the signatures on the current consensus.
+ **/
+static int
+handle_response_upload_signatures(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+  switch (status_code) {
+  case 200: {
+    log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
+               conn->base_.address, conn->base_.port);
+  }
+    break;
+  case 400:
+    log_warn(LD_DIR,"http status 400 (%s) response after uploading "
+             "signatures to dirserver '%s:%d'. Please correct.",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    break;
+  default:
+    log_warn(LD_GENERAL,
+             "HTTP status %d (%s) was unexpected while uploading "
+             "signatures to server '%s:%d'.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    break;
+  }
+  /* return 0 in all cases, since we don't want to mark any
+   * dirservers down just because they don't like us. */
+
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a request for a v2 hidden service
+ * descriptor.
+ **/
+static int
+handle_response_fetch_renddesc_v2(dir_connection_t *conn,
+                                  const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+  const char *body = args->body;
+  const size_t body_len = args->body_len;
+
+#define SEND_HS_DESC_FAILED_EVENT(reason)                               \
+  (control_event_hs_descriptor_failed(conn->rend_data,                  \
+                                      conn->identity_digest,            \
+                                      reason))
+#define SEND_HS_DESC_FAILED_CONTENT()                                   \
+  (control_event_hs_descriptor_content(                                 \
+                                rend_data_get_address(conn->rend_data), \
+                                conn->requested_resource,               \
+                                conn->identity_digest,                  \
+                                NULL))
+
+  tor_assert(conn->rend_data);
+  log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d "
+           "(%s))",
+           (int)body_len, status_code, escaped(reason));
+  switch (status_code) {
+  case 200:
+    {
+      rend_cache_entry_t *entry = NULL;
+
+      if (rend_cache_store_v2_desc_as_client(body,
+                                             conn->requested_resource,
+                                             conn->rend_data, &entry) < 0) {
+        log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
+                 "Retrying at another directory.");
+        /* We'll retry when connection_about_to_close_connection()
+         * cleans this dir conn up. */
+        SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
         SEND_HS_DESC_FAILED_CONTENT();
-        break;
+      } else {
+        char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
+        /* Should never be NULL here if we found the descriptor. */
+        tor_assert(entry);
+        rend_get_service_id(entry->parsed->pk, service_id);
+
+        /* success. notify pending connections about this. */
+        log_info(LD_REND, "Successfully fetched v2 rendezvous "
+                 "descriptor.");
+        control_event_hs_descriptor_received(service_id,
+                                             conn->rend_data,
+                                             conn->identity_digest);
+        control_event_hs_descriptor_content(service_id,
+                                            conn->requested_resource,
+                                            conn->identity_digest,
+                                            body);
+        conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
+        rend_client_desc_trynow(service_id);
+        memwipe(service_id, 0, sizeof(service_id));
+      }
+      break;
     }
+  case 404:
+    /* Not there. We'll retry when
+     * connection_about_to_close_connection() cleans this conn up. */
+    log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
+             "Retrying at another directory.");
+    SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  case 400:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status 400 (%s). Dirserver didn't like our "
+             "v2 rendezvous query? Retrying at another directory.",
+             escaped(reason));
+    SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
+  default:
+    log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
+             "http status %d (%s) response unexpected while "
+             "fetching v2 hidden service descriptor (server '%s:%d'). "
+             "Retrying at another directory.",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
+    SEND_HS_DESC_FAILED_CONTENT();
+    break;
   }
 
-  if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) {
-    #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \
-      control_event_hs_descriptor_upload_failed( \
-        conn->identity_digest, \
-        conn->rend_data->onion_address, \
-        reason) )
-    log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
-             "(%s))",
-             status_code, escaped(reason));
-    /* Without the rend data, we'll have a problem identifying what has been
-     * uploaded for which service. */
-    tor_assert(conn->rend_data);
-    switch (status_code) {
-      case 200:
-        log_info(LD_REND,
-                 "Uploading rendezvous descriptor: finished with status "
-                 "200 (%s)", escaped(reason));
-        control_event_hs_descriptor_uploaded(conn->identity_digest,
-                                             conn->rend_data->onion_address);
-        rend_service_desc_has_uploaded(conn->rend_data);
-        break;
-      case 400:
-        log_warn(LD_REND,"http status 400 (%s) response from dirserver "
-                 "'%s:%d'. Malformed rendezvous descriptor?",
-                 escaped(reason), conn->base_.address, conn->base_.port);
-        SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
-        break;
-      default:
-        log_warn(LD_REND,"http status %d (%s) response unexpected (server "
-                 "'%s:%d').",
-                 status_code, escaped(reason), conn->base_.address,
-                 conn->base_.port);
-        SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
-        break;
-    }
+  return 0;
+}
+
+/**
+ * Handler function: processes a response to a POST request to upload a v2
+ * hidden service descriptor.
+ **/
+static int
+handle_response_upload_renddesc_v2(dir_connection_t *conn,
+                                   const response_handler_args_t *args)
+{
+  tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+  const int status_code = args->status_code;
+  const char *reason = args->reason;
+
+#define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason)                        \
+    (control_event_hs_descriptor_upload_failed(                         \
+                                conn->identity_digest,                  \
+                                rend_data_get_address(conn->rend_data), \
+                                reason))
+
+  log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
+           "(%s))",
+           status_code, escaped(reason));
+  /* Without the rend data, we'll have a problem identifying what has been
+   * uploaded for which service. */
+  tor_assert(conn->rend_data);
+  switch (status_code) {
+  case 200:
+    log_info(LD_REND,
+             "Uploading rendezvous descriptor: finished with status "
+             "200 (%s)", escaped(reason));
+    control_event_hs_descriptor_uploaded(conn->identity_digest,
+                                   rend_data_get_address(conn->rend_data));
+    rend_service_desc_has_uploaded(conn->rend_data);
+    break;
+  case 400:
+    log_warn(LD_REND,"http status 400 (%s) response from dirserver "
+             "'%s:%d'. Malformed rendezvous descriptor?",
+             escaped(reason), conn->base_.address, conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
+    break;
+  default:
+    log_warn(LD_REND,"http status %d (%s) response unexpected (server "
+             "'%s:%d').",
+             status_code, escaped(reason), conn->base_.address,
+             conn->base_.port);
+    SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
+    break;
   }
-  tor_free(body); tor_free(headers); tor_free(reason);
+
   return 0;
 }
 
@@ -2542,7 +3270,8 @@ connection_dir_about_to_close(dir_connection_t *dir_conn)
    * refetching is unnecessary.) */
   if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
       dir_conn->rend_data &&
-      strlen(dir_conn->rend_data->onion_address) == REND_SERVICE_ID_LEN_BASE32)
+      strlen(rend_data_get_address(dir_conn->rend_data)) ==
+             REND_SERVICE_ID_LEN_BASE32)
     rend_client_refetch_v2_renddesc(dir_conn->rend_data);
 }
 
@@ -2553,14 +3282,12 @@ static void
 write_http_status_line(dir_connection_t *conn, int status,
                        const char *reason_phrase)
 {
-  char buf[256];
-  if (tor_snprintf(buf, sizeof(buf), "HTTP/1.0 %d %s\r\n\r\n",
-      status, reason_phrase ? reason_phrase : "OK") < 0) {
-    log_warn(LD_BUG,"status line too long.");
-    return;
-  }
+  char *buf = NULL;
+  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n\r\n",
+               status, reason_phrase ? reason_phrase : "OK");
   log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
   connection_write_to_buf(buf, strlen(buf), TO_CONN(conn));
+  tor_free(buf);
 }
 
 /** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
@@ -2639,14 +3366,114 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
 /** As write_http_response_header_impl, but sets encoding and content-typed
  * based on whether the response will be <b>compressed</b> or not. */
 static void
-write_http_response_header(dir_connection_t *conn, ssize_t length,
-                           int compressed, long cache_lifetime)
+write_http_response_headers(dir_connection_t *conn, ssize_t length,
+                            compress_method_t method,
+                            const char *extra_headers, long cache_lifetime)
 {
+  const char *methodname = compression_method_get_name(method);
+  const char *doctype;
+  if (method == NO_METHOD)
+    doctype = "text/plain";
+  else
+    doctype = "application/octet-stream";
   write_http_response_header_impl(conn, length,
-                          compressed?"application/octet-stream":"text/plain",
-                          compressed?"deflate":"identity",
-                             NULL,
-                             cache_lifetime);
+                                  doctype,
+                                  methodname,
+                                  extra_headers,
+                                  cache_lifetime);
+}
+
+/** As write_http_response_headers, but assumes extra_headers is NULL */
+static void
+write_http_response_header(dir_connection_t *conn, ssize_t length,
+                           compress_method_t method,
+                           long cache_lifetime)
+{
+  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
+}
+
+/** Array of compression methods to use (if supported) for serving
+ * precompressed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_precompressed[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of compression methods to use (if supported) for serving
+ * streamed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_streaming_compression[] = {
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Array of allowed compression methods to use (if supported) when receiving a
+ * response from a request that was required to be anonymous. */
+static compress_method_t client_meth_allowed_anonymous_compression[] = {
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
+ * and convert them to a bitfield where compression method x is supported if
+ * and only if 1 &lt;&lt; x is set in the bitfield. */
+STATIC unsigned
+parse_accept_encoding_header(const char *h)
+{
+  unsigned result = (1u << NO_METHOD);
+  smartlist_t *methods = smartlist_new();
+  smartlist_split_string(methods, h, ",",
+             SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
+    compress_method_t method = compression_method_get_by_name(m);
+    if (method != UNKNOWN_METHOD) {
+      tor_assert(((unsigned)method) < 8*sizeof(unsigned));
+      result |= (1u << method);
+    }
+  } SMARTLIST_FOREACH_END(m);
+  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
+    tor_free(m);
+  } SMARTLIST_FOREACH_END(m);
+  smartlist_free(methods);
+  return result;
+}
+
+/** Array of compression methods to use (if supported) for requesting
+ * compressed data, ordered from best to worst. */
+static compress_method_t client_meth_pref[] = {
+  LZMA_METHOD,
+  ZSTD_METHOD,
+  ZLIB_METHOD,
+  GZIP_METHOD,
+  NO_METHOD
+};
+
+/** Return a newly allocated string containing a comma separated list of
+ * supported encodings. */
+STATIC char *
+accept_encoding_header(void)
+{
+  smartlist_t *methods = smartlist_new();
+  char *header = NULL;
+  compress_method_t method;
+  unsigned i;
+
+  for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
+    method = client_meth_pref[i];
+    if (tor_compress_supports_method(method))
+      smartlist_add(methods, (char *)compression_method_get_name(method));
+  }
+
+  header = smartlist_join_strings(methods, ", ", 0, NULL);
+  smartlist_free(methods);
+
+  return header;
 }
 
 /** Decide whether a client would accept the consensus we have.
@@ -2664,48 +3491,46 @@ write_http_response_header(dir_connection_t *conn, ssize_t length,
  * consensus, 0 otherwise.
  */
 int
-client_likes_consensus(networkstatus_t *v, const char *want_url)
+client_likes_consensus(const struct consensus_cache_entry_t *ent,
+                       const char *want_url)
 {
-  smartlist_t *want_authorities = smartlist_new();
+  smartlist_t *voters = smartlist_new();
   int need_at_least;
   int have = 0;
 
+  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
+    smartlist_free(voters);
+    return 1; // We don't know the voters; assume the client won't mind. */
+  }
+
+  smartlist_t *want_authorities = smartlist_new();
   dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
   need_at_least = smartlist_len(want_authorities)/2+1;
-  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) {
-    char want_digest[DIGEST_LEN];
-    size_t want_len = strlen(d)/2;
-    if (want_len > DIGEST_LEN)
-      want_len = DIGEST_LEN;
-
-    if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2)
-                      != (int) want_len) {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-             "Failed to decode requested authority digest %s.", escaped(d));
-      continue;
-    };
 
-    SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) {
-      if (smartlist_len(vi->sigs) &&
-          tor_memeq(vi->identity_digest, want_digest, want_len)) {
+  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
+
+    SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
+      if (!strcasecmpstart(digest, want_digest)) {
         have++;
         break;
       };
-    } SMARTLIST_FOREACH_END(vi);
+    } SMARTLIST_FOREACH_END(digest);
 
     /* early exit, if we already have enough */
     if (have >= need_at_least)
       break;
-  } SMARTLIST_FOREACH_END(d);
+  } SMARTLIST_FOREACH_END(want_digest);
 
   SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
   smartlist_free(want_authorities);
+  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
+  smartlist_free(voters);
   return (have >= need_at_least);
 }
 
 /** Return the compression level we should use for sending a compressed
  * response of size <b>n_bytes</b>. */
-STATIC zlib_compression_level_t
+STATIC compression_level_t
 choose_compression_level(ssize_t n_bytes)
 {
   if (! have_been_under_memory_pressure()) {
@@ -2723,8 +3548,9 @@ choose_compression_level(ssize_t n_bytes)
 
 /** Information passed to handle a GET request. */
 typedef struct get_handler_args_t {
-  /** True if the client asked for compressed data. */
-  int compressed;
+  /** Bitmask of compression methods that the client said (or implied) it
+   * supported. */
+  unsigned compression_supported;
   /** If nonzero, the time included an if-modified-since header with this
    * value. */
   time_t if_modified_since;
@@ -2762,8 +3588,8 @@ static int handle_get_descriptor(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_keys(dir_connection_t *conn,
                                 const get_handler_args_t *args);
-static int handle_get_rendezvous2(dir_connection_t *conn,
-                                const get_handler_args_t *args);
+static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                                       const get_handler_args_t *args);
 static int handle_get_robots(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
@@ -2779,7 +3605,8 @@ static const url_table_ent_t url_table[] = {
   { "/tor/server/", 1, handle_get_descriptor },
   { "/tor/extra/", 1, handle_get_descriptor },
   { "/tor/keys/", 1, handle_get_keys },
-  { "/tor/rendezvous2/", 1, handle_get_rendezvous2 },
+  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
+  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
   { "/tor/robots.txt", 0, handle_get_robots },
   { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
   { NULL, 0, NULL },
@@ -2791,14 +3618,15 @@ static const url_table_ent_t url_table[] = {
  * conn-\>outbuf.  If the request is unrecognized, send a 404.
  * Return 0 if we handled this successfully, or -1 if we need to close
  * the connection. */
-STATIC int
-directory_handle_command_get(dir_connection_t *conn, const char *headers,
-                             const char *req_body, size_t req_body_len)
+MOCK_IMPL(STATIC int,
+directory_handle_command_get,(dir_connection_t *conn, const char *headers,
+                              const char *req_body, size_t req_body_len))
 {
   char *url, *url_mem, *header;
   time_t if_modified_since = 0;
-  int compressed;
+  int zlib_compressed_in_url;
   size_t url_len;
+  unsigned compression_methods_supported;
 
   /* We ignore the body of a GET request. */
   (void)req_body;
@@ -2829,17 +3657,31 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers,
 
   url_mem = url;
   url_len = strlen(url);
-  compressed = url_len > 2 && !strcmp(url+url_len-2, ".z");
-  if (compressed) {
+
+  zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
+  if (zlib_compressed_in_url) {
     url[url_len-2] = '\0';
     url_len -= 2;
   }
 
+  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
+    compression_methods_supported = parse_accept_encoding_header(header);
+    tor_free(header);
+  } else {
+    compression_methods_supported = (1u << NO_METHOD);
+  }
+  if (zlib_compressed_in_url) {
+    compression_methods_supported |= (1u << ZLIB_METHOD);
+  }
+
+  /* Remove all methods that we don't both support. */
+  compression_methods_supported &= tor_compress_get_supported_method_bitmask();
+
   get_handler_args_t args;
   args.url = url;
   args.headers = headers;
   args.if_modified_since = if_modified_since;
-  args.compressed = compressed;
+  args.compression_supported = compression_methods_supported;
 
   int i, result = -1;
   for (i = 0; url_table[i].string; ++i) {
@@ -2889,134 +3731,523 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
   return 0;
 }
 
-/** Helper function for GET /tor/status-vote/current/consensus
+/** Warn that the cached consensus <b>consensus</b> of type
+ * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
+ * warning to avoid logging an entry on every request.
  */
-static int
-handle_get_current_consensus(dir_connection_t *conn,
-                             const get_handler_args_t *args)
+static void
+warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now)
 {
-  const char *url = args->url;
-  const int compressed = args->compressed;
-  const time_t if_modified_since = args->if_modified_since;
+#define TOO_OLD_WARNING_INTERVAL (60*60)
+  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+  char timestamp[ISO_TIME_LEN+1];
+  time_t valid_until;
+  char *dupes;
 
-  {
-    /* v3 network status fetch. */
-    smartlist_t *dir_fps = smartlist_new();
-    long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
-
-    if (1) {
-      networkstatus_t *v;
-      time_t now = time(NULL);
-      const char *want_fps = NULL;
-      char *flavor = NULL;
-      int flav = FLAV_NS;
-      #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
-      #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
-      /* figure out the flavor if any, and who we wanted to sign the thing */
-      if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
-        const char *f, *cp;
-        f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
-        cp = strchr(f, '/');
-        if (cp) {
-          want_fps = cp+1;
-          flavor = tor_strndup(f, cp-f);
-        } else {
-          flavor = tor_strdup(f);
-        }
-        flav = networkstatus_parse_flavor_name(flavor);
-        if (flav < 0)
-          flav = FLAV_NS;
-      } else {
-        if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
-          want_fps = url+strlen(CONSENSUS_URL_PREFIX);
-      }
+  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
+    return;
 
-      v = networkstatus_get_latest_consensus_by_flavor(flav);
+  if ((dupes = rate_limit_log(&warned, now))) {
+    format_local_iso_time(timestamp, valid_until);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
+             "serve it to clients. It was valid until %s local time and we "
+             "continued to serve it for up to 24 hours after it expired.%s",
+             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+    tor_free(dupes);
+  }
+}
 
-      if (v && want_fps &&
-          !client_likes_consensus(v, want_fps)) {
-        write_http_status_line(conn, 404, "Consensus not signed by sufficient "
-                                          "number of requested authorities");
-        smartlist_free(dir_fps);
-        geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
-        tor_free(flavor);
-        goto done;
-      }
+/**
+ * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
+ * <b>digest</b>. Return 0 on success.  On failure, report that the hash came
+ * from <b>location</b>, report that we are taking <b>action</b> with it, and
+ * return -1.
+ */
+static int
+parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
+                    const char *action)
+{
+  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
+      DIGEST256_LEN) {
+    return 0;
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+           "%s contained bogus digest %s; %s.",
+           location, escaped(hex), action);
+    return -1;
+  }
+}
 
-      {
-        char *fp = tor_malloc_zero(DIGEST_LEN);
-        if (flavor)
-          strlcpy(fp, flavor, DIGEST_LEN);
-        tor_free(flavor);
-        smartlist_add(dir_fps, fp);
+/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
+ * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * hex-encoded digest listed in that header and return 0.  Otherwise return
+ * -1.  */
+static int
+parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
+{
+  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
+  if (hdr == NULL) {
+    return -1;
+  }
+  smartlist_t *hex_digests = smartlist_new();
+  *digests_out = smartlist_new();
+  smartlist_split_string(hex_digests, hdr, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
+    uint8_t digest[DIGEST256_LEN];
+    if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
+                             "ignoring")) {
+      smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
+    }
+  } SMARTLIST_FOREACH_END(hex);
+  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
+  smartlist_free(hex_digests);
+  tor_free(hdr);
+  return 0;
+}
+
+/** Fallback compression method.  The fallback compression method is used in
+ * case a client requests a non-compressed document. We only store compressed
+ * documents, so we use this compression method to fetch the document and let
+ * the spooling system do the streaming decompression.
+ */
+#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
+
+/**
+ * Try to find the best consensus diff possible in order to serve a client
+ * request for a diff from one of the consensuses in <b>digests</b> to the
+ * current consensus of flavor <b>flav</b>.  The client supports the
+ * compression methods listed in the <b>compression_methods</b> bitfield:
+ * place the method chosen (if any) into <b>compression_used_out</b>.
+ */
+static struct consensus_cache_entry_t *
+find_best_diff(const smartlist_t *digests, int flav,
+               unsigned compression_methods,
+               compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    unsigned u;
+    for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+      compress_method_t method = srv_meth_pref_precompressed[u];
+      if (0 == (compression_methods & (1u<<method)))
+        continue; // client doesn't like this one, or we don't have it.
+      if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
+                                     diff_from, DIGEST256_LEN,
+                                     method) == CONSDIFF_AVAILABLE) {
+        tor_assert_nonfatal(result);
+        *compression_used_out = method;
+        return result;
       }
-      lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0;
     }
+  } SMARTLIST_FOREACH_END(diff_from);
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+    if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
+          DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = FALLBACK_COMPRESS_METHOD;
+      return result;
+    }
+  } SMARTLIST_FOREACH_END(diff_from);
 
-    if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
-      write_http_status_line(conn, 503, "Network status object unavailable");
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE);
-      goto done;
+  return NULL;
+}
+
+/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
+ * The prefered set of compression methods should be listed in the
+ * <b>compression_methods</b> bitfield. The compression method chosen (if any)
+ * is stored in <b>compression_used_out</b>. */
+static struct consensus_cache_entry_t *
+find_best_consensus(int flav,
+                    unsigned compression_methods,
+                    compress_method_t *compression_used_out)
+{
+  struct consensus_cache_entry_t *result = NULL;
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+    compress_method_t method = srv_meth_pref_precompressed[u];
+
+    if (0 == (compression_methods & (1u<<method)))
+      continue;
+
+    if (consdiffmgr_find_consensus(&result, flav,
+                                   method) == CONSDIFF_AVAILABLE) {
+      tor_assert_nonfatal(result);
+      *compression_used_out = method;
+      return result;
     }
+  }
 
-    if (!dirserv_remove_old_statuses(dir_fps, if_modified_since)) {
-      write_http_status_line(conn, 404, "Not found");
-      SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
-      goto done;
-    } else if (!smartlist_len(dir_fps)) {
-      write_http_status_line(conn, 304, "Not modified");
-      SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
-      smartlist_free(dir_fps);
-      geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
-      goto done;
+  if (consdiffmgr_find_consensus(&result, flav,
+        FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+    tor_assert_nonfatal(result);
+    *compression_used_out = FALLBACK_COMPRESS_METHOD;
+    return result;
+  }
+
+  return NULL;
+}
+
+/** Try to find the best supported compression method possible from a given
+ * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
+ * compression method could be found. */
+static compress_method_t
+find_best_compression_method(unsigned compression_methods, int stream)
+{
+  unsigned u;
+  compress_method_t *methods;
+  size_t length;
+
+  if (stream) {
+    methods = srv_meth_pref_streaming_compression;
+    length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
+  } else {
+    methods = srv_meth_pref_precompressed;
+    length = ARRAY_LENGTH(srv_meth_pref_precompressed);
+  }
+
+  for (u = 0; u < length; ++u) {
+    compress_method_t method = methods[u];
+    if (compression_methods & (1u<<method))
+      return method;
+  }
+
+  return NO_METHOD;
+}
+
+/** Check if any of the digests in <b>digests</b> matches the latest consensus
+ *  flavor (given in <b>flavor</b>) that we have available. */
+static int
+digest_list_contains_best_consensus(consensus_flavor_t flavor,
+                                    const smartlist_t *digests)
+{
+  const networkstatus_t *ns = NULL;
+
+  if (digests == NULL)
+    return 0;
+
+  ns = networkstatus_get_latest_consensus_by_flavor(flavor);
+
+  if (ns == NULL)
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
+    if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
+      return 1;
+  } SMARTLIST_FOREACH_END(digest);
+
+  return 0;
+}
+
+/** Check if the given compression method is allowed for a connection that is
+ * supposed to be anonymous. Returns 1 if the compression method is allowed,
+ * otherwise 0. */
+STATIC int
+allowed_anonymous_connection_compression_method(compress_method_t method)
+{
+  unsigned u;
+
+  for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
+       ++u) {
+    compress_method_t allowed_method =
+      client_meth_allowed_anonymous_compression[u];
+
+    if (! tor_compress_supports_method(allowed_method))
+      continue;
+
+    if (method == allowed_method)
+      return 1;
+  }
+
+  return 0;
+}
+
+/** Log a warning when a remote server has sent us a document using a
+ * compression method that is not allowed for anonymous directory requests. */
+STATIC void
+warn_disallowed_anonymous_compression_method(compress_method_t method)
+{
+  log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+         "Received a %s HTTP response, which is not "
+         "allowed for anonymous directory requests.",
+         compression_method_get_human_name(method));
+}
+
+/** Encodes the results of parsing a consensus request to figure out what
+ * consensus, and possibly what diffs, the user asked for. */
+typedef struct {
+  /** name of the flavor to retrieve. */
+  char *flavor;
+  /** flavor to retrive, as enum. */
+  consensus_flavor_t flav;
+  /** plus-separated list of authority fingerprints; see
+   * client_likes_consensus(). Aliases the URL in the request passed to
+   * parse_consensus_request(). */
+  const char *want_fps;
+  /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
+   * to return a diff from. */
+  smartlist_t *diff_from_digests;
+  /** If true, never send a full consensus. If there is no diff, send
+   * a 404 instead. */
+  int diff_only;
+} parsed_consensus_request_t;
+
+/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
+ * it is stack-allocated. */
+static void
+parsed_consensus_request_clear(parsed_consensus_request_t *req)
+{
+  if (!req)
+    return;
+  tor_free(req->flavor);
+  if (req->diff_from_digests) {
+    SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(req->diff_from_digests);
+  }
+  memset(req, 0, sizeof(parsed_consensus_request_t));
+}
+
+/**
+ * Parse the URL and relevant headers of <b>args</b> for a current-consensus
+ * request to learn what flavor of consensus we want, what keys it must be
+ * signed with, and what diffs we would accept (or demand) instead. Return 0
+ * on success and -1 on failure.
+ */
+static int
+parse_consensus_request(parsed_consensus_request_t *out,
+                        const get_handler_args_t *args)
+{
+  const char *url = args->url;
+  memset(out, 0, sizeof(parsed_consensus_request_t));
+  out->flav = FLAV_NS;
+
+  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
+  const char CONSENSUS_FLAVORED_PREFIX[] =
+    "/tor/status-vote/current/consensus-";
+
+  /* figure out the flavor if any, and who we wanted to sign the thing */
+  const char *after_flavor = NULL;
+
+  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+    const char *f, *cp;
+    f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+    cp = strchr(f, '/');
+    if (cp) {
+      after_flavor = cp+1;
+      out->flavor = tor_strndup(f, cp-f);
+    } else {
+      out->flavor = tor_strdup(f);
     }
+    int flav = networkstatus_parse_flavor_name(out->flavor);
+    if (flav < 0)
+      flav = FLAV_NS;
+    out->flav = flav;
+  } else {
+    if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+      after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
+  }
 
-    size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed);
-    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
-      log_debug(LD_DIRSERV,
-               "Client asked for network status lists, but we've been "
-               "writing too many bytes lately. Sending 503 Dir busy.");
-      write_http_status_line(conn, 503, "Directory busy, try again later");
-      SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp));
-      smartlist_free(dir_fps);
+  /* see whether we've been asked explicitly for a diff from an older
+   * consensus. (The user might also have said that a diff would be okay,
+   * via X-Or-Diff-From-Consensus */
+  const char DIFF_COMPONENT[] = "diff/";
+  char *diff_hash_in_url = NULL;
+  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
+    after_flavor += strlen(DIFF_COMPONENT);
+    const char *cp = strchr(after_flavor, '/');
+    if (cp) {
+      diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
+      out->want_fps = cp+1;
+    } else {
+      diff_hash_in_url = tor_strdup(after_flavor);
+      out->want_fps = NULL;
+    }
+  } else {
+    out->want_fps = after_flavor;
+  }
 
-      geoip_note_ns_response(GEOIP_REJECT_BUSY);
-      goto done;
+  if (diff_hash_in_url) {
+    uint8_t diff_from[DIGEST256_LEN];
+    out->diff_from_digests = smartlist_new();
+    out->diff_only = 1;
+    int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
+                                  "rejecting");
+    tor_free(diff_hash_in_url);
+    if (ok) {
+      smartlist_add(out->diff_from_digests,
+                    tor_memdup(diff_from, DIGEST256_LEN));
+    } else {
+      return -1;
     }
+  } else {
+    parse_or_diff_from_header(&out->diff_from_digests, args->headers);
+  }
 
-    if (1) {
-      tor_addr_t addr;
-      if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
-        geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
-                               &addr, NULL,
-                               time(NULL));
-        geoip_note_ns_response(GEOIP_SUCCESS);
-        /* Note that a request for a network status has started, so that we
-         * can measure the download time later on. */
-        if (conn->dirreq_id)
-          geoip_start_dirreq(conn->dirreq_id, dlen, DIRREQ_TUNNELED);
-        else
-          geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen,
-                             DIRREQ_DIRECT);
-      }
+  return 0;
+}
+
+/** Helper function for GET /tor/status-vote/current/consensus
+ */
+static int
+handle_get_current_consensus(dir_connection_t *conn,
+                             const get_handler_args_t *args)
+{
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 0);
+  const time_t if_modified_since = args->if_modified_since;
+  int clear_spool = 0;
+
+  /* v3 network status fetch. */
+  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
+  time_t now = time(NULL);
+  parsed_consensus_request_t req;
+
+  if (parse_consensus_request(&req, args) < 0) {
+    write_http_status_line(conn, 404, "Couldn't parse request");
+    goto done;
+  }
+
+  if (digest_list_contains_best_consensus(req.flav,
+                                          req.diff_from_digests)) {
+    write_http_status_line(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+    goto done;
+  }
+
+  struct consensus_cache_entry_t *cached_consensus = NULL;
+
+  compress_method_t compression_used = NO_METHOD;
+  if (req.diff_from_digests) {
+    cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
+                                      args->compression_supported,
+                                      &compression_used);
+  }
+
+  if (req.diff_only && !cached_consensus) {
+    write_http_status_line(conn, 404, "No such diff available");
+    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (! cached_consensus) {
+    cached_consensus = find_best_consensus(req.flav,
+                                           args->compression_supported,
+                                           &compression_used);
+  }
+
+  time_t fresh_until, valid_until;
+  int have_fresh_until = 0, have_valid_until = 0;
+  if (cached_consensus) {
+    have_fresh_until =
+      !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
+    have_valid_until =
+      !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
+  }
+
+  if (cached_consensus && have_valid_until &&
+      !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
+    write_http_status_line(conn, 404, "Consensus is too old");
+    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  }
+
+  if (cached_consensus && req.want_fps &&
+      !client_likes_consensus(cached_consensus, req.want_fps)) {
+    write_http_status_line(conn, 404, "Consensus not signed by sufficient "
+                           "number of requested authorities");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
+    goto done;
+  }
+
+  conn->spool = smartlist_new();
+  clear_spool = 1;
+  {
+    spooled_resource_t *spooled;
+    if (cached_consensus) {
+      spooled = spooled_resource_new_from_cache_entry(cached_consensus);
+      smartlist_add(conn->spool, spooled);
     }
+  }
+
+  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
 
-    write_http_response_header(conn, -1, compressed,
-                               smartlist_len(dir_fps) == 1 ? lifetime : 0);
-    conn->fingerprint_stack = dir_fps;
-    if (! compressed)
-      conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
+  size_t size_guess = 0;
+  int n_expired = 0;
+  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
+                                              compress_method != NO_METHOD,
+                                              &size_guess,
+                                              &n_expired);
 
-    /* Prime the connection with some data. */
-    conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
-    connection_dirserv_flushed_some(conn);
+  if (!smartlist_len(conn->spool) && !n_expired) {
+    write_http_status_line(conn, 404, "Not found");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (!smartlist_len(conn->spool)) {
+    write_http_status_line(conn, 304, "Not modified");
+    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
     goto done;
   }
 
+  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    log_debug(LD_DIRSERV,
+              "Client asked for network status lists, but we've been "
+              "writing too many bytes lately. Sending 503 Dir busy.");
+    write_http_status_line(conn, 503, "Directory busy, try again later");
+    geoip_note_ns_response(GEOIP_REJECT_BUSY);
+    goto done;
+  }
+
+  tor_addr_t addr;
+  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
+    geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
+                           &addr, NULL,
+                           time(NULL));
+    geoip_note_ns_response(GEOIP_SUCCESS);
+    /* Note that a request for a network status has started, so that we
+     * can measure the download time later on. */
+    if (conn->dirreq_id)
+      geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
+    else
+      geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
+                         DIRREQ_DIRECT);
+  }
+
+  /* Use this header to tell caches that the response depends on the
+   * X-Or-Diff-From-Consensus header (or lack thereof). */
+  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
+
+  clear_spool = 0;
+
+  // The compress_method might have been NO_METHOD, but we store the data
+  // compressed. Decompress them using `compression_used`. See fallback code in
+  // find_best_consensus() and find_best_diff().
+  write_http_response_headers(conn, -1,
+                             compress_method == NO_METHOD ?
+                               NO_METHOD : compression_used,
+                             vary_header,
+                             smartlist_len(conn->spool) == 1 ? lifetime : 0);
+
+  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
+    conn->compress_state = tor_compress_new(0, compression_used,
+                                            HIGH_COMPRESSION);
+
+  /* Prime the connection with some data. */
+  const int initial_flush_result = connection_dirserv_flushed_some(conn);
+  tor_assert_nonfatal(initial_flush_result == 0);
+  goto done;
+
  done:
+  parsed_consensus_request_clear(&req);
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
   return 0;
 }
 
@@ -3026,12 +4257,14 @@ static int
 handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
   {
     int current;
     ssize_t body_len = 0;
     ssize_t estimated_len = 0;
+    /* This smartlist holds strings that we can compress on the fly. */
     smartlist_t *items = smartlist_new();
+    /* This smartlist holds cached_dir_t objects that have a precompressed
+     * deflated version. */
     smartlist_t *dir_items = smartlist_new();
     int lifetime = 60; /* XXXX?? should actually use vote intervals. */
     url += strlen("/tor/status-vote/");
@@ -3082,12 +4315,33 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       write_http_status_line(conn, 404, "Not found");
       goto vote_done;
     }
+
+    /* We're sending items from at most one kind of source */
+    tor_assert_nonfatal(smartlist_len(items) == 0 ||
+                        smartlist_len(dir_items) == 0);
+
+    int streaming;
+    unsigned mask;
+    if (smartlist_len(items)) {
+      /* We're taking strings and compressing them on the fly. */
+      streaming = 1;
+      mask = ~0u;
+    } else {
+      /* We're taking cached_dir_t objects. We only have them uncompressed
+       * or deflated. */
+      streaming = 0;
+      mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
+    }
+    const compress_method_t compress_method = find_best_compression_method(
+                              args->compression_supported&mask, streaming);
+
     SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-                      body_len += compressed ? d->dir_z_len : d->dir_len);
+                      body_len += compress_method != NO_METHOD ?
+                        d->dir_compressed_len : d->dir_len);
     estimated_len += body_len;
     SMARTLIST_FOREACH(items, const char *, item, {
         size_t ln = strlen(item);
-        if (compressed) {
+        if (compress_method != NO_METHOD) {
           estimated_len += ln/2;
         } else {
           body_len += ln; estimated_len += ln;
@@ -3098,24 +4352,27 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       write_http_status_line(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
-    write_http_response_header(conn, body_len ? body_len : -1, compressed,
+    write_http_response_header(conn, body_len ? body_len : -1,
+                 compress_method,
                  lifetime);
 
     if (smartlist_len(items)) {
-      if (compressed) {
-        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                    choose_compression_level(estimated_len));
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                           choose_compression_level(estimated_len));
         SMARTLIST_FOREACH(items, const char *, c,
-                 connection_write_to_buf_zlib(c, strlen(c), conn, 0));
-        connection_write_to_buf_zlib("", 0, conn, 1);
+                 connection_write_to_buf_compress(c, strlen(c), conn, 0));
+        connection_write_to_buf_compress("", 0, conn, 1);
       } else {
         SMARTLIST_FOREACH(items, const char *, c,
                          connection_write_to_buf(c, strlen(c), TO_CONN(conn)));
       }
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
-          connection_write_to_buf(compressed ? d->dir_z : d->dir,
-                                  compressed ? d->dir_z_len : d->dir_len,
+          connection_write_to_buf(compress_method != NO_METHOD ?
+                                    d->dir_compressed : d->dir,
+                                  compress_method != NO_METHOD ?
+                                    d->dir_compressed_len : d->dir_len,
                                   TO_CONN(conn)));
     }
   vote_done:
@@ -3133,44 +4390,51 @@ static int
 handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+  int clear_spool = 1;
   {
-    smartlist_t *fps = smartlist_new();
+    conn->spool = smartlist_new();
 
-    dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"),
-                                      fps, NULL,
+    dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
+                                      DIR_SPOOL_MICRODESC,
+                                      conn->spool, NULL,
                                       DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
 
-    if (!dirserv_have_any_microdesc(fps)) {
+    size_t size_guess = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, 0,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, NULL);
+    if (smartlist_len(conn->spool) == 0) {
       write_http_status_line(conn, 404, "Not found");
-      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
-      smartlist_free(fps);
       goto done;
     }
-    size_t dlen = dirserv_estimate_microdesc_size(fps, compressed);
-    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
+    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
       write_http_status_line(conn, 503, "Directory busy, try again later");
-      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
-      smartlist_free(fps);
       goto done;
     }
 
-    write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME);
-    conn->dir_spool_src = DIR_SPOOL_MICRODESC;
-    conn->fingerprint_stack = fps;
+    clear_spool = 0;
+    write_http_response_header(conn, -1,
+                               compress_method,
+                               MICRODESC_CACHE_LIFETIME);
 
-    if (compressed)
-      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                      choose_compression_level(dlen));
+    if (compress_method != NO_METHOD)
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                      choose_compression_level(size_guess));
 
-    connection_dirserv_flushed_some(conn);
+    const int initial_flush_result = connection_dirserv_flushed_some(conn);
+    tor_assert_nonfatal(initial_flush_result == 0);
     goto done;
   }
 
  done:
+  if (clear_spool) {
+    dir_conn_clear_spool(conn);
+  }
   return 0;
 }
 
@@ -3180,71 +4444,92 @@ static int
 handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
   const or_options_t *options = get_options();
+  int clear_spool = 1;
   if (!strcmpstart(url,"/tor/server/") ||
       (!options->BridgeAuthoritativeDir &&
        !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
-    size_t dlen;
     int res;
-    const char *msg;
+    const char *msg = NULL;
     int cache_lifetime = 0;
     int is_extra = !strcmpstart(url,"/tor/extra/");
     url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
-    conn->fingerprint_stack = smartlist_new();
-    res = dirserv_get_routerdesc_fingerprints(conn->fingerprint_stack, url,
-                                          &msg,
-                                          !connection_dir_is_encrypted(conn),
-                                          is_extra);
-
-    if (!strcmpstart(url, "fp/")) {
-      if (smartlist_len(conn->fingerprint_stack) == 1)
-        cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "authority")) {
-      cache_lifetime = ROUTERDESC_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "all")) {
-      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
-    } else if (!strcmpstart(url, "d/")) {
-      if (smartlist_len(conn->fingerprint_stack) == 1)
-        cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
-    }
-    if (!strcmpstart(url, "d/"))
-      conn->dir_spool_src =
+    dir_spool_source_t source;
+    time_t publish_cutoff = 0;
+    if (!strcmpstart(url, "d/")) {
+      source =
         is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
-    else
-      conn->dir_spool_src =
+    } else {
+      source =
         is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
+      /* We only want to apply a publish cutoff when we're requesting
+       * resources by fingerprint. */
+      publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    }
 
-    if (!dirserv_have_any_serverdesc(conn->fingerprint_stack,
-                                     conn->dir_spool_src)) {
-      res = -1;
-      msg = "Not found";
+    conn->spool = smartlist_new();
+    res = dirserv_get_routerdesc_spool(conn->spool, url,
+                                       source,
+                                       connection_dir_is_encrypted(conn),
+                                       &msg);
+
+    if (!strcmpstart(url, "all")) {
+      cache_lifetime = FULL_DIR_CACHE_LIFETIME;
+    } else if (smartlist_len(conn->spool) == 1) {
+      cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
     }
 
-    if (res < 0)
+    size_t size_guess = 0;
+    int n_expired = 0;
+    dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
+                                                compress_method != NO_METHOD,
+                                                &size_guess, &n_expired);
+
+    /* If we are the bridge authority and the descriptor is a bridge
+     * descriptor, remember that we served this descriptor for desc stats. */
+    /* XXXX it's a bit of a kludge to have this here. */
+    if (get_options()->BridgeAuthoritativeDir &&
+        source == DIR_SPOOL_SERVER_BY_FP) {
+      SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
+        const routerinfo_t *router =
+          router_get_by_id_digest((const char *)spooled->digest);
+        /* router can be NULL here when the bridge auth is asked for its own
+         * descriptor. */
+        if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
+          rep_hist_note_desc_served(router->cache_info.identity_digest);
+      } SMARTLIST_FOREACH_END(spooled);
+    }
+
+    if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
+      if (msg == NULL)
+        msg = "Not found";
       write_http_status_line(conn, 404, msg);
-    else {
-      dlen = dirserv_estimate_data_size(conn->fingerprint_stack,
-                                        1, compressed);
-      if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
+    } else {
+      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
         write_http_status_line(conn, 503, "Directory busy, try again later");
-        conn->dir_spool_src = DIR_SPOOL_NONE;
+        dir_conn_clear_spool(conn);
         goto done;
       }
-      write_http_response_header(conn, -1, compressed, cache_lifetime);
-      if (compressed)
-        conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                        choose_compression_level(dlen));
+      write_http_response_header(conn, -1, compress_method, cache_lifetime);
+      if (compress_method != NO_METHOD)
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(size_guess));
+      clear_spool = 0;
       /* Prime the connection with some data. */
-      connection_dirserv_flushed_some(conn);
+      int initial_flush_result = connection_dirserv_flushed_some(conn);
+      tor_assert_nonfatal(initial_flush_result == 0);
     }
     goto done;
   }
  done:
- return 0;
+  if (clear_spool)
+    dir_conn_clear_spool(conn);
+  return 0;
 }
 
 /** Helper function for GET /tor/keys/...
@@ -3253,7 +4538,8 @@ static int
 handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
 {
   const char *url = args->url;
-  const int compressed = args->compressed;
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
   const time_t if_modified_since = args->if_modified_since;
   {
     smartlist_t *certs = smartlist_new();
@@ -3316,20 +4602,26 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) {
+    if (global_write_bucket_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len,
+                                2)) {
       write_http_status_line(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
 
-    write_http_response_header(conn, compressed?-1:len, compressed, 60*60);
-    if (compressed) {
-      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD,
-                                      choose_compression_level(len));
+    write_http_response_header(conn,
+                               compress_method != NO_METHOD ? -1 : len,
+                               compress_method,
+                               60*60);
+    if (compress_method != NO_METHOD) {
+      conn->compress_state = tor_compress_new(1, compress_method,
+                                              choose_compression_level(len));
       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body,
-                                         c->cache_info.signed_descriptor_len,
-                                         conn, 0));
-      connection_write_to_buf_zlib("", 0, conn, 1);
+            connection_write_to_buf_compress(
+                c->cache_info.signed_descriptor_body,
+                c->cache_info.signed_descriptor_len,
+                conn, 0));
+      connection_write_to_buf_compress("", 0, conn, 1);
     } else {
       SMARTLIST_FOREACH(certs, authority_cert_t *, c,
             connection_write_to_buf(c->cache_info.signed_descriptor_body,
@@ -3347,7 +4639,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
 /** Helper function for GET /tor/rendezvous2/
  */
 static int
-handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args)
+handle_get_hs_descriptor_v2(dir_connection_t *conn,
+                            const get_handler_args_t *args)
 {
   const char *url = args->url;
   if (connection_dir_is_encrypted(conn)) {
@@ -3359,7 +4652,7 @@ handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args)
                safe_str(escaped(query)));
       switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
         case 1: /* valid */
-          write_http_response_header(conn, strlen(descp), 0, 0);
+          write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
           connection_write_to_buf(descp, strlen(descp), TO_CONN(conn));
           break;
         case 0: /* well-formed but not present */
@@ -3381,6 +4674,43 @@ handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args)
   return 0;
 }
 
+/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+ */
+STATIC int
+handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                            const get_handler_args_t *args)
+{
+  int retval;
+  const char *desc_str = NULL;
+  const char *pubkey_str = NULL;
+  const char *url = args->url;
+
+  /* Reject unencrypted dir connections */
+  if (!connection_dir_is_encrypted(conn)) {
+    write_http_status_line(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* After the path prefix follows the base64 encoded blinded pubkey which we
+   * use to get the descriptor from the cache. Skip the prefix and get the
+   * pubkey. */
+  tor_assert(!strcmpstart(url, "/tor/hs/3/"));
+  pubkey_str = url + strlen("/tor/hs/3/");
+  retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
+                                  pubkey_str, &desc_str);
+  if (retval <= 0 || desc_str == NULL) {
+    write_http_status_line(conn, 404, "Not found");
+    goto done;
+  }
+
+  /* Found requested descriptor! Pass it to this nice client. */
+  write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
+  connection_write_to_buf(desc_str, strlen(desc_str), TO_CONN(conn));
+
+ done:
+  return 0;
+}
+
 /** Helper function for GET /tor/networkstatus-bridges
  */
 static int
@@ -3413,7 +4743,7 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
     /* all happy now. send an answer. */
     status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
     size_t dlen = strlen(status);
-    write_http_response_header(conn, dlen, 0, 0);
+    write_http_response_header(conn, dlen, NO_METHOD, 0);
     connection_write_to_buf(status, dlen, TO_CONN(conn));
     tor_free(status);
     goto done;
@@ -3430,20 +4760,104 @@ handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
   {
     const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
     size_t len = strlen(robots);
-    write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME);
+    write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
     connection_write_to_buf(robots, len, TO_CONN(conn));
   }
   return 0;
 }
 
+/* Given the <b>url</b> from a POST request, try to extract the version number
+ * using the provided <b>prefix</b>. The version should be after the prefix and
+ * ending with the seperator "/". For instance:
+ *      /tor/hs/3/publish
+ *
+ * On success, <b>end_pos</b> points to the position right after the version
+ * was found. On error, it is set to NULL.
+ *
+ * Return version on success else negative value. */
+STATIC int
+parse_hs_version_from_post(const char *url, const char *prefix,
+                           const char **end_pos)
+{
+  int ok;
+  unsigned long version;
+  const char *start;
+  char *end = NULL;
+
+  tor_assert(url);
+  tor_assert(prefix);
+  tor_assert(end_pos);
+
+  /* Check if the prefix does start the url. */
+  if (strcmpstart(url, prefix)) {
+    goto err;
+  }
+  /* Move pointer to the end of the prefix string. */
+  start = url + strlen(prefix);
+  /* Try this to be the HS version and if we are still at the separator, next
+   * will be move to the right value. */
+  version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
+  if (!ok) {
+    goto err;
+  }
+
+  *end_pos = end;
+  return (int) version;
+ err:
+  *end_pos = NULL;
+  return -1;
+}
+
+/* Handle the POST request for a hidden service descripror. The request is in
+ * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
+ * else return 400 indicating a bad request. */
+STATIC int
+handle_post_hs_descriptor(const char *url, const char *body)
+{
+  int version;
+  const char *end_pos;
+
+  tor_assert(url);
+  tor_assert(body);
+
+  version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
+  if (version < 0) {
+    goto err;
+  }
+
+  /* We have a valid version number, now make sure it's a publish request. Use
+   * the end position just after the version and check for the command. */
+  if (strcmpstart(end_pos, "/publish")) {
+    goto err;
+  }
+
+  switch (version) {
+  case HS_VERSION_THREE:
+    if (hs_cache_store_as_dir(body) < 0) {
+      goto err;
+    }
+    log_info(LD_REND, "Publish request for HS descriptor handled "
+                      "successfully.");
+    break;
+  default:
+    /* Unsupported version, return a bad request. */
+    goto err;
+  }
+
+  return 200;
+ err:
+  /* Bad request. */
+  return 400;
+}
+
 /** Helper function: called when a dirserver gets a complete HTTP POST
  * request.  Look for an uploaded server descriptor or rendezvous
  * service descriptor.  On finding one, process it and write a
  * response into conn-\>outbuf.  If the request is unrecognized, send a
  * 400.  Always return 0. */
-static int
-directory_handle_command_post(dir_connection_t *conn, const char *headers,
-                              const char *body, size_t body_len)
+MOCK_IMPL(STATIC int,
+directory_handle_command_post,(dir_connection_t *conn, const char *headers,
+                               const char *body, size_t body_len))
 {
   char *url = NULL;
   const or_options_t *options = get_options();
@@ -3469,7 +4883,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
   if (connection_dir_is_encrypted(conn) &&
       !strcmpstart(url,"/tor/rendezvous2/publish")) {
     if (rend_cache_store_v2_desc_as_dir(body) < 0) {
-      log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.",
+      log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
                (int)body_len, conn->base_.address);
       write_http_status_line(conn, 400,
                              "Invalid v2 service descriptor rejected");
@@ -3480,6 +4894,21 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
     goto done;
   }
 
+  /* Handle HS descriptor publish request. */
+  /* XXX: This should be disabled with a consensus param until we want to
+   * the prop224 be deployed and thus use. */
+  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+    const char *msg = "HS descriptor stored successfully.";
+
+    /* We most probably have a publish request for an HS descriptor. */
+    int code = handle_post_hs_descriptor(url, body);
+    if (code != 200) {
+      msg = "Invalid HS descriptor. Rejected.";
+    }
+    write_http_status_line(conn, code, msg);
+    goto done;
+  }
+
   if (!authdir_mode(options)) {
     /* we just provide cached directories; we don't want to
      * receive anything. */
@@ -3497,14 +4926,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
                                              conn->base_.address, &msg);
     tor_assert(msg);
 
-    if (r == ROUTER_ADDED_NOTIFY_GENERATOR) {
-      /* Accepted with a message. */
-      log_info(LD_DIRSERV,
-               "Problematic router descriptor or extra-info from %s "
-               "(\"%s\").",
-               conn->base_.address, msg);
-      write_http_status_line(conn, 400, msg);
-    } else if (r == ROUTER_ADDED_SUCCESSFULLY) {
+    if (r == ROUTER_ADDED_SUCCESSFULLY) {
       write_http_status_line(conn, 200, msg);
     } else if (WRA_WAS_OUTDATED(r)) {
       write_http_response_header_impl(conn, -1, NULL, NULL,
@@ -3560,7 +4982,7 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers,
  * from the inbuf, try to process it; otherwise, leave it on the
  * buffer.  Return a 0 on success, or -1 on error.
  */
-static int
+STATIC int
 directory_handle_command(dir_connection_t *conn)
 {
   char *headers=NULL, *body=NULL;
@@ -3631,7 +5053,7 @@ connection_dir_finished_flushing(dir_connection_t *conn)
       conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
       return 0;
     case DIR_CONN_STATE_SERVER_WRITING:
-      if (conn->dir_spool_src != DIR_SPOOL_NONE) {
+      if (conn->spool) {
         log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
         connection_mark_for_close(TO_CONN(conn));
       } else {
@@ -3861,7 +5283,7 @@ download_status_schedule_get_delay(download_status_t *dls,
       delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1);
   } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) {
     /* Check if we missed a reset somehow */
-    if (dls->last_backoff_position > dls_schedule_position) {
+    IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
       dls->last_backoff_position = 0;
       dls->last_delay_used = 0;
     }
@@ -4125,13 +5547,14 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
    * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
 }
 
-/** Called when a connection to download microdescriptors has failed in whole
- * or in part. <b>failed</b> is a list of every microdesc digest we didn't
- * get. <b>status_code</b> is the http status code we received. Reschedule the
- * microdesc downloads as appropriate. */
+/** Called when a connection to download microdescriptors from relay with
+ * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
+ * of every microdesc digest we didn't get. <b>status_code</b> is the http
+ * status code we received. Reschedule the microdesc downloads as
+ * appropriate. */
 static void
 dir_microdesc_download_failed(smartlist_t *failed,
-                              int status_code)
+                              int status_code, const char *dir_id)
 {
   networkstatus_t *consensus
     = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
@@ -4142,17 +5565,26 @@ dir_microdesc_download_failed(smartlist_t *failed,
 
   if (! consensus)
     return;
+
+  /* We failed to fetch a microdescriptor from 'dir_id', note it down
+   * so that we don't try the same relay next time... */
+  microdesc_note_outdated_dirserver(dir_id);
+
   SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
     rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
     if (!rs)
       continue;
     dls = &rs->dl_status;
     if (dls->n_download_failures >=
-        get_options()->TestingMicrodescMaxDownloadTries)
+        get_options()->TestingMicrodescMaxDownloadTries) {
       continue;
-    {
+    }
+
+    { /* Increment the failure count for this md fetch */
       char buf[BASE64_DIGEST256_LEN+1];
       digest256_to_base64(buf, d);
+      log_info(LD_DIR, "Failed to download md %s from %s",
+               buf, hex_str(dir_id, DIGEST_LEN));
       download_status_increment_failure(dls, status_code, buf,
                                         server, now);
     }
@@ -4314,3 +5746,34 @@ dir_split_resource_into_fingerprints(const char *resource,
   return 0;
 }
 
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}
+
diff --git a/src/or/directory.h b/src/or/directory.h
index 629b3ead90..571c30a0fc 100644
--- a/src/or/directory.h
+++ b/src/or/directory.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -41,43 +41,53 @@ typedef enum {
 
 int directory_must_use_begindir(const or_options_t *options);
 
-MOCK_DECL(void, directory_initiate_command_routerstatus,
-                (const routerstatus_t *status,
-                 uint8_t dir_purpose,
-                 uint8_t router_purpose,
-                 dir_indirection_t indirection,
-                 const char *resource,
-                 const char *payload,
-                 size_t payload_len,
-                 time_t if_modified_since));
-
-void directory_initiate_command_routerstatus_rend(const routerstatus_t *status,
-                                                  uint8_t dir_purpose,
-                                                  uint8_t router_purpose,
-                                                 dir_indirection_t indirection,
-                                                  const char *resource,
-                                                  const char *payload,
-                                                  size_t payload_len,
-                                                  time_t if_modified_since,
-                                                const rend_data_t *rend_query);
+/**
+ * A directory_request_t describes the information about a directory request
+ * at the client side.  It describes what we're going to ask for, which
+ * directory we're going to ask for it, how we're going to contact that
+ * directory, and (in some cases) what to do with it when we're done.
+ */
+typedef struct directory_request_t directory_request_t;
+directory_request_t *directory_request_new(uint8_t dir_purpose);
+void directory_request_free(directory_request_t *req);
+void directory_request_set_or_addr_port(directory_request_t *req,
+                                        const tor_addr_port_t *p);
+void directory_request_set_dir_addr_port(directory_request_t *req,
+                                         const tor_addr_port_t *p);
+void directory_request_set_directory_id_digest(directory_request_t *req,
+                                               const char *digest);
+void directory_request_set_guard_state(directory_request_t *req,
+                                       struct circuit_guard_state_t *state);
+void directory_request_set_router_purpose(directory_request_t *req,
+                                          uint8_t router_purpose);
+void directory_request_set_indirection(directory_request_t *req,
+                                       dir_indirection_t indirection);
+void directory_request_set_resource(directory_request_t *req,
+                                    const char *resource);
+void directory_request_set_payload(directory_request_t *req,
+                                   const char *payload,
+                                   size_t payload_len);
+void directory_request_set_if_modified_since(directory_request_t *req,
+                                             time_t if_modified_since);
+void directory_request_set_rend_query(directory_request_t *req,
+                                      const rend_data_t *query);
+
+void directory_request_set_routerstatus(directory_request_t *req,
+                                        const routerstatus_t *rs);
+void directory_request_add_header(directory_request_t *req,
+                                  const char *key,
+                                  const char *val);
+MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
 
 int parse_http_response(const char *headers, int *code, time_t *date,
                         compress_method_t *compression, char **response);
 
-int connection_dir_is_encrypted(dir_connection_t *conn);
+int connection_dir_is_encrypted(const dir_connection_t *conn);
 int connection_dir_reached_eof(dir_connection_t *conn);
 int connection_dir_process_inbuf(dir_connection_t *conn);
 int connection_dir_finished_flushing(dir_connection_t *conn);
 int connection_dir_finished_connecting(dir_connection_t *conn);
 void connection_dir_about_to_close(dir_connection_t *dir_conn);
-void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
-                                const tor_addr_t *dir_addr, uint16_t dir_port,
-                                const char *digest,
-                                uint8_t dir_purpose, uint8_t router_purpose,
-                                dir_indirection_t indirection,
-                                const char *resource,
-                                const char *payload, size_t payload_len,
-                                time_t if_modified_since);
 
 #define DSR_HEX       (1<<0)
 #define DSR_BASE64    (1<<1)
@@ -86,7 +96,12 @@ void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port,
 int dir_split_resource_into_fingerprints(const char *resource,
                                      smartlist_t *fp_out, int *compressed_out,
                                      int flags);
-
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
 int dir_split_resource_into_fingerprint_pairs(const char *res,
                                               smartlist_t *pairs_out);
 char *directory_dump_request_log(void);
@@ -138,30 +153,51 @@ int download_status_get_n_failures(const download_status_t *dls);
 int download_status_get_n_attempts(const download_status_t *dls);
 time_t download_status_get_next_attempt_at(const download_status_t *dls);
 
-/* Yes, these two functions are confusingly similar.
- * Let's sort that out in #20077. */
-int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose);
-int is_sensitive_dir_purpose(uint8_t dir_purpose);
+int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+                            const char *resource);
+
+#ifdef DIRECTORY_PRIVATE
+
+struct get_handler_args_t;
+STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
+                                       const struct get_handler_args_t *args);
+STATIC int directory_handle_command(dir_connection_t *conn);
+STATIC char *accept_encoding_header(void);
+STATIC int allowed_anonymous_connection_compression_method(compress_method_t);
+STATIC void warn_disallowed_anonymous_compression_method(compress_method_t);
+
+struct response_handler_args_t;
+
+STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
+                                 const struct response_handler_args_t *args);
+
+#endif /* defined(DIRECTORY_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
-/* Used only by directory.c and test_dir.c */
+/* Used only by test_dir.c */
 
 STATIC int parse_http_url(const char *headers, char **url);
 STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
                                      const char *resource);
-STATIC int directory_handle_command_get(dir_connection_t *conn,
-                                        const char *headers,
-                                        const char *req_body,
-                                        size_t req_body_len);
+MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
+                                                    const char *headers,
+                                                    const char *req_body,
+                                                    size_t req_body_len));
+MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
+                                                     const char *headers,
+                                                     const char *body,
+                                                     size_t body_len));
 STATIC int download_status_schedule_get_delay(download_status_t *dls,
                                               const smartlist_t *schedule,
                                               int min_delay, int max_delay,
                                               time_t now);
 
+STATIC int handle_post_hs_descriptor(const char *url, const char *body);
+
 STATIC char* authdir_type_to_string(dirinfo_type_t auth);
 STATIC const char * dir_conn_purpose_to_string(int purpose);
 STATIC int should_use_directory_guards(const or_options_t *options);
-STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes);
+STATIC compression_level_t choose_compression_level(ssize_t n_bytes);
 STATIC const smartlist_t *find_dl_schedule(download_status_t *dls,
                                            const or_options_t *options);
 STATIC void find_dl_min_and_max_delay(download_status_t *dls,
@@ -169,6 +205,10 @@ STATIC void find_dl_min_and_max_delay(download_status_t *dls,
                                       int *min, int *max);
 STATIC int next_random_exponential_delay(int delay, int max_delay);
 
+STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
+                                      const char **end_pos);
+
+STATIC unsigned parse_accept_encoding_header(const char *h);
 #endif
 
 #endif
diff --git a/src/or/dirserv.c b/src/or/dirserv.c
index 94290d5dd8..8cbf7d7bc8 100644
--- a/src/or/dirserv.c
+++ b/src/or/dirserv.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRSERV_PRIVATE
@@ -13,6 +13,8 @@
 #include "command.h"
 #include "connection.h"
 #include "connection_or.h"
+#include "conscache.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "directory.h"
 #include "dirserv.h"
@@ -81,14 +83,23 @@ dirserv_get_status_impl(const char *fp, const char *nickname,
                         int severity);
 static void clear_cached_dir(cached_dir_t *d);
 static const signed_descriptor_t *get_signed_descriptor_by_fp(
-                                                        const char *fp,
-                                                        int extrainfo,
-                                                        time_t publish_cutoff);
+                                                        const uint8_t *fp,
+                                                        int extrainfo);
 static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
                                                 const char **msg);
 static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri);
 static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri);
 
+static int spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                                        int conn_is_encrypted,
+                                        const uint8_t **body_out,
+                                        size_t *size_out,
+                                        time_t *published_out);
+static cached_dir_t *spooled_resource_lookup_cached_dir(
+                                   const spooled_resource_t *spooled,
+                                   time_t *published_out);
+static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
+
 /************** Fingerprint handling code ************/
 
 /*                 1  Historically used to indicate Named */
@@ -274,6 +285,13 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
     return FP_REJECT;
   }
 
+  /* Check for the more usual versions to reject a router first. */
+  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
+                                             router->addr, router->or_port,
+                                             router->platform, msg, severity);
+  if (r)
+    return r;
+
   /* dirserv_get_status_impl already rejects versions older than 0.2.4.18-rc,
    * and onion_curve25519_pkey was introduced in 0.2.4.8-alpha.
    * But just in case a relay doesn't provide or lies about its version, or
@@ -324,9 +342,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
     }
   }
 
-  return dirserv_get_status_impl(d, router->nickname,
-                                 router->addr, router->or_port,
-                                 router->platform, msg, severity);
+  return 0;
 }
 
 /** Return true if there is no point in downloading the router described by
@@ -578,6 +594,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
                    !general ? router_purpose_to_string(purpose) : "",
                    !general ? "\n" : "")<0) {
     *msg = "Couldn't format annotations";
+    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
+     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
     return -1;
   }
 
@@ -708,7 +726,12 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
     log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because "
              "its key did not match an older RSA/Ed25519 keypair",
              router_describe(ri), source);
-    *msg = "Looks like your keypair does not match its older value.";
+    *msg = "Looks like your keypair has changed? This authority previously "
+      "recorded a different RSA identity for this Ed25519 identity (or vice "
+      "versa.) Did you replace or copy some of your key files, but not "
+      "the others? You should either restore the expected keypair, or "
+      "delete your keys and restart Tor to start your relay with a new "
+      "identity.";
     r = ROUTER_AUTHDIR_REJECTS;
     goto fail;
   }
@@ -868,6 +891,9 @@ directory_remove_invalid(void)
  * Allocate and return a description of the status of the server <b>desc</b>,
  * for use in a v1-style router-status line.  The server is listed
  * as running iff <b>is_live</b> is true.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
  */
 static char *
 list_single_server_status(const routerinfo_t *desc, int is_live)
@@ -980,6 +1006,9 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
  * *<b>router_status_out</b>.  Return 0 on success, -1 on failure.
  *
  * If for_controller is true, include the routers with very old descriptors.
+ *
+ * This is deprecated: it's only used for controllers that want outputs in
+ * the old format.
  */
 int
 list_server_status_v1(smartlist_t *routers, char **router_status_out,
@@ -1012,7 +1041,7 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
       if (!node->is_running)
         *cp++ = '!';
       router_get_verbose_nickname(cp, ri);
-      smartlist_add(rs_entries, tor_strdup(name_buf));
+      smartlist_add_strdup(rs_entries, name_buf);
     } else if (ri->cache_info.published_on >= cutoff) {
       smartlist_add(rs_entries, list_single_server_status(ri,
                                                           node->is_running));
@@ -1133,8 +1162,10 @@ directory_fetches_dir_info_later(const or_options_t *options)
   return options->UseBridges != 0;
 }
 
-/** Return true iff we want to fetch and keep certificates for authorities
+/** Return true iff we want to serve certificates for authorities
  * that we don't acknowledge as authorities ourself.
+ * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
+ * and keep these certificates.
  */
 int
 directory_caches_unknown_auth_certs(const or_options_t *options)
@@ -1142,11 +1173,14 @@ directory_caches_unknown_auth_certs(const or_options_t *options)
   return dir_server_mode(options) || options->BridgeRelay;
 }
 
-/** Return 1 if we want to keep descriptors, networkstatuses, etc around.
+/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
  * Else return 0.
  * Check options->DirPort_set and directory_permits_begindir_requests()
  * to see if we are willing to serve these directory documents to others via
  * the DirPort and begindir-over-ORPort, respectively.
+ *
+ * To check if we should fetch documents, use we_want_to_fetch_flavor and
+ * we_want_to_fetch_unknown_auth_certs instead of this function.
  */
 int
 directory_caches_dir_info(const or_options_t *options)
@@ -1161,7 +1195,7 @@ directory_caches_dir_info(const or_options_t *options)
     should_refuse_unknown_exits(options);
 }
 
-/** Return 1 if we want to allow remote people to ask us directory
+/** Return 1 if we want to allow remote clients to ask us directory
  * requests via the "begin_dir" interface, which doesn't require
  * having any separate port open. */
 int
@@ -1210,8 +1244,8 @@ new_cached_dir(char *s, time_t published)
   d->dir = s;
   d->dir_len = strlen(s);
   d->published = published;
-  if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len,
-                        ZLIB_METHOD)) {
+  if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len),
+                   d->dir, d->dir_len, ZLIB_METHOD)) {
     log_warn(LD_BUG, "Error compressing directory");
   }
   return d;
@@ -1222,7 +1256,7 @@ static void
 clear_cached_dir(cached_dir_t *d)
 {
   tor_free(d->dir);
-  tor_free(d->dir_z);
+  tor_free(d->dir_compressed);
   memset(d, 0, sizeof(cached_dir_t));
 }
 
@@ -1245,6 +1279,7 @@ void
 dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
                                            const char *flavor_name,
                                            const common_digests_t *digests,
+                                           const uint8_t *sha3_as_signed,
                                            time_t published)
 {
   cached_dir_t *new_networkstatus;
@@ -1254,6 +1289,8 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
 
   new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
   memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
+  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
+         DIGEST256_LEN);
   old_networkstatus = strmap_set(cached_consensuses, flavor_name,
                                  new_networkstatus);
   if (old_networkstatus)
@@ -2013,7 +2050,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                              vrs->status.guardfraction_percentage);
     }
 
-    smartlist_add(chunks, tor_strdup("\n"));
+    smartlist_add_strdup(chunks, "\n");
 
     if (desc) {
       summary = policy_summarize(desc->exit_policy, AF_INET);
@@ -2023,7 +2060,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
 
     if (format == NS_V3_VOTE && vrs) {
       if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
-        smartlist_add(chunks, tor_strdup("id ed25519 none\n"));
+        smartlist_add_strdup(chunks, "id ed25519 none\n");
       } else {
         char ed_b64[BASE64_DIGEST256_LEN+1];
         digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
@@ -2115,12 +2152,8 @@ get_possible_sybil_list(const smartlist_t *routers)
   int addr_count;
   /* Allow at most this number of Tor servers on a single IP address, ... */
   int max_with_same_addr = options->AuthDirMaxServersPerAddr;
-  /* ... unless it's a directory authority, in which case allow more. */
-  int max_with_same_addr_on_authority = options->AuthDirMaxServersPerAuthAddr;
   if (max_with_same_addr <= 0)
     max_with_same_addr = INT_MAX;
-  if (max_with_same_addr_on_authority <= 0)
-    max_with_same_addr_on_authority = INT_MAX;
 
   smartlist_add_all(routers_by_ip, routers);
   smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_);
@@ -2133,9 +2166,7 @@ get_possible_sybil_list(const smartlist_t *routers)
         last_addr = ri->addr;
         addr_count = 1;
       } else if (++addr_count > max_with_same_addr) {
-        if (!router_addr_is_trusted_dir(ri->addr) ||
-            addr_count > max_with_same_addr_on_authority)
-          digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
+        digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri);
       }
   } SMARTLIST_FOREACH_END(ri);
 
@@ -2295,8 +2326,8 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs)
 }
 
 /** Routerstatus <b>rs</b> is part of a group of routers that are on
- * too narrow an IP-space. Clear out its flags: we don't want people
- * using it.
+ * too narrow an IP-space. Clear out its flags since we don't want it be used
+ * because of its Sybil-like appearance.
  *
  * Leave its BadExit flag alone though, since if we think it's a bad exit,
  * we want to vote that way in case all the other authorities are voting
@@ -3045,7 +3076,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     config_line_t *cl;
     for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
       if (validate_recommended_package_line(cl->value))
-        smartlist_add(v3_out->package_lines, tor_strdup(cl->value));
+        smartlist_add_strdup(v3_out->package_lines, cl->value);
     }
   }
 
@@ -3054,9 +3085,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                 "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
                 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   if (vote_on_reachability)
-    smartlist_add(v3_out->known_flags, tor_strdup("Running"));
+    smartlist_add_strdup(v3_out->known_flags, "Running");
   if (listbadexits)
-    smartlist_add(v3_out->known_flags, tor_strdup("BadExit"));
+    smartlist_add_strdup(v3_out->known_flags, "BadExit");
   smartlist_sort_strings(v3_out->known_flags);
 
   if (options->ConsensusParams) {
@@ -3101,58 +3132,61 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
  * requests, adds identity digests.
  */
 int
-dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
-                                    const char **msg, int for_unencrypted_conn,
-                                    int is_extrainfo)
+dirserv_get_routerdesc_spool(smartlist_t *spool_out,
+                             const char *key,
+                             dir_spool_source_t source,
+                             int conn_is_encrypted,
+                             const char **msg_out)
 {
-  int by_id = 1;
-  *msg = NULL;
+  *msg_out = NULL;
 
   if (!strcmp(key, "all")) {
-    routerlist_t *rl = router_get_routerlist();
-    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
-                      smartlist_add(fps_out,
-                      tor_memdup(r->cache_info.identity_digest, DIGEST_LEN)));
-    /* Treat "all" requests as if they were unencrypted */
-    for_unencrypted_conn = 1;
+    const routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH_BEGIN(rl->routers, const routerinfo_t *, r) {
+      spooled_resource_t *spooled;
+      spooled = spooled_resource_new(source,
+                              (const uint8_t *)r->cache_info.identity_digest,
+                              DIGEST_LEN);
+      /* Treat "all" requests as if they were unencrypted */
+      conn_is_encrypted = 0;
+      smartlist_add(spool_out, spooled);
+    } SMARTLIST_FOREACH_END(r);
   } else if (!strcmp(key, "authority")) {
     const routerinfo_t *ri = router_get_my_routerinfo();
     if (ri)
-      smartlist_add(fps_out,
-                    tor_memdup(ri->cache_info.identity_digest, DIGEST_LEN));
+      smartlist_add(spool_out,
+                    spooled_resource_new(source,
+                             (const uint8_t *)ri->cache_info.identity_digest,
+                             DIGEST_LEN));
   } else if (!strcmpstart(key, "d/")) {
-    by_id = 0;
     key += strlen("d/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
   } else if (!strcmpstart(key, "fp/")) {
     key += strlen("fp/");
-    dir_split_resource_into_fingerprints(key, fps_out, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
+    dir_split_resource_into_spoolable(key, source, spool_out, NULL,
+                                  DSR_HEX|DSR_SORT_UNIQ);
   } else {
-    *msg = "Key not recognized";
+    *msg_out = "Not found";
     return -1;
   }
 
-  if (for_unencrypted_conn) {
+  if (! conn_is_encrypted) {
     /* Remove anything that insists it not be sent unencrypted. */
-    SMARTLIST_FOREACH_BEGIN(fps_out, char *, cp) {
-        const signed_descriptor_t *sd;
-        if (by_id)
-          sd = get_signed_descriptor_by_fp(cp,is_extrainfo,0);
-        else if (is_extrainfo)
-          sd = extrainfo_get_by_descriptor_digest(cp);
-        else
-          sd = router_get_by_descriptor_digest(cp);
-        if (sd && !sd->send_unencrypted) {
-          tor_free(cp);
-          SMARTLIST_DEL_CURRENT(fps_out, cp);
-        }
-    } SMARTLIST_FOREACH_END(cp);
+    SMARTLIST_FOREACH_BEGIN(spool_out, spooled_resource_t *, spooled) {
+      const uint8_t *body = NULL;
+      size_t bodylen = 0;
+      int r = spooled_resource_lookup_body(spooled, conn_is_encrypted,
+                                           &body, &bodylen, NULL);
+      if (r < 0 || body == NULL || bodylen == 0) {
+        SMARTLIST_DEL_CURRENT(spool_out, spooled);
+        spooled_resource_free(spooled);
+      }
+    } SMARTLIST_FOREACH_END(spooled);
   }
 
-  if (!smartlist_len(fps_out)) {
-    *msg = "Servers unavailable";
+  if (!smartlist_len(spool_out)) {
+    *msg_out = "Servers unavailable";
     return -1;
   }
   return 0;
@@ -3248,7 +3282,8 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
 void
 dirserv_orconn_tls_done(const tor_addr_t *addr,
                         uint16_t or_port,
-                        const char *digest_rcvd)
+                        const char *digest_rcvd,
+                        const ed25519_public_key_t *ed_id_rcvd)
 {
   node_t *node = NULL;
   tor_addr_port_t orport;
@@ -3260,8 +3295,26 @@ dirserv_orconn_tls_done(const tor_addr_t *addr,
   node = node_get_mutable_by_id(digest_rcvd);
   if (node == NULL || node->ri == NULL)
     return;
+
   ri = node->ri;
 
+  if (get_options()->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node) &&
+      ri->cache_info.signing_key_cert) {
+    /* We allow the node to have an ed25519 key if we haven't been told one in
+     * the routerinfo, but if we *HAVE* been told one in the routerinfo, it
+     * needs to match. */
+    const ed25519_public_key_t *expected_id =
+      &ri->cache_info.signing_key_cert->signing_key;
+    tor_assert(!ed25519_public_key_is_zero(expected_id));
+    if (! ed_id_rcvd || ! ed25519_pubkey_eq(ed_id_rcvd, expected_id)) {
+      log_info(LD_DIRSERV, "Router at %s:%d with RSA ID %s "
+               "did not present expected Ed25519 ID.",
+               fmt_addr(addr), or_port, hex_str(digest_rcvd, DIGEST_LEN));
+      return; /* Don't mark it as reachable. */
+    }
+  }
+
   tor_addr_copy(&orport.addr, addr);
   orport.port = or_port;
   if (router_has_orport(ri, &orport)) {
@@ -3269,7 +3322,7 @@ dirserv_orconn_tls_done(const tor_addr_t *addr,
     if (!authdir_mode_bridge(get_options()) ||
         ri->purpose == ROUTER_PURPOSE_BRIDGE) {
       char addrstr[TOR_ADDR_BUF_LEN];
-      /* This is a bridge or we're not a bridge authorititative --
+      /* This is a bridge or we're not a bridge authority --
          mark it as reachable.  */
       log_info(LD_DIRSERV, "Found router %s to be reachable at %s:%d. Yay.",
                router_describe(ri),
@@ -3317,21 +3370,31 @@ dirserv_should_launch_reachability_test(const routerinfo_t *ri,
 void
 dirserv_single_reachability_test(time_t now, routerinfo_t *router)
 {
+  const or_options_t *options = get_options();
   channel_t *chan = NULL;
-  node_t *node = NULL;
+  const node_t *node = NULL;
   tor_addr_t router_addr;
+  const ed25519_public_key_t *ed_id_key;
   (void) now;
 
   tor_assert(router);
-  node = node_get_mutable_by_id(router->cache_info.identity_digest);
+  node = node_get_by_id(router->cache_info.identity_digest);
   tor_assert(node);
 
+  if (options->AuthDirTestEd25519LinkKeys &&
+      node_supports_ed25519_link_authentication(node)) {
+    ed_id_key = &router->cache_info.signing_key_cert->signing_key;
+  } else {
+    ed_id_key = NULL;
+  }
+
   /* IPv4. */
   log_debug(LD_OR,"Testing reachability of %s at %s:%u.",
             router->nickname, fmt_addr32(router->addr), router->or_port);
   tor_addr_from_ipv4h(&router_addr, router->addr);
   chan = channel_tls_connect(&router_addr, router->or_port,
-                             router->cache_info.identity_digest);
+                             router->cache_info.identity_digest,
+                             ed_id_key);
   if (chan) command_setup_channel(chan);
 
   /* Possible IPv6. */
@@ -3343,7 +3406,8 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
               tor_addr_to_str(addrstr, &router->ipv6_addr, sizeof(addrstr), 1),
               router->ipv6_orport);
     chan = channel_tls_connect(&router->ipv6_addr, router->ipv6_orport,
-                               router->cache_info.identity_digest);
+                               router->cache_info.identity_digest,
+                               ed_id_key);
     if (chan) command_setup_channel(chan);
   }
 }
@@ -3386,410 +3450,502 @@ dirserv_test_reachability(time_t now)
   ctr = (ctr + 1) % REACHABILITY_MODULO_PER_TEST; /* increment ctr */
 }
 
-/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
- * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
- * flavor name if we want a flavored v3 status, return a pointer to the
- * appropriate cached dir object, or NULL if there isn't one available. */
-static cached_dir_t *
-lookup_cached_dir_by_fp(const char *fp)
+/* ==========
+ * Spooling code.
+ * ========== */
+
+spooled_resource_t *
+spooled_resource_new(dir_spool_source_t source,
+                     const uint8_t *digest, size_t digestlen)
 {
-  cached_dir_t *d = NULL;
-  if (tor_digest_is_zero(fp) && cached_consensuses) {
-    d = strmap_get(cached_consensuses, "ns");
-  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses &&
-           (d = strmap_get(cached_consensuses, fp))) {
-    /* this here interface is a nasty hack XXXX */;
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = source;
+  switch (source) {
+    case DIR_SPOOL_NETWORKSTATUS:
+      spooled->spool_eagerly = 0;
+      break;
+    case DIR_SPOOL_SERVER_BY_DIGEST:
+    case DIR_SPOOL_SERVER_BY_FP:
+    case DIR_SPOOL_EXTRA_BY_DIGEST:
+    case DIR_SPOOL_EXTRA_BY_FP:
+    case DIR_SPOOL_MICRODESC:
+    default:
+      spooled->spool_eagerly = 1;
+      break;
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+      tor_assert_unreached();
+      break;
   }
-  return d;
+  tor_assert(digestlen <= sizeof(spooled->digest));
+  if (digest)
+    memcpy(spooled->digest, digest, digestlen);
+  return spooled;
 }
 
-/** Remove from <b>fps</b> every networkstatus key where both
- * a) we have a networkstatus document and
- * b) it is not newer than <b>cutoff</b>.
+/**
+ * Create a new spooled_resource_t to spool the contents of <b>entry</b> to
+ * the user.  Return the spooled object on success, or NULL on failure (which
+ * is probably caused by a failure to map the body of the item from disk).
  *
- * Return 1 if any items were present at all; else return 0.
+ * Adds a reference to entry's reference counter.
  */
-int
-dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff)
-{
-  int found_any = 0;
-  SMARTLIST_FOREACH_BEGIN(fps, char *, digest) {
-    cached_dir_t *d = lookup_cached_dir_by_fp(digest);
-    if (!d)
-      continue;
-    found_any = 1;
-    if (d->published <= cutoff) {
-      tor_free(digest);
-      SMARTLIST_DEL_CURRENT(fps, digest);
-    }
-  } SMARTLIST_FOREACH_END(digest);
-
-  return found_any;
+spooled_resource_t *
+spooled_resource_new_from_cache_entry(consensus_cache_entry_t *entry)
+{
+  spooled_resource_t *spooled = tor_malloc_zero(sizeof(spooled_resource_t));
+  spooled->spool_source = DIR_SPOOL_CONSENSUS_CACHE_ENTRY;
+  spooled->spool_eagerly = 0;
+  consensus_cache_entry_incref(entry);
+  spooled->consensus_cache_entry = entry;
+
+  int r = consensus_cache_entry_get_body(entry,
+                                         &spooled->cce_body,
+                                         &spooled->cce_len);
+  if (r == 0) {
+    return spooled;
+  } else {
+    spooled_resource_free(spooled);
+    return NULL;
+  }
 }
 
-/** Return the cache-info for identity fingerprint <b>fp</b>, or
- * its extra-info document if <b>extrainfo</b> is true. Return
- * NULL if not found or if the descriptor is older than
- * <b>publish_cutoff</b>. */
-static const signed_descriptor_t *
-get_signed_descriptor_by_fp(const char *fp, int extrainfo,
-                            time_t publish_cutoff)
+/** Release all storage held by <b>spooled</b>. */
+void
+spooled_resource_free(spooled_resource_t *spooled)
 {
-  if (router_digest_is_me(fp)) {
-    if (extrainfo)
-      return &(router_get_my_extrainfo()->cache_info);
-    else
-      return &(router_get_my_routerinfo()->cache_info);
-  } else {
-    const routerinfo_t *ri = router_get_by_id_digest(fp);
-    if (ri &&
-        ri->cache_info.published_on > publish_cutoff) {
-      if (extrainfo)
-        return extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-      else
-        return &ri->cache_info;
-    }
+  if (spooled == NULL)
+    return;
+
+  if (spooled->cached_dir_ref) {
+    cached_dir_decref(spooled->cached_dir_ref);
   }
-  return NULL;
-}
 
-/** Return true iff we have any of the documents (extrainfo or routerdesc)
- * specified by the fingerprints in <b>fps</b> and <b>spool_src</b>.  Used to
- * decide whether to send a 404.  */
-int
-dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src)
-{
-  time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH;
-  SMARTLIST_FOREACH_BEGIN(fps, const char *, fp) {
-      switch (spool_src)
-      {
-        case DIR_SPOOL_EXTRA_BY_DIGEST:
-          if (extrainfo_get_by_descriptor_digest(fp)) return 1;
-          break;
-        case DIR_SPOOL_SERVER_BY_DIGEST:
-          if (router_get_by_descriptor_digest(fp)) return 1;
-          break;
-        case DIR_SPOOL_EXTRA_BY_FP:
-        case DIR_SPOOL_SERVER_BY_FP:
-          if (get_signed_descriptor_by_fp(fp,
-                spool_src == DIR_SPOOL_EXTRA_BY_FP, publish_cutoff))
-            return 1;
-          break;
-      }
-  } SMARTLIST_FOREACH_END(fp);
-  return 0;
+  if (spooled->consensus_cache_entry) {
+    consensus_cache_entry_decref(spooled->consensus_cache_entry);
+  }
+
+  tor_free(spooled);
 }
 
-/** Return true iff any of the 256-bit elements in <b>fps</b> is the digest of
- * a microdescriptor we have. */
-int
-dirserv_have_any_microdesc(const smartlist_t *fps)
+/** When spooling data from a cached_dir_t object, we always add
+ * at least this much. */
+#define DIRSERV_CACHED_DIR_CHUNK_SIZE 8192
+
+/** Return an compression ratio for compressing objects from <b>source</b>.
+ */
+static double
+estimate_compression_ratio(dir_spool_source_t source)
 {
-  microdesc_cache_t *cache = get_microdesc_cache();
-  SMARTLIST_FOREACH(fps, const char *, fp,
-                    if (microdesc_cache_lookup_by_digest256(cache, fp))
-                      return 1);
-  return 0;
+  /* We should put in better estimates here, depending on the number of
+     objects and their type */
+  (void) source;
+  return 0.5;
 }
 
-/** Return an approximate estimate of the number of bytes that will
- * be needed to transmit the server descriptors (if is_serverdescs --
- * they can be either d/ or fp/ queries) or networkstatus objects (if
- * !is_serverdescs) listed in <b>fps</b>.  If <b>compressed</b> is set,
- * we guess how large the data will be after compression.
+/** Return an estimated number of bytes needed for transmitting the
+ * resource in <b>spooled</b> on <b>conn</b>
  *
- * The return value is an estimate; it might be larger or smaller.
- **/
-size_t
-dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
-                           int compressed)
-{
-  size_t result;
-  tor_assert(fps);
-  if (is_serverdescs) {
-    int n = smartlist_len(fps);
-    const routerinfo_t *me = router_get_my_routerinfo();
-    result = (me?me->cache_info.signed_descriptor_len:2048) * n;
-    if (compressed)
-      result /= 2; /* observed compressibility is between 35 and 55%. */
+ * As a convenient side-effect, set *<b>published_out</b> to the resource's
+ * publication time.
+ */
+static size_t
+spooled_resource_estimate_size(const spooled_resource_t *spooled,
+                               dir_connection_t *conn,
+                               int compressed,
+                               time_t *published_out)
+{
+  if (spooled->spool_eagerly) {
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen,
+                                         published_out);
+    if (r == -1 || body == NULL || bodylen == 0)
+      return 0;
+    if (compressed) {
+      double ratio = estimate_compression_ratio(spooled->spool_source);
+      bodylen = (size_t)(bodylen * ratio);
+    }
+    return bodylen;
   } else {
-    result = 0;
-    SMARTLIST_FOREACH(fps, const char *, digest, {
-        cached_dir_t *dir = lookup_cached_dir_by_fp(digest);
-        if (dir)
-          result += compressed ? dir->dir_z_len : dir->dir_len;
-      });
+    cached_dir_t *cached;
+    if (spooled->consensus_cache_entry) {
+      if (published_out) {
+        consensus_cache_entry_get_valid_after(
+            spooled->consensus_cache_entry, published_out);
+      }
+
+      return spooled->cce_len;
+    }
+    if (spooled->cached_dir_ref) {
+      cached = spooled->cached_dir_ref;
+    } else {
+      cached = spooled_resource_lookup_cached_dir(spooled,
+                                                  published_out);
+    }
+    if (cached == NULL) {
+      return 0;
+    }
+    size_t result = compressed ? cached->dir_compressed_len : cached->dir_len;
+    return result;
   }
-  return result;
 }
 
-/** Given a list of microdescriptor hashes, guess how many bytes will be
- * needed to transmit them, and return the guess. */
-size_t
-dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed)
-{
-  size_t result = smartlist_len(fps) * microdesc_average_size(NULL);
-  if (compressed)
-    result /= 2;
-  return result;
-}
+/** Return code for spooled_resource_flush_some */
+typedef enum {
+  SRFS_ERR = -1,
+  SRFS_MORE = 0,
+  SRFS_DONE
+} spooled_resource_flush_status_t;
 
-/** When we're spooling data onto our outbuf, add more whenever we dip
- * below this threshold. */
-#define DIRSERV_BUFFER_MIN 16384
+/** Flush some or all of the bytes from <b>spooled</b> onto <b>conn</b>.
+ * Return SRFS_ERR on error, SRFS_MORE if there are more bytes to flush from
+ * this spooled resource, or SRFS_DONE if we are done flushing this spooled
+ * resource.
+ */
+static spooled_resource_flush_status_t
+spooled_resource_flush_some(spooled_resource_t *spooled,
+                            dir_connection_t *conn)
+{
+  if (spooled->spool_eagerly) {
+    /* Spool_eagerly resources are sent all-at-once. */
+    const uint8_t *body = NULL;
+    size_t bodylen = 0;
+    int r = spooled_resource_lookup_body(spooled,
+                                         connection_dir_is_encrypted(conn),
+                                         &body, &bodylen, NULL);
+    if (r == -1 || body == NULL || bodylen == 0) {
+      /* Absent objects count as "done". */
+      return SRFS_DONE;
+    }
+    if (conn->compress_state) {
+      connection_write_to_buf_compress((const char*)body, bodylen, conn, 0);
+    } else {
+      connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn));
+    }
+    return SRFS_DONE;
+  } else {
+    cached_dir_t *cached = spooled->cached_dir_ref;
+    consensus_cache_entry_t *cce = spooled->consensus_cache_entry;
+    if (cached == NULL && cce == NULL) {
+      /* The cached_dir_t hasn't been materialized yet. So let's look it up. */
+      cached = spooled->cached_dir_ref =
+        spooled_resource_lookup_cached_dir(spooled, NULL);
+      if (!cached) {
+        /* Absent objects count as done. */
+        return SRFS_DONE;
+      }
+      ++cached->refcnt;
+      tor_assert_nonfatal(spooled->cached_dir_offset == 0);
+    }
 
-/** Spooling helper: called when we have no more data to spool to <b>conn</b>.
- * Flushes any remaining data to be (un)compressed, and changes the spool
- * source to NONE.  Returns 0 on success, negative on failure. */
-static int
-connection_dirserv_finish_spooling(dir_connection_t *conn)
-{
-  if (conn->zlib_state) {
-    connection_write_to_buf_zlib("", 0, conn, 1);
-    tor_zlib_free(conn->zlib_state);
-    conn->zlib_state = NULL;
+    if (BUG(!cached && !cce))
+      return SRFS_DONE;
+
+    int64_t total_len;
+    const char *ptr;
+    if (cached) {
+      total_len = cached->dir_compressed_len;
+      ptr = cached->dir_compressed;
+    } else {
+      total_len = spooled->cce_len;
+      ptr = (const char *)spooled->cce_body;
+    }
+    /* How many bytes left to flush? */
+    int64_t remaining;
+    remaining = total_len - spooled->cached_dir_offset;
+    if (BUG(remaining < 0))
+      return SRFS_ERR;
+    ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
+    if (conn->compress_state) {
+      connection_write_to_buf_compress(
+              ptr + spooled->cached_dir_offset,
+              bytes, conn, 0);
+    } else {
+      connection_write_to_buf(ptr + spooled->cached_dir_offset,
+                              bytes, TO_CONN(conn));
+    }
+    spooled->cached_dir_offset += bytes;
+    if (spooled->cached_dir_offset >= (off_t)total_len) {
+      return SRFS_DONE;
+    } else {
+      return SRFS_MORE;
+    }
   }
-  conn->dir_spool_src = DIR_SPOOL_NONE;
-  return 0;
 }
 
-/** Spooling helper: called when we're sending a bunch of server descriptors,
- * and the outbuf has become too empty. Pulls some entries from
- * fingerprint_stack, and writes the corresponding servers onto outbuf.  If we
- * run out of entries, flushes the zlib state and sets the spool source to
- * NONE.  Returns 0 on success, negative on failure.
+/** Helper: find the cached_dir_t for a spooled_resource_t, for
+ * sending it to <b>conn</b>. Set *<b>published_out</b>, if provided,
+ * to the published time of the cached_dir_t.
+ *
+ * DOES NOT increase the reference count on the result.  Callers must do that
+ * themselves if they mean to hang on to it.
  */
+static cached_dir_t *
+spooled_resource_lookup_cached_dir(const spooled_resource_t *spooled,
+                                   time_t *published_out)
+{
+  tor_assert(spooled->spool_eagerly == 0);
+  cached_dir_t *d = lookup_cached_dir_by_fp(spooled->digest);
+  if (d != NULL) {
+    if (published_out)
+      *published_out = d->published;
+  }
+  return d;
+}
+
+/** Helper: Look up the body for an eagerly-served spooled_resource.  If
+ * <b>conn_is_encrypted</b> is false, don't look up any resource that
+ * shouldn't be sent over an unencrypted connection.  On success, set
+ * <b>body_out</b>, <b>size_out</b>, and <b>published_out</b> to refer
+ * to the resource's body, size, and publication date, and return 0.
+ * On failure return -1. */
 static int
-connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn)
+spooled_resource_lookup_body(const spooled_resource_t *spooled,
+                             int conn_is_encrypted,
+                             const uint8_t **body_out,
+                             size_t *size_out,
+                             time_t *published_out)
 {
-  int by_fp = (conn->dir_spool_src == DIR_SPOOL_SERVER_BY_FP ||
-               conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP);
-  int extra = (conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_FP ||
-               conn->dir_spool_src == DIR_SPOOL_EXTRA_BY_DIGEST);
-  time_t publish_cutoff = time(NULL)-ROUTER_MAX_AGE_TO_PUBLISH;
+  tor_assert(spooled->spool_eagerly == 1);
 
-  const or_options_t *options = get_options();
+  const signed_descriptor_t *sd = NULL;
 
-  while (smartlist_len(conn->fingerprint_stack) &&
-         connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    const char *body;
-    char *fp = smartlist_pop_last(conn->fingerprint_stack);
-    const signed_descriptor_t *sd = NULL;
-    if (by_fp) {
-      sd = get_signed_descriptor_by_fp(fp, extra, publish_cutoff);
-    } else {
-      sd = extra ? extrainfo_get_by_descriptor_digest(fp)
-        : router_get_by_descriptor_digest(fp);
+  switch (spooled->spool_source) {
+    case DIR_SPOOL_EXTRA_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 1);
+      break;
     }
-    tor_free(fp);
-    if (!sd)
-      continue;
-    if (!connection_dir_is_encrypted(conn) && !sd->send_unencrypted) {
-      /* we did this check once before (so we could have an accurate size
-       * estimate and maybe send a 404 if somebody asked for only bridges on a
-       * connection), but we need to do it again in case a previously
-       * unknown bridge descriptor has shown up between then and now. */
-      continue;
+    case DIR_SPOOL_SERVER_BY_FP: {
+      sd = get_signed_descriptor_by_fp(spooled->digest, 0);
+      break;
     }
-
-    /** If we are the bridge authority and the descriptor is a bridge
-     * descriptor, remember that we served this descriptor for desc stats. */
-    if (options->BridgeAuthoritativeDir && by_fp) {
-      const routerinfo_t *router =
-          router_get_by_id_digest(sd->identity_digest);
-      /* router can be NULL here when the bridge auth is asked for its own
-       * descriptor. */
-      if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
-        rep_hist_note_desc_served(sd->identity_digest);
-    }
-    body = signed_descriptor_get_body(sd);
-    if (conn->zlib_state) {
-      int last = ! smartlist_len(conn->fingerprint_stack);
-      connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn,
-                                   last);
-      if (last) {
-        tor_zlib_free(conn->zlib_state);
-        conn->zlib_state = NULL;
+    case DIR_SPOOL_SERVER_BY_DIGEST: {
+      sd = router_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_EXTRA_BY_DIGEST: {
+      sd = extrainfo_get_by_descriptor_digest((const char *)spooled->digest);
+      break;
+    }
+    case DIR_SPOOL_MICRODESC: {
+      microdesc_t *md = microdesc_cache_lookup_by_digest256(
+                                  get_microdesc_cache(),
+                                  (const char *)spooled->digest);
+      if (! md || ! md->body) {
+        return -1;
       }
-    } else {
-      connection_write_to_buf(body,
-                              sd->signed_descriptor_len,
-                              TO_CONN(conn));
+      *body_out = (const uint8_t *)md->body;
+      *size_out = md->bodylen;
+      if (published_out)
+        *published_out = TIME_MAX;
+      return 0;
     }
+    case DIR_SPOOL_NETWORKSTATUS:
+    case DIR_SPOOL_CONSENSUS_CACHE_ENTRY:
+    default:
+      /* LCOV_EXCL_START */
+      tor_assert_nonfatal_unreached();
+      return -1;
+      /* LCOV_EXCL_STOP */
   }
 
-  if (!smartlist_len(conn->fingerprint_stack)) {
-    /* We just wrote the last one; finish up. */
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib("", 0, conn, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
-    conn->dir_spool_src = DIR_SPOOL_NONE;
-    smartlist_free(conn->fingerprint_stack);
-    conn->fingerprint_stack = NULL;
+  /* If we get here, then we tried to set "sd" to a signed_descriptor_t. */
+
+  if (sd == NULL) {
+    return -1;
   }
+  if (sd->send_unencrypted == 0 && ! conn_is_encrypted) {
+    /* we did this check once before (so we could have an accurate size
+     * estimate and maybe send a 404 if somebody asked for only bridges on
+     * a connection), but we need to do it again in case a previously
+     * unknown bridge descriptor has shown up between then and now. */
+    return -1;
+  }
+  *body_out = (const uint8_t *) signed_descriptor_get_body(sd);
+  *size_out = sd->signed_descriptor_len;
+  if (published_out)
+    *published_out = sd->published_on;
   return 0;
 }
 
-/** Spooling helper: called when we're sending a bunch of microdescriptors,
- * and the outbuf has become too empty. Pulls some entries from
- * fingerprint_stack, and writes the corresponding microdescs onto outbuf.  If
- * we run out of entries, flushes the zlib state and sets the spool source to
- * NONE.  Returns 0 on success, negative on failure.
- */
-static int
-connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn)
-{
-  microdesc_cache_t *cache = get_microdesc_cache();
-  while (smartlist_len(conn->fingerprint_stack) &&
-         connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    char *fp256 = smartlist_pop_last(conn->fingerprint_stack);
-    microdesc_t *md = microdesc_cache_lookup_by_digest256(cache, fp256);
-    tor_free(fp256);
-    if (!md || !md->body)
-      continue;
-    if (conn->zlib_state) {
-      int last = !smartlist_len(conn->fingerprint_stack);
-      connection_write_to_buf_zlib(md->body, md->bodylen, conn, last);
-      if (last) {
-        tor_zlib_free(conn->zlib_state);
-        conn->zlib_state = NULL;
-      }
-    } else {
-      connection_write_to_buf(md->body, md->bodylen, TO_CONN(conn));
-    }
-  }
-  if (!smartlist_len(conn->fingerprint_stack)) {
-    if (conn->zlib_state) {
-      connection_write_to_buf_zlib("", 0, conn, 1);
-      tor_zlib_free(conn->zlib_state);
-      conn->zlib_state = NULL;
-    }
-    conn->dir_spool_src = DIR_SPOOL_NONE;
-    smartlist_free(conn->fingerprint_stack);
-    conn->fingerprint_stack = NULL;
+/** Given a fingerprint <b>fp</b> which is either set if we're looking for a
+ * v2 status, or zeroes if we're looking for a v3 status, or a NUL-padded
+ * flavor name if we want a flavored v3 status, return a pointer to the
+ * appropriate cached dir object, or NULL if there isn't one available. */
+static cached_dir_t *
+lookup_cached_dir_by_fp(const uint8_t *fp)
+{
+  cached_dir_t *d = NULL;
+  if (tor_digest_is_zero((const char *)fp) && cached_consensuses) {
+    d = strmap_get(cached_consensuses, "ns");
+  } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses) {
+    /* this here interface is a nasty hack: we're shoving a flavor into
+     * a digest field. */
+    d = strmap_get(cached_consensuses, (const char *)fp);
   }
-  return 0;
+  return d;
 }
 
-/** Spooling helper: Called when we're sending a directory or networkstatus,
- * and the outbuf has become too empty.  Pulls some bytes from
- * <b>conn</b>-\>cached_dir-\>dir_z, uncompresses them if appropriate, and
- * puts them on the outbuf.  If we run out of entries, flushes the zlib state
- * and sets the spool source to NONE.  Returns 0 on success, negative on
- * failure. */
-static int
-connection_dirserv_add_dir_bytes_to_outbuf(dir_connection_t *conn)
-{
-  ssize_t bytes;
-  int64_t remaining;
-
-  bytes = DIRSERV_BUFFER_MIN - connection_get_outbuf_len(TO_CONN(conn));
-  tor_assert(bytes > 0);
-  tor_assert(conn->cached_dir);
-  if (bytes < 8192)
-    bytes = 8192;
-  remaining = conn->cached_dir->dir_z_len - conn->cached_dir_offset;
-  if (bytes > remaining)
-    bytes = (ssize_t) remaining;
-
-  if (conn->zlib_state) {
-    connection_write_to_buf_zlib(
-                             conn->cached_dir->dir_z + conn->cached_dir_offset,
-                             bytes, conn, bytes == remaining);
-  } else {
-    connection_write_to_buf(conn->cached_dir->dir_z + conn->cached_dir_offset,
-                            bytes, TO_CONN(conn));
+/** Try to guess the number of bytes that will be needed to send the
+ * spooled objects for <b>conn</b>'s outgoing spool.  In the process,
+ * remove every element of the spool that refers to an absent object, or
+ * which was published earlier than <b>cutoff</b>.  Set *<b>size_out</b>
+ * to the number of bytes, and *<b>n_expired_out</b> to the number of
+ * objects removed for being too old. */
+void
+dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                            time_t cutoff,
+                                            int compression,
+                                            size_t *size_out,
+                                            int *n_expired_out)
+{
+  if (BUG(!conn))
+    return;
+
+  smartlist_t *spool = conn->spool;
+  if (!spool) {
+    if (size_out)
+      *size_out = 0;
+    if (n_expired_out)
+      *n_expired_out = 0;
+    return;
   }
-  conn->cached_dir_offset += bytes;
-  if (conn->cached_dir_offset == (int)conn->cached_dir->dir_z_len) {
-    /* We just wrote the last one; finish up. */
-    connection_dirserv_finish_spooling(conn);
-    cached_dir_decref(conn->cached_dir);
-    conn->cached_dir = NULL;
+  int n_expired = 0;
+  uint64_t total = 0;
+  SMARTLIST_FOREACH_BEGIN(spool, spooled_resource_t *, spooled) {
+    time_t published = TIME_MAX;
+    size_t sz = spooled_resource_estimate_size(spooled, conn,
+                                               compression, &published);
+    if (published < cutoff) {
+      ++n_expired;
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else if (sz == 0) {
+      SMARTLIST_DEL_CURRENT(spool, spooled);
+      spooled_resource_free(spooled);
+    } else {
+      total += sz;
+    }
+  } SMARTLIST_FOREACH_END(spooled);
+
+  if (size_out) {
+    *size_out = (total > SIZE_MAX) ? SIZE_MAX : (size_t)total;
   }
-  return 0;
+  if (n_expired_out)
+    *n_expired_out = n_expired;
 }
 
-/** Spooling helper: Called when we're spooling networkstatus objects on
- * <b>conn</b>, and the outbuf has become too empty.  If the current
- * networkstatus object (in <b>conn</b>-\>cached_dir) has more data, pull data
- * from there.  Otherwise, pop the next fingerprint from fingerprint_stack,
- * and start spooling the next networkstatus.  (A digest of all 0 bytes is
- * treated as a request for the current consensus.) If we run out of entries,
- * flushes the zlib state and sets the spool source to NONE.  Returns 0 on
- * success, negative on failure. */
+/** Helper: used to sort a connection's spool. */
 static int
-connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn)
-{
-
-  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN) {
-    if (conn->cached_dir) {
-      int uncompressing = (conn->zlib_state != NULL);
-      int r = connection_dirserv_add_dir_bytes_to_outbuf(conn);
-      if (conn->dir_spool_src == DIR_SPOOL_NONE) {
-        /* add_dir_bytes thinks we're done with the cached_dir.  But we
-         * may have more cached_dirs! */
-        conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS;
-        /* This bit is tricky.  If we were uncompressing the last
-         * networkstatus, we may need to make a new zlib object to
-         * uncompress the next one. */
-        if (uncompressing && ! conn->zlib_state &&
-            conn->fingerprint_stack &&
-            smartlist_len(conn->fingerprint_stack)) {
-          conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-        }
-      }
-      if (r) return r;
-    } else if (conn->fingerprint_stack &&
-               smartlist_len(conn->fingerprint_stack)) {
-      /* Add another networkstatus; start serving it. */
-      char *fp = smartlist_pop_last(conn->fingerprint_stack);
-      cached_dir_t *d = lookup_cached_dir_by_fp(fp);
-      tor_free(fp);
-      if (d) {
-        ++d->refcnt;
-        conn->cached_dir = d;
-        conn->cached_dir_offset = 0;
-      }
-    } else {
-      connection_dirserv_finish_spooling(conn);
-      smartlist_free(conn->fingerprint_stack);
-      conn->fingerprint_stack = NULL;
-      return 0;
+dirserv_spool_sort_comparison_(const void **a_, const void **b_)
+{
+  const spooled_resource_t *a = *a_;
+  const spooled_resource_t *b = *b_;
+  return fast_memcmp(a->digest, b->digest, sizeof(a->digest));
+}
+
+/** Sort all the entries in <b>conn</b> by digest. */
+void
+dirserv_spool_sort(dir_connection_t *conn)
+{
+  if (conn->spool == NULL)
+    return;
+  smartlist_sort(conn->spool, dirserv_spool_sort_comparison_);
+}
+
+/** Return the cache-info for identity fingerprint <b>fp</b>, or
+ * its extra-info document if <b>extrainfo</b> is true. Return
+ * NULL if not found or if the descriptor is older than
+ * <b>publish_cutoff</b>. */
+static const signed_descriptor_t *
+get_signed_descriptor_by_fp(const uint8_t *fp, int extrainfo)
+{
+  if (router_digest_is_me((const char *)fp)) {
+    if (extrainfo)
+      return &(router_get_my_extrainfo()->cache_info);
+    else
+      return &(router_get_my_routerinfo()->cache_info);
+  } else {
+    const routerinfo_t *ri = router_get_by_id_digest((const char *)fp);
+    if (ri) {
+      if (extrainfo)
+        return extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+      else
+        return &ri->cache_info;
     }
   }
-  return 0;
+  return NULL;
 }
 
-/** Called whenever we have flushed some directory data in state
- * SERVER_WRITING. */
+/** When we're spooling data onto our outbuf, add more whenever we dip
+ * below this threshold. */
+#define DIRSERV_BUFFER_MIN 16384
+
+/**
+ * Called whenever we have flushed some directory data in state
+ * SERVER_WRITING, or whenever we want to fill the buffer with initial
+ * directory data (so that subsequent writes will occur, and trigger this
+ * function again.)
+ *
+ * Return 0 on success, and -1 on failure.
+ */
 int
 connection_dirserv_flushed_some(dir_connection_t *conn)
 {
   tor_assert(conn->base_.state == DIR_CONN_STATE_SERVER_WRITING);
-
-  if (connection_get_outbuf_len(TO_CONN(conn)) >= DIRSERV_BUFFER_MIN)
+  if (conn->spool == NULL)
     return 0;
 
-  switch (conn->dir_spool_src) {
-    case DIR_SPOOL_EXTRA_BY_DIGEST:
-    case DIR_SPOOL_EXTRA_BY_FP:
-    case DIR_SPOOL_SERVER_BY_DIGEST:
-    case DIR_SPOOL_SERVER_BY_FP:
-      return connection_dirserv_add_servers_to_outbuf(conn);
-    case DIR_SPOOL_MICRODESC:
-      return connection_dirserv_add_microdescs_to_outbuf(conn);
-    case DIR_SPOOL_CACHED_DIR:
-      return connection_dirserv_add_dir_bytes_to_outbuf(conn);
-    case DIR_SPOOL_NETWORKSTATUS:
-      return connection_dirserv_add_networkstatus_bytes_to_outbuf(conn);
-    case DIR_SPOOL_NONE:
-    default:
+  while (connection_get_outbuf_len(TO_CONN(conn)) < DIRSERV_BUFFER_MIN &&
+         smartlist_len(conn->spool)) {
+    spooled_resource_t *spooled =
+      smartlist_get(conn->spool, smartlist_len(conn->spool)-1);
+    spooled_resource_flush_status_t status;
+    status = spooled_resource_flush_some(spooled, conn);
+    if (status == SRFS_ERR) {
+      return -1;
+    } else if (status == SRFS_MORE) {
       return 0;
+    }
+    tor_assert(status == SRFS_DONE);
+
+    /* If we're here, we're done flushing this resource. */
+    tor_assert(smartlist_pop_last(conn->spool) == spooled);
+    spooled_resource_free(spooled);
+  }
+
+  if (smartlist_len(conn->spool) > 0) {
+    /* We're still spooling something. */
+    return 0;
+  }
+
+  /* If we get here, we're done. */
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
+  if (conn->compress_state) {
+    /* Flush the compression state: there could be more bytes pending in there,
+     * and we don't want to omit bytes. */
+    connection_write_to_buf_compress("", 0, conn, 1);
+    tor_compress_free(conn->compress_state);
+    conn->compress_state = NULL;
   }
+  return 0;
+}
+
+/** Remove every element from <b>conn</b>'s outgoing spool, and delete
+ * the spool. */
+void
+dir_conn_clear_spool(dir_connection_t *conn)
+{
+  if (!conn || ! conn->spool)
+    return;
+  SMARTLIST_FOREACH(conn->spool, spooled_resource_t *, s,
+                    spooled_resource_free(s));
+  smartlist_free(conn->spool);
+  conn->spool = NULL;
 }
 
 /** Return true iff <b>line</b> is a valid RecommendedPackages line.
diff --git a/src/or/dirserv.h b/src/or/dirserv.h
index 1e4f27e3d7..480174d5bb 100644
--- a/src/or/dirserv.h
+++ b/src/or/dirserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,6 +32,61 @@
 /** Maximum allowable length of a version line in a networkstatus. */
 #define MAX_V_LINE_LEN 128
 
+/** Ways to convert a spoolable_resource_t to a bunch of bytes. */
+typedef enum dir_spool_source_t {
+    DIR_SPOOL_SERVER_BY_DIGEST=1, DIR_SPOOL_SERVER_BY_FP,
+    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
+    DIR_SPOOL_MICRODESC,
+    DIR_SPOOL_NETWORKSTATUS,
+    DIR_SPOOL_CONSENSUS_CACHE_ENTRY,
+} dir_spool_source_t;
+#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
+
+/** Object to remember the identity of an object that we are spooling,
+ * or about to spool, in response to a directory request.
+ *
+ * (Why do we spool?  Because some directory responses are very large,
+ * and we don't want to just shove the complete answer into the output
+ * buffer: that would take a ridiculous amount of RAM.)
+ *
+ * If the spooled resource is relatively small (like microdescriptors,
+ * descriptors, etc), we look them up by ID as needed, and add the whole
+ * thing onto the output buffer at once.  If the spooled reseource is
+ * big (like networkstatus documents), we reference-count it, and add it
+ * a few K at a time.
+ */
+typedef struct spooled_resource_t {
+  /**
+   * If true, we add the entire object to the outbuf.  If false,
+   * we spool the object a few K at a time.
+   */
+  unsigned spool_eagerly : 1;
+  /**
+   * Tells us what kind of object to get, and how to look it up.
+   */
+  dir_spool_source_bitfield_t spool_source : 7;
+  /**
+   * Tells us the specific object to spool.
+   */
+  uint8_t digest[DIGEST256_LEN];
+  /**
+   * A large object that we're spooling. Holds a reference count.  Only
+   * used when spool_eagerly is false.
+   */
+  struct cached_dir_t *cached_dir_ref;
+  /**
+   * A different kind of large object that we might be spooling. Also
+   * reference-counted.  Also only used when spool_eagerly is false.
+   */
+  struct consensus_cache_entry_t *consensus_cache_entry;
+  const uint8_t *cce_body;
+  size_t cce_len;
+  /**
+   * The current offset into cached_dir or cce_body. Only used when
+   * spool_eagerly is false */
+  off_t cached_dir_offset;
+} spooled_resource_t;
+
 int connection_dirserv_flushed_some(dir_connection_t *conn);
 
 int dirserv_add_own_fingerprint(crypto_pk_t *pk);
@@ -63,17 +118,19 @@ cached_dir_t *dirserv_get_consensus(const char *flavor_name);
 void dirserv_set_cached_consensus_networkstatus(const char *consensus,
                                               const char *flavor_name,
                                               const common_digests_t *digests,
+                                              const uint8_t *sha3_as_signed,
                                               time_t published);
 void dirserv_clear_old_networkstatuses(time_t cutoff);
-int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key,
-                                        const char **msg,
-                                        int for_unencrypted_conn,
-                                        int is_extrainfo);
+int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
+                                 dir_spool_source_t source,
+                                 int conn_is_encrytped,
+                                 const char **msg_out);
 int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
                             const char **msg);
 void dirserv_orconn_tls_done(const tor_addr_t *addr,
                              uint16_t or_port,
-                             const char *digest_rcvd);
+                             const char *digest_rcvd,
+                             const ed25519_public_key_t *ed_id_rcvd);
 int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
                                             const routerinfo_t *ri_old);
 void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
@@ -88,13 +145,6 @@ void dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                       uint32_t authstatus);
 
 int dirserv_would_reject_router(const routerstatus_t *rs);
-int dirserv_remove_old_statuses(smartlist_t *fps, time_t cutoff);
-int dirserv_have_any_serverdesc(smartlist_t *fps, int spool_src);
-int dirserv_have_any_microdesc(const smartlist_t *fps);
-size_t dirserv_estimate_data_size(smartlist_t *fps, int is_serverdescs,
-                                  int compressed);
-size_t dirserv_estimate_microdesc_size(const smartlist_t *fps, int compressed);
-
 char *routerstatus_format_entry(
                               const routerstatus_t *rs,
                               const char *version,
@@ -140,5 +190,19 @@ int dirserv_read_measured_bandwidths(const char *from_file,
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);
 
+spooled_resource_t *spooled_resource_new(dir_spool_source_t source,
+                                         const uint8_t *digest,
+                                         size_t digestlen);
+spooled_resource_t *spooled_resource_new_from_cache_entry(
+                                      struct consensus_cache_entry_t *entry);
+void spooled_resource_free(spooled_resource_t *spooled);
+void dirserv_spool_remove_missing_and_guess_size(dir_connection_t *conn,
+                                                 time_t cutoff,
+                                                 int compression,
+                                                 size_t *size_out,
+                                                 int *n_expired_out);
+void dirserv_spool_sort(dir_connection_t *conn);
+void dir_conn_clear_spool(dir_connection_t *conn);
+
 #endif
 
diff --git a/src/or/dirvote.c b/src/or/dirvote.c
index 738ab35bc1..f5e29eb786 100644
--- a/src/or/dirvote.c
+++ b/src/or/dirvote.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRVOTE_PRIVATE
@@ -26,6 +26,39 @@
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
+ *
+ * This module is the center of the consensus-voting based directory
+ * authority system.  With this system, a set of authorities first
+ * publish vote based on their opinions of the network, and then compute
+ * a consensus from those votes.  Each authority signs the consensus,
+ * and clients trust the consensus if enough known authorities have
+ * signed it.
+ *
+ * The code in this module is only invoked on directory authorities.  It's
+ * responsible for:
+ *
+ * <ul>
+ *   <li>Generating this authority's vote networkstatus, based on the
+ *       authority's view of the network as represented in dirserv.c
+ *   <li>Formatting the vote networkstatus objects.
+ *   <li>Generating the microdescriptors that correspond to our own
+ *       vote.
+ *   <li>Sending votes to all the other authorities.
+ *   <li>Trying to fetch missing votes from other authorities.
+ *   <li>Computing the consensus from a set of votes, as well as
+ *       a "detached signature" object for other authorities to fetch.
+ *   <li>Collecting other authorities' signatures on the same consensus,
+ *       until there are enough.
+ *   <li>Publishing the consensus to the reset of the directory system.
+ *   <li>Scheduling all of the above operations.
+ * </ul>
+ *
+ * The main entry points are in dirvote_act(), which handles scheduled
+ * actions; and dirvote_add_vote() and dirvote_add_signatures(), which
+ * handle uploaded and downloaded votes and signatures.
+ *
+ * (See dir-spec.txt from torspec.git for a complete specification of
+ * the directory protocol and voting algorithms.)
  **/
 
 /** A consensus that we have built and are appending signatures to.  Once it's
@@ -250,11 +283,11 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       smartlist_add(chunks, rsf);
 
     for (h = vrs->microdesc; h; h = h->next) {
-      smartlist_add(chunks, tor_strdup(h->microdesc_hash_line));
+      smartlist_add_strdup(chunks, h->microdesc_hash_line);
     }
   } SMARTLIST_FOREACH_END(vrs);
 
-  smartlist_add(chunks, tor_strdup("directory-footer\n"));
+  smartlist_add_strdup(chunks, "directory-footer\n");
 
   /* The digest includes everything up through the space after
    * directory-signature.  (Yuck.) */
@@ -894,7 +927,7 @@ networkstatus_check_weights(int64_t Wgg, int64_t Wgd, int64_t Wmg,
  *
  * It returns true if weights could be computed, false otherwise.
  */
-static int
+int
 networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
                                      int64_t M, int64_t E, int64_t D,
                                      int64_t T, int64_t weight_scale)
@@ -976,7 +1009,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
         Wgd = weight_scale;
       }
     } else { // Subcase b: R+D >= S
-      casename = "Case 2b1 (Wgg=1, Wmd=Wgd)";
+      casename = "Case 2b1 (Wgg=weight_scale, Wmd=Wgd)";
       Wee = (weight_scale*(E - G + M))/E;
       Wed = (weight_scale*(D - 2*E + 4*G - 2*M))/(3*D);
       Wme = (weight_scale*(G-M))/E;
@@ -989,7 +1022,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
                                        weight_scale, G, M, E, D, T, 10, 1);
 
       if (berr) {
-        casename = "Case 2b2 (Wgg=1, Wee=1)";
+        casename = "Case 2b2 (Wgg=weight_scale, Wee=weight_scale)";
         Wgg = weight_scale;
         Wee = weight_scale;
         Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
@@ -1058,7 +1091,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
     } else { // Subcase b: S+D >= T/3
       // D != 0 because S+D >= T/3
       if (G < E) {
-        casename = "Case 3bg (G scarce, Wgg=1, Wmd == Wed)";
+        casename = "Case 3bg (G scarce, Wgg=weight_scale, Wmd == Wed)";
         Wgg = weight_scale;
         Wgd = (weight_scale*(D - 2*G + E + M))/(3*D);
         Wmg = 0;
@@ -1070,7 +1103,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
         berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee,
                     Wed, weight_scale, G, M, E, D, T, 10, 1);
       } else { // G >= E
-        casename = "Case 3be (E scarce, Wee=1, Wmd == Wgd)";
+        casename = "Case 3be (E scarce, Wee=weight_scale, Wmd == Wgd)";
         Wee = weight_scale;
         Wed = (weight_scale*(D - 2*E + G + M))/(3*D);
         Wme = 0;
@@ -1104,7 +1137,7 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
   tor_assert(0 < weight_scale && weight_scale <= INT32_MAX);
 
   /*
-   * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine
+   * Provide Wgm=Wgg, Wmm=weight_scale, Wem=Wee, Weg=Wed. May later determine
    * that middle nodes need different bandwidth weights for dirport traffic,
    * or that weird exit policies need special weight, or that bridges
    * need special weight.
@@ -1287,7 +1320,17 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes)
  * value in a newly allocated string.
  *
  * Note: this function DOES NOT check whether the votes are from
- * recognized authorities.   (dirvote_add_vote does that.) */
+ * recognized authorities.   (dirvote_add_vote does that.)
+ *
+ * <strong>WATCH OUT</strong>: You need to think before you change the
+ * behavior of this function, or of the functions it calls! If some
+ * authorities compute the consensus with a different algorithm than
+ * others, they will not reach the same result, and they will not all
+ * sign the same thing!  If you really need to change the algorithm
+ * here, you should allocate a new "consensus_method" for the new
+ * behavior, and make the new behavior conditional on a new-enough
+ * consensus_method.
+ **/
 char *
 networkstatus_compute_consensus(smartlist_t *votes,
                                 int total_authorities,
@@ -1306,7 +1349,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   smartlist_t *flags;
   const char *flavor_name;
   uint32_t max_unmeasured_bw_kb = DEFAULT_MAX_UNMEASURED_BW_KB;
-  int64_t G=0, M=0, E=0, D=0, T=0; /* For bandwidth weights */
+  int64_t G, M, E, D, T; /* For bandwidth weights */
   const routerstatus_format_type_t rs_format =
     flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
   char *params = NULL;
@@ -1338,6 +1381,16 @@ networkstatus_compute_consensus(smartlist_t *votes,
     consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD;
   }
 
+  if (consensus_method >= MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE) {
+    /* It's smarter to initialize these weights to 1, so that later on,
+     * we can't accidentally divide by zero. */
+    G = M = E = D = 1;
+    T = 4;
+  } else {
+    /* ...but originally, they were set to zero. */
+    G = M = E = D = T = 0;
+  }
+
   /* Compute medians of time-related things, and figure out how many
    * routers we might need to talk about. */
   {
@@ -1377,7 +1430,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         smartlist_free(sv); /* elements get freed later. */
       }
       SMARTLIST_FOREACH(v->known_flags, const char *, cp,
-                        smartlist_add(flags, tor_strdup(cp)));
+                        smartlist_add_strdup(flags, cp));
     } SMARTLIST_FOREACH_END(v);
     valid_after = median_time(va_times, n_votes);
     fresh_until = median_time(fu_times, n_votes);
@@ -1410,7 +1463,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_free(combined_client_versions);
 
     if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING)
-      smartlist_add(flags, tor_strdup("NoEdConsensus"));
+      smartlist_add_strdup(flags, "NoEdConsensus");
 
     smartlist_sort_strings(flags);
     smartlist_uniq_strings(flags);
@@ -1474,9 +1527,9 @@ networkstatus_compute_consensus(smartlist_t *votes,
                                       total_authorities);
   if (smartlist_len(param_list)) {
     params = smartlist_join_strings(param_list, " ", 0, NULL);
-    smartlist_add(chunks, tor_strdup("params "));
+    smartlist_add_strdup(chunks, "params ");
     smartlist_add(chunks, params);
-    smartlist_add(chunks, tor_strdup("\n"));
+    smartlist_add_strdup(chunks, "\n");
   }
 
   if (consensus_method >= MIN_METHOD_FOR_SHARED_RANDOM) {
@@ -2063,10 +2116,10 @@ networkstatus_compute_consensus(smartlist_t *votes,
                     smartlist_join_strings(chosen_flags, " ", 0, NULL));
       /*     Now the version line. */
       if (chosen_version) {
-        smartlist_add(chunks, tor_strdup("\nv "));
-        smartlist_add(chunks, tor_strdup(chosen_version));
+        smartlist_add_strdup(chunks, "\nv ");
+        smartlist_add_strdup(chunks, chosen_version);
       }
-      smartlist_add(chunks, tor_strdup("\n"));
+      smartlist_add_strdup(chunks, "\n");
       if (chosen_protocol_list &&
           consensus_method >= MIN_METHOD_FOR_RS_PROTOCOLS) {
         smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list);
@@ -2119,7 +2172,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
   }
 
   /* Mark the directory footer region */
-  smartlist_add(chunks, tor_strdup("directory-footer\n"));
+  smartlist_add_strdup(chunks, "directory-footer\n");
 
   {
     int64_t weight_scale = BW_WEIGHT_SCALE;
@@ -2170,7 +2223,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     const char *algname = crypto_digest_algorithm_get_name(digest_alg);
     char *signature;
 
-    smartlist_add(chunks, tor_strdup("directory-signature "));
+    smartlist_add_strdup(chunks, "directory-signature ");
 
     /* Compute the hash of the chunks. */
     crypto_digest_smartlist(digest, digest_len, chunks, "", digest_alg);
@@ -2197,7 +2250,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
     smartlist_add(chunks, signature);
 
     if (legacy_id_key_digest && legacy_signing_key) {
-      smartlist_add(chunks, tor_strdup("directory-signature "));
+      smartlist_add_strdup(chunks, "directory-signature ");
       base16_encode(fingerprint, sizeof(fingerprint),
                     legacy_id_key_digest, DIGEST_LEN);
       crypto_pk_get_fingerprint(legacy_signing_key,
@@ -2510,7 +2563,7 @@ networkstatus_format_signatures(networkstatus_t *consensus,
       base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len,
                     BASE64_ENCODE_MULTILINE);
       strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf));
-      smartlist_add(elements, tor_strdup(buf));
+      smartlist_add_strdup(elements, buf);
     } SMARTLIST_FOREACH_END(sig);
   } SMARTLIST_FOREACH_END(v);
 
@@ -3620,8 +3673,8 @@ dirvote_add_signatures(const char *detached_signatures_body,
                        "Queuing it for the next consensus.", source);
     if (!pending_consensus_signature_list)
       pending_consensus_signature_list = smartlist_new();
-    smartlist_add(pending_consensus_signature_list,
-                  tor_strdup(detached_signatures_body));
+    smartlist_add_strdup(pending_consensus_signature_list,
+                  detached_signatures_body);
     *msg = "Signature queued";
     return 0;
   }
diff --git a/src/or/dirvote.h b/src/or/dirvote.h
index efd233ef5f..e342dc78ea 100644
--- a/src/or/dirvote.h
+++ b/src/or/dirvote.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -55,7 +55,7 @@
 #define MIN_SUPPORTED_CONSENSUS_METHOD 13
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 25
+#define MAX_SUPPORTED_CONSENSUS_METHOD 26
 
 /** Lowest consensus method where microdesc consensuses omit any entry
  * with no microdesc. */
@@ -111,6 +111,10 @@
  * entries. */
 #define MIN_METHOD_FOR_RS_PROTOCOLS 25
 
+/** Lowest consensus method where authorities initialize bandwidth weights to 1
+ * instead of 0. See #14881 */
+#define MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE 26
+
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
  * get confused with the above macros.) */
@@ -234,6 +238,10 @@ STATIC smartlist_t *dirvote_compute_params(smartlist_t *votes, int method,
                              int total_authorities);
 STATIC char *compute_consensus_package_lines(smartlist_t *votes);
 STATIC char *make_consensus_method_list(int low, int high, const char *sep);
+STATIC int
+networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G,
+                                     int64_t M, int64_t E, int64_t D,
+                                     int64_t T, int64_t weight_scale);
 #endif
 
 #endif
diff --git a/src/or/dns.c b/src/or/dns.c
index c1e3c3256e..b5344469b5 100644
--- a/src/or/dns.c
+++ b/src/or/dns.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -160,8 +160,9 @@ evdns_log_cb(int warn, const char *msg)
   }
   if (!strcmpstart(msg, "Nameserver ") && (cp=strstr(msg, " has failed: "))) {
     char *ns = tor_strndup(msg+11, cp-(msg+11));
-    const char *err = strchr(cp, ':')+2;
-    tor_assert(err);
+    const char *colon = strchr(cp, ':');
+    tor_assert(colon);
+    const char *err = colon+2;
     /* Don't warn about a single failed nameserver; we'll warn with 'all
      * nameservers have failed' if we're completely out of nameservers;
      * otherwise, the situation is tolerable. */
@@ -1759,7 +1760,7 @@ wildcard_increment_answer(const char *id)
         "invalid addresses. Apparently they are hijacking DNS failures. "
         "I'll try to correct for this by treating future occurrences of "
         "\"%s\" as 'not found'.", id, *ip, id);
-      smartlist_add(dns_wildcard_list, tor_strdup(id));
+      smartlist_add_strdup(dns_wildcard_list, id);
     }
     if (!dns_wildcard_notice_given)
       control_event_server_status(LOG_NOTICE, "DNS_HIJACKED");
@@ -1783,7 +1784,7 @@ add_wildcarded_test_address(const char *address)
   n_test_addrs = get_options()->ServerDNSTestAddresses ?
     smartlist_len(get_options()->ServerDNSTestAddresses) : 0;
 
-  smartlist_add(dns_wildcarded_test_address_list, tor_strdup(address));
+  smartlist_add_strdup(dns_wildcarded_test_address_list, address);
   n = smartlist_len(dns_wildcarded_test_address_list);
   if (n > n_test_addrs/2) {
     tor_log(dns_wildcarded_test_address_notice_given ? LOG_INFO : LOG_NOTICE,
@@ -2103,8 +2104,8 @@ assert_cache_ok_(void)
 
 #endif
 
-cached_resolve_t
-*dns_get_cache_entry(cached_resolve_t *query)
+cached_resolve_t *
+dns_get_cache_entry(cached_resolve_t *query)
 {
   return HT_FIND(cache_map, &cache_root, query);
 }
diff --git a/src/or/dns.h b/src/or/dns.h
index 951a2a3467..a81cbd20da 100644
--- a/src/or/dns.h
+++ b/src/or/dns.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/dns_structs.h b/src/or/dns_structs.h
index bc6067213d..dc00e9f7b9 100644
--- a/src/or/dns_structs.h
+++ b/src/or/dns_structs.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c
index f5a4f2ac0f..54a22a5150 100644
--- a/src/or/dnsserv.c
+++ b/src/or/dnsserv.c
@@ -1,12 +1,24 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file dnsserv.c
- * \brief Implements client-side DNS proxy server code.  Note:
- * this is the DNS Server code, not the Server DNS code.  Confused?  This code
- * runs on client-side, and acts as a DNS server.  The code in dns.c, on the
- * other hand, runs on Tor servers, and acts as a DNS client.
+ * \brief Implements client-side DNS proxy server code.
+ *
+ * When a user enables the DNSPort configuration option to have their local
+ * Tor client handle DNS requests, this module handles it.  It functions as a
+ * "DNS Server" on the client side, which client applications use.
+ *
+ * Inbound DNS requests are represented as entry_connection_t here (since
+ * that's how Tor represents client-side streams), which are kept associated
+ * with an evdns_server_request structure as exposed by Libevent's
+ * evdns code.
+ *
+ * Upon receiving a DNS request, libevent calls our evdns_server_callback()
+ * function here, which causes this module to create an entry_connection_t
+ * request as appropriate.  Later, when that request is answered,
+ * connection_edge.c calls dnsserv_resolved() so we can finish up and tell the
+ * DNS client.
  **/
 
 #include "or.h"
@@ -272,7 +284,7 @@ dnsserv_reject_request(entry_connection_t *conn)
 }
 
 /** Look up the original name that corresponds to 'addr' in req.  We use this
- * to preserve case in order to facilitate people using 0x20-hacks to avoid
+ * to preserve case in order to facilitate clients using 0x20-hacks to avoid
  * DNS poisoning. */
 static const char *
 evdns_get_orig_address(const struct evdns_server_request *req,
diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h
index ad0e248c83..6c0643b8dc 100644
--- a/src/or/dnsserv.h
+++ b/src/or/dnsserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c
index 265b6dcda1..2cbc8019d4 100644
--- a/src/or/entrynodes.c
+++ b/src/or/entrynodes.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,18 +10,118 @@
  *
  * Entry nodes can be guards (for general use) or bridges (for censorship
  * circumvention).
+ *
+ * In general, we use entry guards to prevent traffic-sampling attacks:
+ * if we chose every circuit independently, an adversary controlling
+ * some fraction of paths on the network would observe a sample of every
+ * user's traffic. Using guards gives users a chance of not being
+ * profiled.
+ *
+ * The current entry guard selection code is designed to try to avoid
+ * _ever_ trying every guard on the network, to try to stick to guards
+ * that we've used before, to handle hostile/broken networks, and
+ * to behave sanely when the network goes up and down.
+ *
+ * Our algorithm works as follows: First, we maintain a SAMPLE of guards
+ * we've seen in the networkstatus consensus.  We maintain this sample
+ * over time, and store it persistently; it is chosen without reference
+ * to our configuration or firewall rules.  Guards remain in the sample
+ * as they enter and leave the consensus.  We expand this sample as
+ * needed, up to a maximum size.
+ *
+ * As a subset of the sample, we maintain a FILTERED SET of the guards
+ * that we would be willing to use if we could connect to them.  The
+ * filter removes all the guards that we're excluding because they're
+ * bridges (or not bridges), because we have restrictive firewall rules,
+ * because of ExcludeNodes, because we of path bias restrictions,
+ * because they're absent from the network at present, and so on.
+ *
+ * As a subset of the filtered set, we keep a REACHABLE FILTERED SET
+ * (also called a "usable filtered set") of those guards that we call
+ * "reachable" or "maybe reachable".  A guard is reachable if we've
+ * connected to it more recently than we've failed.  A guard is "maybe
+ * reachable" if we have never tried to connect to it, or if we
+ * failed to connect to it so long ago that we no longer think our
+ * failure means it's down.
+ *
+ * As a persistent ordered list whose elements are taken from the
+ * sampled set, we track a CONFIRMED GUARDS LIST.  A guard becomes
+ * confirmed when we successfully build a circuit through it, and decide
+ * to use that circuit.  We order the guards on this list by the order
+ * in which they became confirmed.
+ *
+ * And as a final group, we have an ordered list of PRIMARY GUARDS,
+ * whose elements are taken from the filtered set. We prefer
+ * confirmed guards to non-confirmed guards for this list, and place
+ * other restrictions on it.  The primary guards are the ones that we
+ * connect to "when nothing is wrong" -- circuits through them can be used
+ * immediately.
+ *
+ * To build circuits, we take a primary guard if possible -- or a
+ * reachable filtered confirmed guard if no primary guard is possible --
+ * or a random reachable filtered guard otherwise.  If the guard is
+ * primary, we can use the circuit immediately on success.  Otherwise,
+ * the guard is now "pending" -- we won't use its circuit unless all
+ * of the circuits we're trying to build through better guards have
+ * definitely failed.
+ *
+ * While we're building circuits, we track a little "guard state" for
+ * each circuit. We use this to keep track of whether the circuit is
+ * one that we can use as soon as it's done, or whether it's one that
+ * we should keep around to see if we can do better.  In the latter case,
+ * a periodic call to entry_guards_upgrade_waiting_circuits() will
+ * eventually upgrade it.
  **/
+/* DOCDOC -- expand this.
+ *
+ * Information invariants:
+ *
+ * [x] whenever a guard becomes unreachable, clear its usable_filtered flag.
+ *
+ * [x] Whenever a guard becomes reachable or maybe-reachable, if its filtered
+ * flag is set, set its usable_filtered flag.
+ *
+ * [x] Whenever we get a new consensus, call update_from_consensus(). (LATER.)
+ *
+ * [x] Whenever the configuration changes in a relevant way, update the
+ * filtered/usable flags. (LATER.)
+ *
+ * [x] Whenever we add a guard to the sample, make sure its filtered/usable
+ * flags are set as possible.
+ *
+ * [x] Whenever we remove a guard from the sample, remove it from the primary
+ * and confirmed lists.
+ *
+ * [x] When we make a guard confirmed, update the primary list.
+ *
+ * [x] When we make a guard filtered or unfiltered, update the primary list.
+ *
+ * [x] When we are about to pick a guard, make sure that the primary list is
+ * full.
+ *
+ * [x] Before calling sample_reachable_filtered_entry_guards(), make sure
+ * that the filtered, primary, and confirmed flags are up-to-date.
+ *
+ * [x] Call entry_guard_consider_retry every time we are about to check
+ * is_usable_filtered or is_reachable, and every time we set
+ * is_filtered to 1.
+ *
+ * [x] Call entry_guards_changed_for_guard_selection() whenever we update
+ * a persistent field.
+ */
 
 #define ENTRYNODES_PRIVATE
 
 #include "or.h"
+#include "channel.h"
+#include "bridges.h"
 #include "circpathbias.h"
 #include "circuitbuild.h"
+#include "circuitlist.h"
 #include "circuitstats.h"
 #include "config.h"
 #include "confparse.h"
 #include "connection.h"
-#include "connection_or.h"
 #include "control.h"
 #include "directory.h"
 #include "entrynodes.h"
@@ -37,2509 +137,3468 @@
 #include "transports.h"
 #include "statefile.h"
 
-/** Information about a configured bridge. Currently this just matches the
- * ones in the torrc file, but one day we may be able to learn about new
- * bridges on our own, and remember them in the state file. */
-typedef struct {
-  /** Address of the bridge. */
-  tor_addr_t addr;
-  /** TLS port for the bridge. */
-  uint16_t port;
-  /** Boolean: We are re-parsing our bridge list, and we are going to remove
-   * this one if we don't find it in the list of configured bridges. */
-  unsigned marked_for_removal : 1;
-  /** Expected identity digest, or all zero bytes if we don't know what the
-   * digest should be. */
-  char identity[DIGEST_LEN];
-
-  /** Name of pluggable transport protocol taken from its config line. */
-  char *transport_name;
-
-  /** When should we next try to fetch a descriptor for this bridge? */
-  download_status_t fetch_status;
-
-  /** A smartlist of k=v values to be passed to the SOCKS proxy, if
-      transports are used for this bridge. */
-  smartlist_t *socks_args;
-} bridge_info_t;
-
-/** A list of our chosen entry guards. */
-static smartlist_t *entry_guards = NULL;
-/** A value of 1 means that the entry_guards list has changed
+/** A list of existing guard selection contexts. */
+static smartlist_t *guard_contexts = NULL;
+/** The currently enabled guard selection context. */
+static guard_selection_t *curr_guard_context = NULL;
+
+/** A value of 1 means that at least one context has changed,
  * and those changes need to be flushed to disk. */
 static int entry_guards_dirty = 0;
 
-static void bridge_free(bridge_info_t *bridge);
-static const node_t *choose_random_entry_impl(cpath_build_state_t *state,
-                                              int for_directory,
-                                              dirinfo_type_t dirtype,
-                                              int *n_options_out);
-static int num_bridges_usable(void);
-
-/* Default number of entry guards in the case where the NumEntryGuards
- * consensus parameter is not set */
-#define DEFAULT_N_GUARDS 1
-/* Minimum and maximum number of entry guards (in case the NumEntryGuards
- * consensus parameter is set). */
-#define MIN_N_GUARDS 1
-#define MAX_N_GUARDS 10
-
-/** Return the list of entry guards, creating it if necessary. */
-const smartlist_t *
-get_entry_guards(void)
-{
-  if (! entry_guards)
-    entry_guards = smartlist_new();
-  return entry_guards;
-}
-
-/** Check whether the entry guard <b>e</b> is usable, given the directory
- * authorities' opinion about the router (stored in <b>ri</b>) and the user's
- * configuration (in <b>options</b>). Set <b>e</b>->bad_since
- * accordingly. Return true iff the entry guard's status changes.
- *
- * If it's not usable, set *<b>reason</b> to a static string explaining why.
- */
-static int
-entry_guard_set_status(entry_guard_t *e, const node_t *node,
-                       time_t now, const or_options_t *options,
-                       const char **reason)
+static void entry_guard_set_filtered_flags(const or_options_t *options,
+                                           guard_selection_t *gs,
+                                           entry_guard_t *guard);
+static void pathbias_check_use_success_count(entry_guard_t *guard);
+static void pathbias_check_close_success_count(entry_guard_t *guard);
+static int node_is_possible_guard(const node_t *node);
+static int node_passes_guard_filter(const or_options_t *options,
+                                    const node_t *node);
+static entry_guard_t *entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport);
+static entry_guard_t *get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                              const tor_addr_port_t *addrport);
+static int entry_guard_obeys_restriction(const entry_guard_t *guard,
+                                         const entry_guard_restriction_t *rst);
+
+/** Return 0 if we should apply guardfraction information found in the
+ *  consensus. A specific consensus can be specified with the
+ *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
+int
+should_apply_guardfraction(const networkstatus_t *ns)
 {
-  char buf[HEX_DIGEST_LEN+1];
-  int changed = 0;
+  /* We need to check the corresponding torrc option and the consensus
+   * parameter if we need to. */
+  const or_options_t *options = get_options();
+
+  /* If UseGuardFraction is 'auto' then check the same-named consensus
+   * parameter. If the consensus parameter is not present, default to
+   * "off". */
+  if (options->UseGuardFraction == -1) {
+    return networkstatus_get_param(ns, "UseGuardFraction",
+                                   0, /* default to "off" */
+                                   0, 1);
+  }
 
-  *reason = NULL;
+  return options->UseGuardFraction;
+}
 
-  /* Do we want to mark this guard as bad? */
+/** Return true iff we know a descriptor for <b>guard</b> */
+static int
+guard_has_descriptor(const entry_guard_t *guard)
+{
+  const node_t *node = node_get_by_id(guard->identity);
   if (!node)
-    *reason = "unlisted";
-  else if (!node->is_running)
-    *reason = "down";
-  else if (options->UseBridges && (!node->ri ||
-                                   node->ri->purpose != ROUTER_PURPOSE_BRIDGE))
-    *reason = "not a bridge";
-  else if (options->UseBridges && !node_is_a_configured_bridge(node))
-    *reason = "not a configured bridge";
-  else if (!options->UseBridges && !node->is_possible_guard &&
-           !routerset_contains_node(options->EntryNodes,node))
-    *reason = "not recommended as a guard";
-  else if (routerset_contains_node(options->ExcludeNodes, node))
-    *reason = "excluded";
-  /* We only care about OR connection connectivity for entry guards. */
-  else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
-    *reason = "unreachable by config";
-  else if (e->path_bias_disabled)
-    *reason = "path-biased";
-
-  if (*reason && ! e->bad_since) {
-    /* Router is newly bad. */
-    base16_encode(buf, sizeof(buf), e->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard %s (%s) is %s: marking as unusable.",
-             e->nickname, buf, *reason);
-
-    e->bad_since = now;
-    control_event_guard(e->nickname, e->identity, "BAD");
-    changed = 1;
-  } else if (!*reason && e->bad_since) {
-    /* There's nothing wrong with the router any more. */
-    base16_encode(buf, sizeof(buf), e->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard %s (%s) is no longer unusable: "
-             "marking as ok.", e->nickname, buf);
-
-    e->bad_since = 0;
-    control_event_guard(e->nickname, e->identity, "GOOD");
-    changed = 1;
+    return 0;
+  return node_has_descriptor(node);
+}
+
+/**
+ * Try to determine the correct type for a selection named "name",
+ * if <b>type</b> is GS_TYPE_INFER.
+ */
+STATIC guard_selection_type_t
+guard_selection_infer_type(guard_selection_type_t type,
+                           const char *name)
+{
+  if (type == GS_TYPE_INFER) {
+    if (!strcmp(name, "bridges"))
+      type = GS_TYPE_BRIDGE;
+    else if (!strcmp(name, "restricted"))
+      type = GS_TYPE_RESTRICTED;
+    else
+      type = GS_TYPE_NORMAL;
   }
+  return type;
+}
 
-  if (node) {
-    int is_dir = node_is_dir(node);
-    if (options->UseBridges && node_is_a_configured_bridge(node))
-      is_dir = 1;
-    if (e->is_dir_cache != is_dir) {
-      e->is_dir_cache = is_dir;
-      changed = 1;
-    }
+/**
+ * Allocate and return a new guard_selection_t, with the name <b>name</b>.
+ */
+STATIC guard_selection_t *
+guard_selection_new(const char *name,
+                    guard_selection_type_t type)
+{
+  guard_selection_t *gs;
+
+  type = guard_selection_infer_type(type, name);
+
+  gs = tor_malloc_zero(sizeof(*gs));
+  gs->name = tor_strdup(name);
+  gs->type = type;
+  gs->sampled_entry_guards = smartlist_new();
+  gs->confirmed_entry_guards = smartlist_new();
+  gs->primary_entry_guards = smartlist_new();
+
+  return gs;
+}
+
+/**
+ * Return the guard selection called <b>name</b>. If there is none, and
+ * <b>create_if_absent</b> is true, then create and return it.  If there
+ * is none, and <b>create_if_absent</b> is false, then return NULL.
+ */
+STATIC guard_selection_t *
+get_guard_selection_by_name(const char *name,
+                            guard_selection_type_t type,
+                            int create_if_absent)
+{
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
   }
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    if (!strcmp(gs->name, name))
+      return gs;
+  } SMARTLIST_FOREACH_END(gs);
+
+  if (! create_if_absent)
+    return NULL;
+
+  log_debug(LD_GUARD, "Creating a guard selection called %s", name);
+  guard_selection_t *new_selection = guard_selection_new(name, type);
+  smartlist_add(guard_contexts, new_selection);
 
-  return changed;
+  return new_selection;
 }
 
-/** Return true iff enough time has passed since we last tried to connect
- * to the unreachable guard <b>e</b> that we're willing to try again. */
-STATIC int
-entry_is_time_to_retry(const entry_guard_t *e, time_t now)
+/**
+ * Allocate the first guard context that we're planning to use,
+ * and make it the current context.
+ */
+static void
+create_initial_guard_context(void)
 {
-  struct guard_retry_period_s {
-    time_t period_duration;
-    time_t interval_during_period;
-  };
+  tor_assert(! curr_guard_context);
+  if (!guard_contexts) {
+    guard_contexts = smartlist_new();
+  }
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(
+                             get_options(),
+                             networkstatus_get_live_consensus(approx_time()),
+                             NULL,
+                             &type);
+  tor_assert(name); // "name" can only be NULL if we had an old name.
+  tor_assert(type != GS_TYPE_INFER);
+  log_notice(LD_GUARD, "Starting with guard context \"%s\"", name);
+  curr_guard_context = get_guard_selection_by_name(name, type, 1);
+}
+
+/** Get current default guard_selection_t, creating it if necessary */
+guard_selection_t *
+get_guard_selection_info(void)
+{
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+  }
 
-  struct guard_retry_period_s periods[] = {
-     {    6*60*60,    60*60 }, /* For first 6 hrs., retry hourly; */
-     { 3*24*60*60,  4*60*60 }, /* Then retry every 4 hrs. until the
-                                  3-day mark; */
-     { 7*24*60*60, 18*60*60 }, /* After 3 days, retry every 18 hours until
-                                  1 week mark. */
-     {   TIME_MAX, 36*60*60 }  /* After 1 week, retry every 36 hours. */
-  };
+  return curr_guard_context;
+}
 
-  time_t ith_deadline_for_retry;
-  time_t unreachable_for;
-  unsigned i;
+/** Return a statically allocated human-readable description of <b>guard</b>
+ */
+const char *
+entry_guard_describe(const entry_guard_t *guard)
+{
+  static char buf[256];
+  tor_snprintf(buf, sizeof(buf),
+               "%s ($%s)",
+               strlen(guard->nickname) ? guard->nickname : "[bridge]",
+               hex_str(guard->identity, DIGEST_LEN));
+  return buf;
+}
 
-  if (e->last_attempted < e->unreachable_since)
-    return 1;
+/** Return <b>guard</b>'s 20-byte RSA identity digest */
+const char *
+entry_guard_get_rsa_id_digest(const entry_guard_t *guard)
+{
+  return guard->identity;
+}
 
-  unreachable_for = now - e->unreachable_since;
+/** Return the pathbias state associated with <b>guard</b>. */
+guard_pathbias_t *
+entry_guard_get_pathbias_state(entry_guard_t *guard)
+{
+  return &guard->pb;
+}
 
-  for (i = 0; i < ARRAY_LENGTH(periods); i++) {
-    if (unreachable_for <= periods[i].period_duration) {
-      ith_deadline_for_retry = e->last_attempted +
-                               periods[i].interval_during_period;
+HANDLE_IMPL(entry_guard, entry_guard_t, ATTR_UNUSED STATIC)
 
-      return (now > ith_deadline_for_retry);
-    }
-  }
-  return 0;
+/** Return an interval betweeen 'now' and 'max_backdate' seconds in the past,
+ * chosen uniformly at random.  We use this before recording persistent
+ * dates, so that we aren't leaking exactly when we recorded it.
+ */
+MOCK_IMPL(STATIC time_t,
+randomize_time,(time_t now, time_t max_backdate))
+{
+  tor_assert(max_backdate > 0);
+
+  time_t earliest = now - max_backdate;
+  time_t latest = now;
+  if (earliest <= 0)
+    earliest = 1;
+  if (latest <= earliest)
+    latest = earliest + 1;
+
+  return crypto_rand_time_range(earliest, latest);
 }
 
-/** Return the node corresponding to <b>e</b>, if <b>e</b> is
- * working well enough that we are willing to use it as an entry
- * right now. (Else return NULL.) In particular, it must be
- * - Listed as either up or never yet contacted;
- * - Present in the routerlist;
- * - Listed as 'stable' or 'fast' by the current dirserver consensus,
- *   if demanded by <b>need_uptime</b> or <b>need_capacity</b>
- *   (unless it's a configured EntryNode);
- * - Allowed by our current ReachableORAddresses config option; and
- * - Currently thought to be reachable by us (unless <b>assume_reachable</b>
- *   is true).
- *
- * If the answer is no, set *<b>msg</b> to an explanation of why.
+/**
+ * @name parameters for networkstatus algorithm
  *
- * If need_descriptor is true, only return the node if we currently have
- * a descriptor (routerinfo or microdesc) for it.
+ * These parameters are taken from the consensus; some are overrideable in
+ * the torrc.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this fraction
+ * of the guards on the network.
  */
-STATIC const node_t *
-entry_is_live(const entry_guard_t *e, entry_is_live_flags_t flags,
-              const char **msg)
+STATIC double
+get_max_sample_threshold(void)
 {
-  const node_t *node;
-  const or_options_t *options = get_options();
-  int need_uptime = (flags & ENTRY_NEED_UPTIME) != 0;
-  int need_capacity = (flags & ENTRY_NEED_CAPACITY) != 0;
-  const int assume_reachable = (flags & ENTRY_ASSUME_REACHABLE) != 0;
-  const int need_descriptor = (flags & ENTRY_NEED_DESCRIPTOR) != 0;
-
-  tor_assert(msg);
+  int32_t pct =
+    networkstatus_get_param(NULL, "guard-max-sample-threshold-percent",
+                            DFLT_MAX_SAMPLE_THRESHOLD_PERCENT,
+                            1, 100);
+  return pct / 100.0;
+}
+/**
+ * We never let our sampled guard set grow larger than this number.
+ */
+STATIC int
+get_max_sample_size_absolute(void)
+{
+  return (int) networkstatus_get_param(NULL, "guard-max-sample-size",
+                                       DFLT_MAX_SAMPLE_SIZE,
+                                       1, INT32_MAX);
+}
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+STATIC int
+get_min_filtered_sample_size(void)
+{
+  return networkstatus_get_param(NULL, "guard-min-filtered-sample-size",
+                                 DFLT_MIN_FILTERED_SAMPLE_SIZE,
+                                 1, INT32_MAX);
+}
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+STATIC int
+get_remove_unlisted_guards_after_days(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-remove-unlisted-guards-after-days",
+                                 DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS,
+                                 1, 365*10);
+}
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+STATIC int
+get_guard_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL,
+                                 "guard-lifetime-days",
+                                 DFLT_GUARD_LIFETIME_DAYS, 1, 365*10);
+  return days * 86400;
+}
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+STATIC int
+get_guard_confirmed_min_lifetime(void)
+{
+  if (get_options()->GuardLifetime >= 86400)
+    return get_options()->GuardLifetime;
+  int32_t days;
+  days = networkstatus_get_param(NULL, "guard-confirmed-min-lifetime-days",
+                                 DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS,
+                                 1, 365*10);
+  return days * 86400;
+}
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+STATIC int
+get_n_primary_guards(void)
+{
+  const int n = get_options()->NumEntryGuards;
+  const int n_dir = get_options()->NumDirectoryGuards;
+  if (n > 5) {
+    return MAX(n_dir, n + n / 2);
+  } else if (n >= 1) {
+    return MAX(n_dir, n * 2);
+  }
 
-  if (e->path_bias_disabled) {
-    *msg = "path-biased";
-    return NULL;
+  return networkstatus_get_param(NULL,
+                                 "guard-n-primary-guards",
+                                 DFLT_N_PRIMARY_GUARDS, 1, INT32_MAX);
+}
+/**
+ * Return the number of the live primary guards we should look at when
+ * making a circuit.
+ */
+STATIC int
+get_n_primary_guards_to_use(guard_usage_t usage)
+{
+  int configured;
+  const char *param_name;
+  int param_default;
+  if (usage == GUARD_USAGE_DIRGUARD) {
+    configured = get_options()->NumDirectoryGuards;
+    param_name = "guard-n-primary-dir-guards-to-use";
+    param_default = DFLT_N_PRIMARY_DIR_GUARDS_TO_USE;
+  } else {
+    configured = get_options()->NumEntryGuards;
+    param_name = "guard-n-primary-guards-to-use";
+    param_default = DFLT_N_PRIMARY_GUARDS_TO_USE;
   }
-  if (e->bad_since) {
-    *msg = "bad";
-    return NULL;
+  if (configured >= 1) {
+    return configured;
   }
-  /* no good if it's unreachable, unless assume_unreachable or can_retry. */
-  if (!assume_reachable && !e->can_retry &&
-      e->unreachable_since && !entry_is_time_to_retry(e, time(NULL))) {
-    *msg = "unreachable";
-    return NULL;
+  return networkstatus_get_param(NULL,
+                                 param_name, param_default, 1, INT32_MAX);
+}
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+STATIC int
+get_internet_likely_down_interval(void)
+{
+  return networkstatus_get_param(NULL, "guard-internet-likely-down-interval",
+                                 DFLT_INTERNET_LIKELY_DOWN_INTERVAL,
+                                 1, INT32_MAX);
+}
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+STATIC int
+get_nonprimary_guard_connect_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-connect-timeout",
+                                 DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+STATIC int
+get_nonprimary_guard_idle_timeout(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "guard-nonprimary-guard-idle-timeout",
+                                 DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT,
+                                 1, INT32_MAX);
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+STATIC double
+get_meaningful_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-meaningful-restriction-percent",
+                                        DFLT_MEANINGFUL_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+STATIC double
+get_extreme_restriction_threshold(void)
+{
+  int32_t pct = networkstatus_get_param(NULL,
+                                        "guard-extreme-restriction-percent",
+                                        DFLT_EXTREME_RESTRICTION_PERCENT,
+                                        1, INT32_MAX);
+  return pct / 100.0;
+}
+
+/* Mark <b>guard</b> as maybe reachable again. */
+static void
+mark_guard_maybe_reachable(entry_guard_t *guard)
+{
+  if (guard->is_reachable != GUARD_REACHABLE_NO) {
+    return;
   }
-  node = node_get_by_id(e->identity);
-  if (!node) {
-    *msg = "no node info";
-    return NULL;
+
+  /* Note that we do not clear failing_since: this guard is now only
+   * _maybe-reachable_. */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+}
+
+/**
+ * Called when the network comes up after having seemed to be down for
+ * a while: Mark the primary guards as maybe-reachable so that we'll
+ * try them again.
+ */
+STATIC void
+mark_primary_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/* Called when we exhaust all guards in our sampled set: Marks all guards as
+   maybe-reachable so that we 'll try them again. */
+static void
+mark_all_guards_maybe_reachable(guard_selection_t *gs)
+{
+  tor_assert(gs);
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    mark_guard_maybe_reachable(guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**@}*/
+
+/**
+ * Given our options and our list of nodes, return the name of the
+ * guard selection that we should use.  Return NULL for "use the
+ * same selection you were using before.
+ */
+STATIC const char *
+choose_guard_selection(const or_options_t *options,
+                       const networkstatus_t *live_ns,
+                       const guard_selection_t *old_selection,
+                       guard_selection_type_t *type_out)
+{
+  tor_assert(options);
+  tor_assert(type_out);
+
+  if (options->UseBridges) {
+    *type_out = GS_TYPE_BRIDGE;
+    return "bridges";
   }
-  if (need_descriptor && !node_has_descriptor(node)) {
-    *msg = "no descriptor";
-    return NULL;
+
+  if (! live_ns) {
+    /* without a networkstatus, we can't tell any more than that. */
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
   }
-  if (get_options()->UseBridges) {
-    if (node_get_purpose(node) != ROUTER_PURPOSE_BRIDGE) {
-      *msg = "not a bridge";
-      return NULL;
-    }
-    if (!node_is_a_configured_bridge(node)) {
-      *msg = "not a configured bridge";
-      return NULL;
-    }
-  } else { /* !get_options()->UseBridges */
-    if (node_get_purpose(node) != ROUTER_PURPOSE_GENERAL) {
-      *msg = "not general-purpose";
-      return NULL;
+
+  const smartlist_t *nodes = nodelist_get_list();
+  int n_guards = 0, n_passing_filter = 0;
+  SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+    if (node_is_possible_guard(node)) {
+      ++n_guards;
+      if (node_passes_guard_filter(options, node)) {
+        ++n_passing_filter;
+      }
     }
+  } SMARTLIST_FOREACH_END(node);
+
+  /* We use separate 'high' and 'low' thresholds here to prevent flapping
+   * back and forth */
+  const int meaningful_threshold_high =
+    (int)(n_guards * get_meaningful_restriction_threshold() * 1.05);
+  const int meaningful_threshold_mid =
+    (int)(n_guards * get_meaningful_restriction_threshold());
+  const int meaningful_threshold_low =
+    (int)(n_guards * get_meaningful_restriction_threshold() * .95);
+  const int extreme_threshold =
+    (int)(n_guards * get_extreme_restriction_threshold());
+
+  /*
+    If we have no previous selection, then we're "restricted" iff we are
+    below the meaningful restriction threshold.  That's easy enough.
+
+    But if we _do_ have a previous selection, we make it a little
+    "sticky": we only move from "restricted" to "default" when we find
+    that we're above the threshold plus 5%, and we only move from
+    "default" to "restricted" when we're below the threshold minus 5%.
+    That should prevent us from flapping back and forth if we happen to
+    be hovering very close to the default.
+
+    The extreme threshold is for warning only.
+  */
+
+  static int have_warned_extreme_threshold = 0;
+  if (n_guards &&
+      n_passing_filter < extreme_threshold &&
+      ! have_warned_extreme_threshold) {
+    have_warned_extreme_threshold = 1;
+    const double exclude_frac =
+      (n_guards - n_passing_filter) / (double)n_guards;
+    log_warn(LD_GUARD, "Your configuration excludes %d%% of all possible "
+             "guards. That's likely to make you stand out from the "
+             "rest of the world.", (int)(exclude_frac * 100));
   }
-  if (routerset_contains_node(options->EntryNodes, node)) {
-    /* they asked for it, they get it */
-    need_uptime = need_capacity = 0;
-  }
-  if (node_is_unreliable(node, need_uptime, need_capacity, 0)) {
-    *msg = "not fast/stable";
-    return NULL;
+
+  /* Easy case: no previous selection. Just check if we are in restricted or
+     normal guard selection. */
+  if (old_selection == NULL) {
+    if (n_passing_filter >= meaningful_threshold_mid) {
+      *type_out = GS_TYPE_NORMAL;
+      return "default";
+    } else {
+      *type_out = GS_TYPE_RESTRICTED;
+      return "restricted";
+    }
   }
-  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0)) {
-    *msg = "unreachable by config";
-    return NULL;
+
+  /* Trickier case: we do have a previous guard selection context. */
+  tor_assert(old_selection);
+
+  /* Use high and low thresholds to decide guard selection, and if we fall in
+     the middle then keep the current guard selection context. */
+  if (n_passing_filter >= meaningful_threshold_high) {
+    *type_out = GS_TYPE_NORMAL;
+    return "default";
+  } else if (n_passing_filter < meaningful_threshold_low) {
+    *type_out = GS_TYPE_RESTRICTED;
+    return "restricted";
+  } else {
+    /* we are in the middle: maintain previous guard selection */
+    *type_out = old_selection->type;
+    return old_selection->name;
   }
-  return node;
 }
 
-/** Return the number of entry guards that we think are usable. */
+/**
+ * Check whether we should switch from our current guard selection to a
+ * different one.  If so, switch and return 1.  Return 0 otherwise.
+ *
+ * On a 1 return, the caller should mark all currently live circuits unusable
+ * for new streams, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
 int
-num_live_entry_guards(int for_directory)
+update_guard_selection_choice(const or_options_t *options)
 {
-  int n = 0;
-  const char *msg;
-  /* Set the entry node attributes we are interested in. */
-  entry_is_live_flags_t entry_flags = ENTRY_NEED_CAPACITY;
-  if (!for_directory) {
-    entry_flags |= ENTRY_NEED_DESCRIPTOR;
+  if (!curr_guard_context) {
+    create_initial_guard_context();
+    return 1;
   }
 
-  if (! entry_guards)
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-      if (for_directory && !entry->is_dir_cache)
-        continue;
-      if (entry_is_live(entry, entry_flags, &msg))
-        ++n;
-  } SMARTLIST_FOREACH_END(entry);
-  return n;
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *new_name = choose_guard_selection(
+                             options,
+                             networkstatus_get_live_consensus(approx_time()),
+                             curr_guard_context,
+                             &type);
+  tor_assert(new_name);
+  tor_assert(type != GS_TYPE_INFER);
+
+  const char *cur_name = curr_guard_context->name;
+  if (! strcmp(cur_name, new_name)) {
+    log_debug(LD_GUARD,
+              "Staying with guard context \"%s\" (no change)", new_name);
+    return 0; // No change
+  }
+
+  log_notice(LD_GUARD, "Switching to guard context \"%s\" (was using \"%s\")",
+             new_name, cur_name);
+  guard_selection_t *new_guard_context;
+  new_guard_context = get_guard_selection_by_name(new_name, type, 1);
+  tor_assert(new_guard_context);
+  tor_assert(new_guard_context != curr_guard_context);
+  curr_guard_context = new_guard_context;
+
+  return 1;
 }
 
-/** If <b>digest</b> matches the identity of any node in the
- * entry_guards list, return that node. Else return NULL. */
-entry_guard_t *
-entry_guard_get_by_id_digest(const char *digest)
+/**
+ * Return true iff <b>node</b> has all the flags needed for us to consider it
+ * a possible guard when sampling guards.
+ */
+static int
+node_is_possible_guard(const node_t *node)
 {
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry,
-                    if (tor_memeq(digest, entry->identity, DIGEST_LEN))
-                      return entry;
-                   );
+  /* The "GUARDS" set is all nodes in the nodelist for which this predicate
+   * holds. */
+
+  tor_assert(node);
+  return (node->is_possible_guard &&
+          node->is_stable &&
+          node->is_fast &&
+          node->is_valid &&
+          node_is_dir(node) &&
+          !router_digest_is_me(node->identity));
+}
+
+/**
+ * Return the sampled guard with the RSA identity digest <b>rsa_id</b>, or
+ * NULL if we don't have one. */
+STATIC entry_guard_t *
+get_sampled_guard_with_id(guard_selection_t *gs,
+                          const uint8_t *rsa_id)
+{
+  tor_assert(gs);
+  tor_assert(rsa_id);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (tor_memeq(guard->identity, rsa_id, DIGEST_LEN))
+      return guard;
+  } SMARTLIST_FOREACH_END(guard);
   return NULL;
 }
 
-/** Dump a description of our list of entry guards to the log at level
- * <b>severity</b>. */
-static void
-log_entry_guards(int severity)
-{
-  smartlist_t *elements = smartlist_new();
-  char *s;
-
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e)
-    {
-      const char *msg = NULL;
-      if (entry_is_live(e, ENTRY_NEED_CAPACITY, &msg))
-        smartlist_add_asprintf(elements, "%s [%s] (up %s)",
-                     e->nickname,
-                     hex_str(e->identity, DIGEST_LEN),
-                     e->made_contact ? "made-contact" : "never-contacted");
-      else
-        smartlist_add_asprintf(elements, "%s [%s] (%s, %s)",
-                     e->nickname,
-                     hex_str(e->identity, DIGEST_LEN),
-                     msg,
-                     e->made_contact ? "made-contact" : "never-contacted");
-    }
-  SMARTLIST_FOREACH_END(e);
+/** If <b>gs</b> contains a sampled entry guard matching <b>bridge</b>,
+ * return that guard. Otherwise return NULL. */
+static entry_guard_t *
+get_sampled_guard_for_bridge(guard_selection_t *gs,
+                             const bridge_info_t *bridge)
+{
+  const uint8_t *id = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+  entry_guard_t *guard;
+  if (BUG(!addrport))
+    return NULL; // LCOV_EXCL_LINE
+  guard = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (! guard || (id && tor_memneq(id, guard->identity, DIGEST_LEN)))
+    return NULL;
+  else
+    return guard;
+}
 
-  s = smartlist_join_strings(elements, ",", 0, NULL);
-  SMARTLIST_FOREACH(elements, char*, cp, tor_free(cp));
-  smartlist_free(elements);
-  log_fn(severity,LD_CIRC,"%s",s);
-  tor_free(s);
+/** If we know a bridge_info_t matching <b>guard</b>, return that
+ * bridge.  Otherwise return NULL. */
+static bridge_info_t *
+get_bridge_info_for_guard(const entry_guard_t *guard)
+{
+  const uint8_t *identity = NULL;
+  if (! tor_digest_is_zero(guard->identity)) {
+    identity = (const uint8_t *)guard->identity;
+  }
+  if (BUG(guard->bridge_addr == NULL))
+    return NULL;
+
+  return get_configured_bridge_by_exact_addr_port_digest(
+                                                 &guard->bridge_addr->addr,
+                                                 guard->bridge_addr->port,
+                                                 (const char*)identity);
 }
 
-/** Called when one or more guards that we would previously have used for some
- * purpose are no longer in use because a higher-priority guard has become
- * usable again. */
-static void
-control_event_guard_deferred(void)
-{
-  /* XXXX We don't actually have a good way to figure out _how many_ entries
-   * are live for some purpose.  We need an entry_is_even_slightly_live()
-   * function for this to work right.  NumEntryGuards isn't reliable: if we
-   * need guards with weird properties, we can have more than that number
-   * live.
-   **/
-#if 0
-  int n = 0;
-  const char *msg;
-  const or_options_t *options = get_options();
-  if (!entry_guards)
-    return;
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, entry,
-    {
-      if (entry_is_live(entry, 0, 1, 0, &msg)) {
-        if (n++ == options->NumEntryGuards) {
-          control_event_guard(entry->nickname, entry->identity, "DEFERRED");
-          return;
-        }
-      }
-    });
-#endif
+/**
+ * Return true iff we have a sampled guard with the RSA identity digest
+ * <b>rsa_id</b>. */
+static inline int
+have_sampled_guard_with_id(guard_selection_t *gs, const uint8_t *rsa_id)
+{
+  return get_sampled_guard_with_id(gs, rsa_id) != NULL;
 }
 
-/** Largest amount that we'll backdate chosen_on_date */
-#define CHOSEN_ON_DATE_SLOP (30*86400)
+/**
+ * Allocate a new entry_guard_t object for <b>node</b>, add it to the
+ * sampled entry guards in <b>gs</b>, and return it. <b>node</b> must
+ * not currently be a sampled guard in <b>gs</b>.
+ */
+STATIC entry_guard_t *
+entry_guard_add_to_sample(guard_selection_t *gs,
+                          const node_t *node)
+{
+  log_info(LD_GUARD, "Adding %s as to the entry guard sample set.",
+           node_describe(node));
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(have_sampled_guard_with_id(gs, (const uint8_t*)node->identity)))
+    return NULL; // LCOV_EXCL_LINE
 
-/** Add a new (preferably stable and fast) router to our
- * entry_guards list. Return a pointer to the router if we succeed,
- * or NULL if we can't find any more suitable entries.
- *
- * If <b>chosen</b> is defined, use that one, and if it's not
- * already in our entry_guards list, put it at the *beginning*.
- * Else, put the one we pick at the end of the list. */
-STATIC const node_t *
-add_an_entry_guard(const node_t *chosen, int reset_status, int prepend,
-                   int for_discovery, int for_directory)
+  return entry_guard_add_to_sample_impl(gs,
+                                        (const uint8_t*)node->identity,
+                                        node_get_nickname(node),
+                                        NULL);
+}
+
+/**
+ * Backend: adds a new sampled guard to <b>gs</b>, with given identity,
+ * nickname, and ORPort.  rsa_id_digest and bridge_addrport are optional, but
+ * we need one of them. nickname is optional. The caller is responsible for
+ * maintaining the size limit of the SAMPLED_GUARDS set.
+ */
+static entry_guard_t *
+entry_guard_add_to_sample_impl(guard_selection_t *gs,
+                               const uint8_t *rsa_id_digest,
+                               const char *nickname,
+                               const tor_addr_port_t *bridge_addrport)
+{
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  tor_assert(gs);
+
+  // XXXX #20827 take ed25519 identity here too.
+
+  /* Make sure we can actually identify the guard. */
+  if (BUG(!rsa_id_digest && !bridge_addrport))
+    return NULL; // LCOV_EXCL_LINE
+
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+
+  /* persistent fields */
+  guard->is_persistent = (rsa_id_digest != NULL);
+  guard->selection_name = tor_strdup(gs->name);
+  if (rsa_id_digest)
+    memcpy(guard->identity, rsa_id_digest, DIGEST_LEN);
+  if (nickname)
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  guard->sampled_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+  tor_free(guard->sampled_by_version);
+  guard->sampled_by_version = tor_strdup(VERSION);
+  guard->currently_listed = 1;
+  guard->confirmed_idx = -1;
+
+  /* non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+  if (bridge_addrport)
+    guard->bridge_addr = tor_memdup(bridge_addrport, sizeof(*bridge_addrport));
+
+  smartlist_add(gs->sampled_entry_guards, guard);
+  guard->in_selection = gs;
+  entry_guard_set_filtered_flags(get_options(), gs, guard);
+  entry_guards_changed_for_guard_selection(gs);
+  return guard;
+}
+
+/**
+ * Add an entry guard to the "bridges" guard selection sample, with
+ * information taken from <b>bridge</b>. Return that entry guard.
+ */
+static entry_guard_t *
+entry_guard_add_bridge_to_sample(guard_selection_t *gs,
+                                 const bridge_info_t *bridge)
 {
-  const node_t *node;
-  entry_guard_t *entry;
-
-  if (chosen) {
-    node = chosen;
-    entry = entry_guard_get_by_id_digest(node->identity);
-    if (entry) {
-      if (reset_status) {
-        entry->bad_since = 0;
-        entry->can_retry = 1;
-      }
-      entry->is_dir_cache = node_is_dir(node);
-      if (get_options()->UseBridges && node_is_a_configured_bridge(node))
-        entry->is_dir_cache = 1;
+  const uint8_t *id_digest = bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
 
-      return NULL;
-    }
-  } else if (!for_directory) {
-    node = choose_good_entry_server(CIRCUIT_PURPOSE_C_GENERAL, NULL);
-    if (!node)
-      return NULL;
-  } else {
-    const routerstatus_t *rs;
-    rs = router_pick_directory_server(MICRODESC_DIRINFO|V3_DIRINFO,
-                                      PDS_FOR_GUARD);
-    if (!rs)
-      return NULL;
-    node = node_get_by_id(rs->identity_digest);
-    if (!node)
-      return NULL;
-  }
-  if (node->using_as_guard)
+  tor_assert(addrport);
+
+  /* make sure that the guard is not already sampled. */
+  if (BUG(get_sampled_guard_for_bridge(gs, bridge)))
+    return NULL; // LCOV_EXCL_LINE
+
+  return entry_guard_add_to_sample_impl(gs, id_digest, NULL, addrport);
+}
+
+/**
+ * Return the entry_guard_t in <b>gs</b> whose address is <b>addrport</b>,
+ * or NULL if none exists.
+*/
+static entry_guard_t *
+get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
+                                 const tor_addr_port_t *addrport)
+{
+  if (! gs)
     return NULL;
-  if (entry_guard_get_by_id_digest(node->identity) != NULL) {
-    log_info(LD_CIRC, "I was about to add a duplicate entry guard.");
-    /* This can happen if we choose a guard, then the node goes away, then
-     * comes back. */
-    ((node_t*) node)->using_as_guard = 1;
+  if (BUG(!addrport))
     return NULL;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    if (g->bridge_addr && tor_addr_port_eq(addrport, g->bridge_addr))
+      return g;
+  } SMARTLIST_FOREACH_END(g);
+  return NULL;
+}
+
+/** Update the guard subsystem's knowledge of the identity of the bridge
+ * at <b>addrport</b>.  Idempotent.
+ */
+void
+entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                    const uint8_t *rsa_id_digest)
+{
+  guard_selection_t *gs = get_guard_selection_by_name("bridges",
+                                                      GS_TYPE_BRIDGE,
+                                                      0);
+  if (!gs)
+    return;
+
+  entry_guard_t *g = get_sampled_guard_by_bridge_addr(gs, addrport);
+  if (!g)
+    return;
+
+  int make_persistent = 0;
+
+  if (tor_digest_is_zero(g->identity)) {
+    memcpy(g->identity, rsa_id_digest, DIGEST_LEN);
+    make_persistent = 1;
+  } else if (tor_memeq(g->identity, rsa_id_digest, DIGEST_LEN)) {
+    /* Nothing to see here; we learned something we already knew. */
+    if (BUG(! g->is_persistent))
+      make_persistent = 1;
+  } else {
+    char old_id[HEX_DIGEST_LEN+1];
+    base16_encode(old_id, sizeof(old_id), g->identity, sizeof(g->identity));
+    log_warn(LD_BUG, "We 'learned' an identity %s for a bridge at %s:%d, but "
+             "we already knew a different one (%s). Ignoring the new info as "
+             "possibly bogus.",
+             hex_str((const char *)rsa_id_digest, DIGEST_LEN),
+             fmt_and_decorate_addr(&addrport->addr), addrport->port,
+             old_id);
+    return; // redundant, but let's be clear: we're not making this persistent.
   }
-  entry = tor_malloc_zero(sizeof(entry_guard_t));
-  log_info(LD_CIRC, "Chose %s as new entry guard.",
-           node_describe(node));
-  strlcpy(entry->nickname, node_get_nickname(node), sizeof(entry->nickname));
-  memcpy(entry->identity, node->identity, DIGEST_LEN);
-  entry->is_dir_cache = node_is_dir(node);
-  if (get_options()->UseBridges && node_is_a_configured_bridge(node))
-    entry->is_dir_cache = 1;
-
-  /* Choose expiry time smudged over the past month. The goal here
-   * is to a) spread out when Tor clients rotate their guards, so they
-   * don't all select them on the same day, and b) avoid leaving a
-   * precise timestamp in the state file about when we first picked
-   * this guard. For details, see the Jan 2010 or-dev thread. */
-  time_t now = time(NULL);
-  entry->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now);
-  entry->chosen_by_version = tor_strdup(VERSION);
-
-  /* Are we picking this guard because all of our current guards are
-   * down so we need another one (for_discovery is 1), or because we
-   * decided we need more variety in our guard list (for_discovery is 0)?
-   *
-   * Currently we hack this behavior into place by setting "made_contact"
-   * for guards of the latter variety, so we'll be willing to use any of
-   * them right off the bat.
-   */
-  if (!for_discovery)
-    entry->made_contact = 1;
 
-  ((node_t*)node)->using_as_guard = 1;
-  if (prepend)
-    smartlist_insert(entry_guards, 0, entry);
-  else
-    smartlist_add(entry_guards, entry);
-  control_event_guard(entry->nickname, entry->identity, "NEW");
-  control_event_guard_deferred();
-  log_entry_guards(LOG_INFO);
-  return node;
+  if (make_persistent) {
+    g->is_persistent = 1;
+    entry_guards_changed_for_guard_selection(gs);
+  }
 }
 
-/** Choose how many entry guards or directory guards we'll use. If
- * <b>for_directory</b> is true, we return how many directory guards to
- * use; else we return how many entry guards to use. */
+/**
+ * Return the number of sampled guards in <b>gs</b> that are "filtered"
+ * (that is, we're willing to connect to them) and that are "usable"
+ * (that is, either "reachable" or "maybe reachable").
+ *
+ * If a restriction is provided in <b>rst</b>, do not count any guards that
+ * violate it.
+ */
 STATIC int
-decide_num_guards(const or_options_t *options, int for_directory)
+num_reachable_filtered_guards(const guard_selection_t *gs,
+                              const entry_guard_restriction_t *rst)
 {
-  if (for_directory) {
-    int answer;
-    if (options->NumDirectoryGuards != 0)
-      return options->NumDirectoryGuards;
-    answer = networkstatus_get_param(NULL, "NumDirectoryGuards", 0, 0, 10);
-    if (answer) /* non-zero means use the consensus value */
-      return answer;
-  }
-
-  if (options->NumEntryGuards)
-    return options->NumEntryGuards;
+  int n_reachable_filtered_guards = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_usable_filtered_guard)
+      ++n_reachable_filtered_guards;
+  } SMARTLIST_FOREACH_END(guard);
+  return n_reachable_filtered_guards;
+}
 
-  /* Use the value from the consensus, or 3 if no guidance. */
-  return networkstatus_get_param(NULL, "NumEntryGuards", DEFAULT_N_GUARDS,
-                                 MIN_N_GUARDS, MAX_N_GUARDS);
+/** Return the actual maximum size for the sample in <b>gs</b>,
+ * given that we know about <b>n_guards</b> total. */
+static int
+get_max_sample_size(guard_selection_t *gs,
+                    int n_guards)
+{
+  const int using_bridges = (gs->type == GS_TYPE_BRIDGE);
+  const int min_sample = get_min_filtered_sample_size();
+
+  /* If we are in bridge mode, expand our sample set as needed without worrying
+   * about max size. We should respect the user's wishes to use many bridges if
+   * that's what they have specified in their configuration file. */
+  if (using_bridges)
+    return INT_MAX;
+
+  const int max_sample_by_pct = (int)(n_guards * get_max_sample_threshold());
+  const int max_sample_absolute = get_max_sample_size_absolute();
+  const int max_sample = MIN(max_sample_by_pct, max_sample_absolute);
+  if (max_sample < min_sample)
+    return min_sample;
+  else
+    return max_sample;
 }
 
-/** If the use of entry guards is configured, choose more entry guards
- * until we have enough in the list. */
-static void
-pick_entry_guards(const or_options_t *options, int for_directory)
-{
-  int changed = 0;
-  const int num_needed = decide_num_guards(options, for_directory);
+/**
+ * Return a smartlist of the all the guards that are not currently
+ * members of the sample (GUARDS - SAMPLED_GUARDS).  The elements of
+ * this list are node_t pointers in the non-bridge case, and
+ * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out/b>
+ * to the number of guards that we found in GUARDS, including those
+ * that were already sampled.
+ */
+static smartlist_t *
+get_eligible_guards(const or_options_t *options,
+                    guard_selection_t *gs,
+                    int *n_guards_out)
+{
+  /* Construct eligible_guards as GUARDS - SAMPLED_GUARDS */
+  smartlist_t *eligible_guards = smartlist_new();
+  int n_guards = 0; // total size of "GUARDS"
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const smartlist_t *bridges = bridge_list_get();
+    SMARTLIST_FOREACH_BEGIN(bridges, bridge_info_t *, bridge) {
+      ++n_guards;
+      if (NULL != get_sampled_guard_for_bridge(gs, bridge)) {
+        continue;
+      }
+      smartlist_add(eligible_guards, bridge);
+    } SMARTLIST_FOREACH_END(bridge);
+  } else {
+    const smartlist_t *nodes = nodelist_get_list();
+    const int n_sampled = smartlist_len(gs->sampled_entry_guards);
+
+    /* Build a bloom filter of our current guards: let's keep this O(N). */
+    digestset_t *sampled_guard_ids = digestset_new(n_sampled);
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, const entry_guard_t *,
+                            guard) {
+      digestset_add(sampled_guard_ids, guard->identity);
+    } SMARTLIST_FOREACH_END(guard);
+
+    SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
+      if (! node_is_possible_guard(node))
+        continue;
+      if (gs->type == GS_TYPE_RESTRICTED) {
+        /* In restricted mode, we apply the filter BEFORE sampling, so
+         * that we are sampling from the nodes that we might actually
+         * select. If we sampled first, we might wind up with a sample
+         * that didn't include any EntryNodes at all. */
+        if (! node_passes_guard_filter(options, node))
+          continue;
+      }
+      ++n_guards;
+      if (digestset_contains(sampled_guard_ids, node->identity))
+        continue;
+      smartlist_add(eligible_guards, (node_t*)node);
+    } SMARTLIST_FOREACH_END(node);
 
-  tor_assert(entry_guards);
+    /* Now we can free that bloom filter. */
+    digestset_free(sampled_guard_ids);
+  }
 
-  while (num_live_entry_guards(for_directory) < num_needed) {
-    if (!add_an_entry_guard(NULL, 0, 0, 0, for_directory))
-      break;
-    changed = 1;
+  *n_guards_out = n_guards;
+  return eligible_guards;
+}
+
+/** Helper: given a smartlist of either bridge_info_t (if gs->type is
+ * GS_TYPE_BRIDGE) or node_t (otherwise), pick one that can be a guard,
+ * add it as a guard, remove it from the list, and return a new
+ * entry_guard_t.  Return NULL on failure. */
+static entry_guard_t *
+select_and_add_guard_item_for_sample(guard_selection_t *gs,
+                                     smartlist_t *eligible_guards)
+{
+  entry_guard_t *added_guard;
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = smartlist_choose(eligible_guards);
+    if (BUG(!bridge))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, bridge);
+    added_guard = entry_guard_add_bridge_to_sample(gs, bridge);
+  } else {
+    const node_t *node =
+      node_sl_choose_by_bandwidth(eligible_guards, WEIGHT_FOR_GUARD);
+    if (BUG(!node))
+      return NULL; // LCOV_EXCL_LINE
+    smartlist_remove(eligible_guards, node);
+    added_guard = entry_guard_add_to_sample(gs, node);
   }
-  if (changed)
-    entry_guards_changed();
-}
 
-/** How long (in seconds) do we allow an entry guard to be nonfunctional,
- * unlisted, excluded, or otherwise nonusable before we give up on it? */
-#define ENTRY_GUARD_REMOVE_AFTER (30*24*60*60)
+  return added_guard;
+}
 
-/** Release all storage held by <b>e</b>. */
-static void
-entry_guard_free(entry_guard_t *e)
+/** Return true iff we need a consensus to maintain our  */
+static int
+live_consensus_is_missing(const guard_selection_t *gs)
 {
-  if (!e)
-    return;
-  tor_free(e->chosen_by_version);
-  tor_free(e);
+  tor_assert(gs);
+  if (gs->type == GS_TYPE_BRIDGE) {
+    /* We don't update bridges from the consensus; they aren't there. */
+    return 0;
+  }
+  return networkstatus_get_live_consensus(approx_time()) == NULL;
 }
 
 /**
- * Return the minimum lifetime of working entry guard, in seconds,
- * as given in the consensus networkstatus.  (Plus CHOSEN_ON_DATE_SLOP,
- * so that we can do the chosen_on_date randomization while achieving the
- * desired minimum lifetime.)
+ * Add new guards to the sampled guards in <b>gs</b> until there are
+ * enough usable filtered guards, but never grow the sample beyond its
+ * maximum size.  Return the last guard added, or NULL if none were
+ * added.
  */
-static int32_t
-guards_get_lifetime(void)
+STATIC entry_guard_t *
+entry_guards_expand_sample(guard_selection_t *gs)
 {
+  tor_assert(gs);
   const or_options_t *options = get_options();
-#define DFLT_GUARD_LIFETIME (86400 * 60)   /* Two months. */
-#define MIN_GUARD_LIFETIME  (86400 * 30)   /* One months. */
-#define MAX_GUARD_LIFETIME  (86400 * 1826) /* Five years. */
 
-  if (options->GuardLifetime >= 1) {
-    return CLAMP(MIN_GUARD_LIFETIME,
-                 options->GuardLifetime,
-                 MAX_GUARD_LIFETIME) + CHOSEN_ON_DATE_SLOP;
+  if (live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not expanding the sample guard set; we have "
+             "no live consensus.");
+    return NULL;
   }
 
-  return networkstatus_get_param(NULL, "GuardLifetime",
-                                 DFLT_GUARD_LIFETIME,
-                                 MIN_GUARD_LIFETIME,
-                                 MAX_GUARD_LIFETIME) + CHOSEN_ON_DATE_SLOP;
-}
-
-/** Remove any entry guard which was selected by an unknown version of Tor,
- * or which was selected by a version of Tor that's known to select
- * entry guards badly, or which was selected more 2 months ago. */
-/* XXXX The "obsolete guards" and "chosen long ago guards" things should
- * probably be different functions. */
-static int
-remove_obsolete_entry_guards(time_t now)
-{
-  int changed = 0, i;
-  int32_t guard_lifetime = guards_get_lifetime();
-
-  for (i = 0; i < smartlist_len(entry_guards); ++i) {
-    entry_guard_t *entry = smartlist_get(entry_guards, i);
-    const char *ver = entry->chosen_by_version;
-    const char *msg = NULL;
-    tor_version_t v;
-    int version_is_bad = 0, date_is_bad = 0;
-    if (!ver) {
-      msg = "does not say what version of Tor it was selected by";
-      version_is_bad = 1;
-    } else if (tor_version_parse(ver, &v)) {
-      msg = "does not seem to be from any recognized version of Tor";
-      version_is_bad = 1;
-    }
-    if (!version_is_bad && entry->chosen_on_date + guard_lifetime < now) {
-      /* It's been too long since the date listed in our state file. */
-      msg = "was selected several months ago";
-      date_is_bad = 1;
+  int n_sampled = smartlist_len(gs->sampled_entry_guards);
+  entry_guard_t *added_guard = NULL;
+  int n_usable_filtered_guards = num_reachable_filtered_guards(gs, NULL);
+  int n_guards = 0;
+  smartlist_t *eligible_guards = get_eligible_guards(options, gs, &n_guards);
+
+  const int max_sample = get_max_sample_size(gs, n_guards);
+  const int min_filtered_sample = get_min_filtered_sample_size();
+
+  log_info(LD_GUARD, "Expanding the sample guard set. We have %d guards "
+           "in the sample, and %d eligible guards to extend it with.",
+           n_sampled, smartlist_len(eligible_guards));
+
+  while (n_usable_filtered_guards < min_filtered_sample) {
+    /* Has our sample grown too large to expand? */
+    if (n_sampled >= max_sample) {
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just hit the maximum sample threshold of %d",
+               max_sample);
+      goto done;
     }
 
-    if (version_is_bad || date_is_bad) { /* we need to drop it */
-      char dbuf[HEX_DIGEST_LEN+1];
-      tor_assert(msg);
-      base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-      log_fn(version_is_bad ? LOG_NOTICE : LOG_INFO, LD_CIRC,
-             "Entry guard '%s' (%s) %s. (Version=%s.) Replacing it.",
-             entry->nickname, dbuf, msg, ver?escaped(ver):"none");
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, i--);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
+    /* Did we run out of guards? */
+    if (smartlist_len(eligible_guards) == 0) {
+      /* LCOV_EXCL_START
+         As long as MAX_SAMPLE_THRESHOLD makes can't be adjusted to
+         allow all guards to be sampled, this can't be reached.
+       */
+      log_info(LD_GUARD, "Not expanding the guard sample any further; "
+               "just ran out of eligible guards");
+      goto done;
+      /* LCOV_EXCL_STOP */
     }
+
+    /* Otherwise we can add at least one new guard. */
+    added_guard = select_and_add_guard_item_for_sample(gs, eligible_guards);
+    if (!added_guard)
+      goto done; // LCOV_EXCL_LINE -- only fails on BUG.
+
+    ++n_sampled;
+
+    if (added_guard->is_usable_filtered_guard)
+      ++n_usable_filtered_guards;
   }
 
-  return changed ? 1 : 0;
+ done:
+  smartlist_free(eligible_guards);
+  return added_guard;
 }
 
-/** Remove all entry guards that have been down or unlisted for so
- * long that we don't think they'll come up again. Return 1 if we
- * removed any, or 0 if we did nothing. */
-static int
-remove_dead_entry_guards(time_t now)
-{
-  char dbuf[HEX_DIGEST_LEN+1];
-  char tbuf[ISO_TIME_LEN+1];
-  int i;
-  int changed = 0;
-
-  for (i = 0; i < smartlist_len(entry_guards); ) {
-    entry_guard_t *entry = smartlist_get(entry_guards, i);
-    if (entry->bad_since &&
-        ! entry->path_bias_disabled &&
-        entry->bad_since + ENTRY_GUARD_REMOVE_AFTER < now) {
-
-      base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-      format_local_iso_time(tbuf, entry->bad_since);
-      log_info(LD_CIRC, "Entry guard '%s' (%s) has been down or unlisted "
-               "since %s local time; removing.",
-               entry->nickname, dbuf, tbuf);
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, i);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    } else
-      ++i;
-  }
-  return changed ? 1 : 0;
-}
-
-/** Remove all currently listed entry guards. So new ones will be chosen. */
-void
-remove_all_entry_guards(void)
+/**
+ * Helper: <b>guard</b> has just been removed from the sampled guards:
+ * also remove it from primary and confirmed. */
+static void
+remove_guard_from_confirmed_and_primary_lists(guard_selection_t *gs,
+                                              entry_guard_t *guard)
 {
-  char dbuf[HEX_DIGEST_LEN+1];
+  if (guard->is_primary) {
+    guard->is_primary = 0;
+    smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+  } else {
+    if (BUG(smartlist_contains(gs->primary_entry_guards, guard))) {
+      smartlist_remove_keeporder(gs->primary_entry_guards, guard);
+    }
+  }
 
-  while (smartlist_len(entry_guards)) {
-    entry_guard_t *entry = smartlist_get(entry_guards, 0);
-    base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN);
-    log_info(LD_CIRC, "Entry guard '%s' (%s) has been dropped.",
-             entry->nickname, dbuf);
-    control_event_guard(entry->nickname, entry->identity, "DROPPED");
-    entry_guard_free(entry);
-    smartlist_del(entry_guards, 0);
+  if (guard->confirmed_idx >= 0) {
+    smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+    guard->confirmed_idx = -1;
+    guard->confirmed_on_date = 0;
+  } else {
+    if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard))) {
+      // LCOV_EXCL_START
+      smartlist_remove_keeporder(gs->confirmed_entry_guards, guard);
+      // LCOV_EXCL_STOP
+    }
   }
-  log_entry_guards(LOG_INFO);
-  entry_guards_changed();
 }
 
-/** A new directory or router-status has arrived; update the down/listed
- * status of the entry guards.
- *
- * An entry is 'down' if the directory lists it as nonrunning.
- * An entry is 'unlisted' if the directory doesn't include it.
- *
- * Don't call this on startup; only on a fresh download. Otherwise we'll
- * think that things are unlisted.
- */
-void
-entry_guards_compute_status(const or_options_t *options, time_t now)
+/** Return true iff <b>guard</b> is currently "listed" -- that is, it
+ * appears in the consensus, or as a configured bridge (as
+ * appropriate) */
+MOCK_IMPL(STATIC int,
+entry_guard_is_listed,(guard_selection_t *gs, const entry_guard_t *guard))
 {
-  int changed = 0;
-  digestmap_t *reasons;
+  if (gs->type == GS_TYPE_BRIDGE) {
+    return NULL != get_bridge_info_for_guard(guard);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+
+    return node && node_is_possible_guard(node);
+  }
+}
 
-  if (! entry_guards)
+/**
+ * Update the status of all sampled guards based on the arrival of a
+ * new consensus networkstatus document.  This will include marking
+ * some guards as listed or unlisted, and removing expired guards. */
+STATIC void
+sampled_guards_update_from_consensus(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  const int REMOVE_UNLISTED_GUARDS_AFTER =
+    (get_remove_unlisted_guards_after_days() * 86400);
+  const int unlisted_since_slop = REMOVE_UNLISTED_GUARDS_AFTER / 5;
+
+  // It's important to use only a live consensus here; we don't want to
+  // make changes based on anything expired or old.
+  if (live_consensus_is_missing(gs)) {
+    log_info(LD_GUARD, "Not updating the sample guard set; we have "
+             "no live consensus.");
     return;
+  }
+  log_info(LD_GUARD, "Updating sampled guard status based on received "
+           "consensus.");
+
+  int n_changes = 0;
+
+  /* First: Update listed/unlisted. */
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    /* XXXX #20827 check ed ID too */
+    const int is_listed = entry_guard_is_listed(gs, guard);
+
+    if (is_listed && ! guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 1;
+      guard->unlisted_since_date = 0;
+      log_info(LD_GUARD, "Sampled guard %s is now listed again.",
+               entry_guard_describe(guard));
+    } else if (!is_listed && guard->currently_listed) {
+      ++n_changes;
+      guard->currently_listed = 0;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_info(LD_GUARD, "Sampled guard %s is now unlisted.",
+               entry_guard_describe(guard));
+    } else if (is_listed && guard->currently_listed) {
+      log_debug(LD_GUARD, "Sampled guard %s is still listed.",
+               entry_guard_describe(guard));
+    } else {
+      tor_assert(! is_listed && ! guard->currently_listed);
+      log_debug(LD_GUARD, "Sampled guard %s is still unlisted.",
+                entry_guard_describe(guard));
+    }
 
-  if (options->EntryNodes) /* reshuffle the entry guard list if needed */
-    entry_nodes_should_be_added();
-
-  reasons = digestmap_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry)
-    {
-      const node_t *r = node_get_by_id(entry->identity);
-      const char *reason = NULL;
-      if (entry_guard_set_status(entry, r, now, options, &reason))
-        changed = 1;
-
-      if (entry->bad_since)
-        tor_assert(reason);
-      if (reason)
-        digestmap_set(reasons, entry->identity, (char*)reason);
+    /* Clean up unlisted_since_date, just in case. */
+    if (guard->currently_listed && guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = 0;
+      log_warn(LD_BUG, "Sampled guard %s was listed, but with "
+               "unlisted_since_date set. Fixing.",
+               entry_guard_describe(guard));
+    } else if (!guard->currently_listed && ! guard->unlisted_since_date) {
+      ++n_changes;
+      guard->unlisted_since_date = randomize_time(approx_time(),
+                                                  unlisted_since_slop);
+      log_warn(LD_BUG, "Sampled guard %s was unlisted, but with "
+               "unlisted_since_date unset. Fixing.",
+               entry_guard_describe(guard));
     }
-  SMARTLIST_FOREACH_END(entry);
-
-  if (remove_dead_entry_guards(now))
-    changed = 1;
-  if (remove_obsolete_entry_guards(now))
-    changed = 1;
-
-  if (changed) {
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-      const char *reason = digestmap_get(reasons, entry->identity);
-      const char *live_msg = "";
-      const node_t *r = entry_is_live(entry, ENTRY_NEED_CAPACITY, &live_msg);
-      log_info(LD_CIRC, "Summary: Entry %s [%s] is %s, %s%s%s, and %s%s.",
-               entry->nickname,
-               hex_str(entry->identity, DIGEST_LEN),
-               entry->unreachable_since ? "unreachable" : "reachable",
-               entry->bad_since ? "unusable" : "usable",
-               reason ? ", ": "",
-               reason ? reason : "",
-               r ? "live" : "not live / ",
-               r ? "" : live_msg);
-    } SMARTLIST_FOREACH_END(entry);
-    log_info(LD_CIRC, "    (%d/%d entry guards are usable/new)",
-             num_live_entry_guards(0), smartlist_len(entry_guards));
-    log_entry_guards(LOG_INFO);
-    entry_guards_changed();
+  } SMARTLIST_FOREACH_END(guard);
+
+  const time_t remove_if_unlisted_since =
+    approx_time() - REMOVE_UNLISTED_GUARDS_AFTER;
+  const time_t maybe_remove_if_sampled_before =
+    approx_time() - get_guard_lifetime();
+  const time_t remove_if_confirmed_before =
+    approx_time() - get_guard_confirmed_min_lifetime();
+
+  /* Then: remove the ones that have been junk for too long */
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    int rmv = 0;
+
+    if (guard->currently_listed == 0 &&
+        guard->unlisted_since_date < remove_if_unlisted_since) {
+      /*
+        "We have a live consensus, and {IS_LISTED} is false, and
+         {FIRST_UNLISTED_AT} is over {REMOVE_UNLISTED_GUARDS_AFTER}
+         days in the past."
+      */
+      log_info(LD_GUARD, "Removing sampled guard %s: it has been unlisted "
+               "for over %d days", entry_guard_describe(guard),
+               get_remove_unlisted_guards_after_days());
+      rmv = 1;
+    } else if (guard->sampled_on_date < maybe_remove_if_sampled_before) {
+      /* We have a live consensus, and {ADDED_ON_DATE} is over
+        {GUARD_LIFETIME} ago, *and* {CONFIRMED_ON_DATE} is either
+        "never", or over {GUARD_CONFIRMED_MIN_LIFETIME} ago.
+      */
+      if (guard->confirmed_on_date == 0) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, but never confirmed.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400);
+      } else if (guard->confirmed_on_date < remove_if_confirmed_before) {
+        rmv = 1;
+        log_info(LD_GUARD, "Removing sampled guard %s: it was sampled "
+                 "over %d days ago, and confirmed over %d days ago.",
+                 entry_guard_describe(guard),
+                 get_guard_lifetime() / 86400,
+                 get_guard_confirmed_min_lifetime() / 86400);
+      }
+    }
+
+    if (rmv) {
+      ++n_changes;
+      SMARTLIST_DEL_CURRENT(gs->sampled_entry_guards, guard);
+      remove_guard_from_confirmed_and_primary_lists(gs, guard);
+      entry_guard_free(guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (n_changes) {
+    gs->primary_guards_up_to_date = 0;
+    entry_guards_update_filtered_sets(gs);
+    /* We don't need to rebuild the confirmed list right here -- we may have
+     * removed confirmed guards above, but we can't have added any new
+     * confirmed guards.
+     */
+    entry_guards_changed_for_guard_selection(gs);
   }
+}
+
+/**
+ * Return true iff <b>node</b> is a Tor relay that we are configured to
+ * be able to connect to. */
+static int
+node_passes_guard_filter(const or_options_t *options,
+                         const node_t *node)
+{
+  /* NOTE: Make sure that this function stays in sync with
+   * options_transition_affects_entry_guards */
+  if (routerset_contains_node(options->ExcludeNodes, node))
+    return 0;
+
+  if (options->EntryNodes &&
+      !routerset_contains_node(options->EntryNodes, node))
+    return 0;
 
-  digestmap_free(reasons, NULL);
+  if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0))
+    return 0;
+
+  if (node_is_a_configured_bridge(node))
+    return 0;
+
+  return 1;
 }
 
-/** Called when a connection to an OR with the identity digest <b>digest</b>
- * is established (<b>succeeded</b>==1) or has failed (<b>succeeded</b>==0).
- * If the OR is an entry, change that entry's up/down status.
- * Return 0 normally, or -1 if we want to tear down the new connection.
- *
- * If <b>mark_relay_status</b>, also call router_set_status() on this
- * relay.
- */
-/* XXX We could change succeeded and mark_relay_status into 'int flags'.
- * Too many boolean arguments is a recipe for confusion.
- */
-int
-entry_guard_register_connect_status(const char *digest, int succeeded,
-                                    int mark_relay_status, time_t now)
+/** Helper: Return true iff <b>bridge</b> passes our configuration
+ * filter-- if it is a relay that we are configured to be able to
+ * connect to. */
+static int
+bridge_passes_guard_filter(const or_options_t *options,
+                           const bridge_info_t *bridge)
 {
-  int changed = 0;
-  int refuse_conn = 0;
-  int first_contact = 0;
-  entry_guard_t *entry = NULL;
-  int idx = -1;
-  char buf[HEX_DIGEST_LEN+1];
+  tor_assert(bridge);
+  if (!bridge)
+    return 0;
 
-  if (! entry_guards)
+  if (routerset_contains_bridge(options->ExcludeNodes, bridge))
     return 0;
 
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-    tor_assert(e);
-    if (tor_memeq(e->identity, digest, DIGEST_LEN)) {
-      entry = e;
-      idx = e_sl_idx;
-      break;
-    }
-  } SMARTLIST_FOREACH_END(e);
+  /* Ignore entrynodes */
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
 
-  if (!entry)
+  if (!fascist_firewall_allows_address_addr(&addrport->addr,
+                                            addrport->port,
+                                            FIREWALL_OR_CONNECTION,
+                                            0, 0))
     return 0;
 
-  base16_encode(buf, sizeof(buf), entry->identity, DIGEST_LEN);
-
-  if (succeeded) {
-    if (entry->unreachable_since) {
-      log_info(LD_CIRC, "Entry guard '%s' (%s) is now reachable again. Good.",
-               entry->nickname, buf);
-      entry->can_retry = 0;
-      entry->unreachable_since = 0;
-      entry->last_attempted = now;
-      control_event_guard(entry->nickname, entry->identity, "UP");
-      changed = 1;
-    }
-    if (!entry->made_contact) {
-      entry->made_contact = 1;
-      first_contact = changed = 1;
-    }
-  } else { /* ! succeeded */
-    if (!entry->made_contact) {
-      /* We've never connected to this one. */
-      log_info(LD_CIRC,
-               "Connection to never-contacted entry guard '%s' (%s) failed. "
-               "Removing from the list. %d/%d entry guards usable/new.",
-               entry->nickname, buf,
-               num_live_entry_guards(0)-1, smartlist_len(entry_guards)-1);
-      control_event_guard(entry->nickname, entry->identity, "DROPPED");
-      entry_guard_free(entry);
-      smartlist_del_keeporder(entry_guards, idx);
-      log_entry_guards(LOG_INFO);
-      changed = 1;
-    } else if (!entry->unreachable_since) {
-      log_info(LD_CIRC, "Unable to connect to entry guard '%s' (%s). "
-               "Marking as unreachable.", entry->nickname, buf);
-      entry->unreachable_since = entry->last_attempted = now;
-      control_event_guard(entry->nickname, entry->identity, "DOWN");
-      changed = 1;
-      entry->can_retry = 0; /* We gave it an early chance; no good. */
-    } else {
-      char tbuf[ISO_TIME_LEN+1];
-      format_iso_time(tbuf, entry->unreachable_since);
-      log_debug(LD_CIRC, "Failed to connect to unreachable entry guard "
-                "'%s' (%s).  It has been unreachable since %s.",
-                entry->nickname, buf, tbuf);
-      entry->last_attempted = now;
-      entry->can_retry = 0; /* We gave it an early chance; no good. */
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> is a Tor relay that we are configured to
+ * be able to connect to, and we haven't disabled it for omission from
+ * the consensus or path bias issues. */
+static int
+entry_guard_passes_filter(const or_options_t *options, guard_selection_t *gs,
+                          entry_guard_t *guard)
+{
+  if (guard->currently_listed == 0)
+    return 0;
+  if (guard->pb.path_bias_disabled)
+    return 0;
+
+  if (gs->type == GS_TYPE_BRIDGE) {
+    const bridge_info_t *bridge = get_bridge_info_for_guard(guard);
+    if (bridge == NULL)
+      return 0;
+    return bridge_passes_guard_filter(options, bridge);
+  } else {
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node == NULL) {
+      // This can happen when currently_listed is true, and we're not updating
+      // it because we don't have a live consensus.
+      return 0;
     }
+
+    return node_passes_guard_filter(options, node);
   }
+}
 
-  /* if the caller asked us to, also update the is_running flags for this
-   * relay */
-  if (mark_relay_status)
-    router_set_status(digest, succeeded);
-
-  if (first_contact) {
-    /* We've just added a new long-term entry guard. Perhaps the network just
-     * came back? We should give our earlier entries another try too,
-     * and close this connection so we don't use it before we've given
-     * the others a shot. */
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-        if (e == entry)
-          break;
-        if (e->made_contact) {
-          const char *msg;
-          const node_t *r = entry_is_live(e,
-                     ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE,
-                     &msg);
-          if (r && e->unreachable_since) {
-            refuse_conn = 1;
-            e->can_retry = 1;
-          }
-        }
-    } SMARTLIST_FOREACH_END(e);
-    if (refuse_conn) {
-      log_info(LD_CIRC,
-               "Connected to new entry guard '%s' (%s). Marking earlier "
-               "entry guards up. %d/%d entry guards usable/new.",
-               entry->nickname, buf,
-               num_live_entry_guards(0), smartlist_len(entry_guards));
-      log_entry_guards(LOG_INFO);
-      changed = 1;
+/** Return true iff <b>guard</b> is in the same family as <b>node</b>.
+ */
+static int
+guard_in_node_family(const entry_guard_t *guard, const node_t *node)
+{
+  const node_t *guard_node = node_get_by_id(guard->identity);
+  if (guard_node) {
+    return nodes_in_same_family(guard_node, node);
+  } else {
+    /* If we don't have a node_t for the guard node, we might have
+     * a bridge_info_t for it. So let's check to see whether the bridge
+     * address matches has any family issues.
+     *
+     * (Strictly speaking, I believe this check is unnecessary, since we only
+     * use it to avoid the exit's family when building circuits, and we don't
+     * build multihop circuits until we have a routerinfo_t for the
+     * bridge... at which point, we'll also have a node_t for the
+     * bridge. Nonetheless, it seems wise to include it, in case our
+     * assumptions change down the road.  -nickm.)
+     */
+    if (get_options()->EnforceDistinctSubnets && guard->bridge_addr) {
+      tor_addr_t node_addr;
+      node_get_addr(node, &node_addr);
+      if (addrs_in_same_network_family(&node_addr,
+                                       &guard->bridge_addr->addr)) {
+        return 1;
+      }
     }
+    return 0;
   }
+}
 
-  if (changed)
-    entry_guards_changed();
-  return refuse_conn ? -1 : 0;
+/* Allocate and return a new exit guard restriction (where <b>exit_id</b> is of
+ * size DIGEST_LEN) */
+STATIC entry_guard_restriction_t *
+guard_create_exit_restriction(const uint8_t *exit_id)
+{
+  entry_guard_restriction_t *rst = NULL;
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_EXIT_NODE;
+  memcpy(rst->exclude_id, exit_id, DIGEST_LEN);
+  return rst;
 }
 
-/** When we try to choose an entry guard, should we parse and add
- * config's EntryNodes first? */
-static int should_add_entry_nodes = 0;
+/** If we have fewer than this many possible usable guards, don't set
+ * MD-availability-based restrictions: we might blacklist all of them. */
+#define MIN_GUARDS_FOR_MD_RESTRICTION 10
 
-/** Called when the value of EntryNodes changes in our configuration. */
-void
-entry_nodes_should_be_added(void)
+/** Return true if we should set md dirserver restrictions. We might not want
+ *  to set those if our guard options are too restricted, since we don't want
+ *  to blacklist all of them. */
+static int
+should_set_md_dirserver_restriction(void)
 {
-  log_info(LD_CIRC, "EntryNodes config option set. Putting configured "
-           "relays at the front of the entry guard list.");
-  should_add_entry_nodes = 1;
+  const guard_selection_t *gs = get_guard_selection_info();
+  int num_usable_guards = num_reachable_filtered_guards(gs, NULL);
+
+  /* Don't set restriction if too few reachable filtered guards. */
+  if (num_usable_guards < MIN_GUARDS_FOR_MD_RESTRICTION) {
+    log_info(LD_GUARD, "Not setting md restriction: only %d"
+             " usable guards.", num_usable_guards);
+    return 0;
+  }
+
+  /* We have enough usable guards: set MD restriction */
+  return 1;
 }
 
-/** Update the using_as_guard fields of all the nodes. We do this after we
- * remove entry guards from the list: This is the only function that clears
- * the using_as_guard field. */
-static void
-update_node_guard_status(void)
+/** Allocate and return an outdated md guard restriction. Return NULL if no
+ *  such restriction is needed. */
+STATIC entry_guard_restriction_t *
+guard_create_dirserver_md_restriction(void)
 {
-  smartlist_t *nodes = nodelist_get_list();
-  SMARTLIST_FOREACH(nodes, node_t *, node, node->using_as_guard = 0);
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) {
-    node_t *node = node_get_mutable_by_id(entry->identity);
-    if (node)
-      node->using_as_guard = 1;
-  } SMARTLIST_FOREACH_END(entry);
+  entry_guard_restriction_t *rst = NULL;
+
+  if (!should_set_md_dirserver_restriction()) {
+    log_debug(LD_GUARD, "Not setting md restriction: too few "
+              "filtered guards.");
+    return NULL;
+  }
+
+  rst = tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  rst->type = RST_OUTDATED_MD_DIRSERVER;
+
+  return rst;
 }
 
-/** Adjust the entry guards list so that it only contains entries from
- * EntryNodes, adding new entries from EntryNodes to the list as needed. */
-STATIC void
-entry_guards_set_from_config(const or_options_t *options)
+/* Return True if <b>guard</b> obeys the exit restriction <b>rst</b>. */
+static int
+guard_obeys_exit_restriction(const entry_guard_t *guard,
+                             const entry_guard_restriction_t *rst)
 {
-  smartlist_t *entry_nodes, *worse_entry_nodes, *entry_fps;
-  smartlist_t *old_entry_guards_on_list, *old_entry_guards_not_on_list;
-  const int numentryguards = decide_num_guards(options, 0);
-  tor_assert(entry_guards);
+  tor_assert(rst->type == RST_EXIT_NODE);
 
-  should_add_entry_nodes = 0;
+  // Exclude the exit ID and all of its family.
+  const node_t *node = node_get_by_id((const char*)rst->exclude_id);
+  if (node && guard_in_node_family(guard, node))
+    return 0;
 
-  if (!options->EntryNodes) {
-    /* It's possible that a controller set EntryNodes, thus making
-     * should_add_entry_nodes set, then cleared it again, all before the
-     * call to choose_random_entry() that triggered us. If so, just return.
-     */
-    return;
+  return tor_memneq(guard->identity, rst->exclude_id, DIGEST_LEN);
+}
+
+/** Return True if <b>guard</b> should be used as a dirserver for fetching
+ *  microdescriptors. */
+static int
+guard_obeys_md_dirserver_restriction(const entry_guard_t *guard)
+{
+  /* If this guard is an outdated dirserver, don't use it. */
+  if (microdesc_relay_is_outdated_dirserver(guard->identity)) {
+    log_info(LD_GENERAL, "Skipping %s dirserver: outdated",
+             hex_str(guard->identity, DIGEST_LEN));
+    return 0;
   }
 
-  {
-    char *string = routerset_to_string(options->EntryNodes);
-    log_info(LD_CIRC,"Adding configured EntryNodes '%s'.", string);
-    tor_free(string);
+  log_debug(LD_GENERAL, "%s dirserver obeys md restrictions",
+            hex_str(guard->identity, DIGEST_LEN));
+
+  return 1;
+}
+
+/**
+ * Return true iff <b>guard</b> obeys the restrictions defined in <b>rst</b>.
+ * (If <b>rst</b> is NULL, there are no restrictions.)
+ */
+static int
+entry_guard_obeys_restriction(const entry_guard_t *guard,
+                              const entry_guard_restriction_t *rst)
+{
+  tor_assert(guard);
+  if (! rst)
+    return 1; // No restriction?  No problem.
+
+  if (rst->type == RST_EXIT_NODE) {
+    return guard_obeys_exit_restriction(guard, rst);
+  } else if (rst->type == RST_OUTDATED_MD_DIRSERVER) {
+    return guard_obeys_md_dirserver_restriction(guard);
   }
 
-  entry_nodes = smartlist_new();
-  worse_entry_nodes = smartlist_new();
-  entry_fps = smartlist_new();
-  old_entry_guards_on_list = smartlist_new();
-  old_entry_guards_not_on_list = smartlist_new();
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
 
-  /* Split entry guards into those on the list and those not. */
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flags on <b>guard</b>. */
+void
+entry_guard_set_filtered_flags(const or_options_t *options,
+                               guard_selection_t *gs,
+                               entry_guard_t *guard)
+{
+  unsigned was_filtered = guard->is_filtered_guard;
+  guard->is_filtered_guard = 0;
+  guard->is_usable_filtered_guard = 0;
 
-  routerset_get_all_nodes(entry_nodes, options->EntryNodes,
-                          options->ExcludeNodes, 0);
-  SMARTLIST_FOREACH(entry_nodes, const node_t *,node,
-                    smartlist_add(entry_fps, (void*)node->identity));
+  if (entry_guard_passes_filter(options, gs, guard)) {
+    guard->is_filtered_guard = 1;
 
-  SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e, {
-    if (smartlist_contains_digest(entry_fps, e->identity))
-      smartlist_add(old_entry_guards_on_list, e);
-    else
-      smartlist_add(old_entry_guards_not_on_list, e);
-  });
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      guard->is_usable_filtered_guard = 1;
 
-  /* Remove all currently configured guard nodes, excluded nodes, unreachable
-   * nodes, or non-Guard nodes from entry_nodes. */
-  SMARTLIST_FOREACH_BEGIN(entry_nodes, const node_t *, node) {
-    if (entry_guard_get_by_id_digest(node->identity)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
+    entry_guard_consider_retry(guard);
+  }
+  log_debug(LD_GUARD, "Updated sampled guard %s: filtered=%d; "
+            "reachable_filtered=%d.", entry_guard_describe(guard),
+            guard->is_filtered_guard, guard->is_usable_filtered_guard);
+
+  if (!bool_eq(was_filtered, guard->is_filtered_guard)) {
+    /* This guard might now be primary or nonprimary. */
+    gs->primary_guards_up_to_date = 0;
+  }
+}
+
+/**
+ * Update the <b>is_filtered_guard</b> and <b>is_usable_filtered_guard</b>
+ * flag on every guard in <b>gs</b>. */
+STATIC void
+entry_guards_update_filtered_sets(guard_selection_t *gs)
+{
+  const or_options_t *options = get_options();
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_set_filtered_flags(options, gs, guard);
+  } SMARTLIST_FOREACH_END(guard);
+}
+
+/**
+ * Return a random guard from the reachable filtered sample guards
+ * in <b>gs</b>, subject to the exclusion rules listed in <b>flags</b>.
+ * Return NULL if no such guard can be found.
+ *
+ * Make sure that the sample is big enough, and that all the filter flags
+ * are set correctly, before calling this function.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it.
+ **/
+STATIC entry_guard_t *
+sample_reachable_filtered_entry_guards(guard_selection_t *gs,
+                                       const entry_guard_restriction_t *rst,
+                                       unsigned flags)
+{
+  tor_assert(gs);
+  entry_guard_t *result = NULL;
+  const unsigned exclude_confirmed = flags & SAMPLE_EXCLUDE_CONFIRMED;
+  const unsigned exclude_primary = flags & SAMPLE_EXCLUDE_PRIMARY;
+  const unsigned exclude_pending = flags & SAMPLE_EXCLUDE_PENDING;
+  const unsigned no_update_primary = flags & SAMPLE_NO_UPDATE_PRIMARY;
+  const unsigned need_descriptor = flags & SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  const int n_reachable_filtered = num_reachable_filtered_guards(gs, rst);
+
+  log_info(LD_GUARD, "Trying to sample a reachable guard: We know of %d "
+           "in the USABLE_FILTERED set.", n_reachable_filtered);
+
+  const int min_filtered_sample = get_min_filtered_sample_size();
+  if (n_reachable_filtered < min_filtered_sample) {
+    log_info(LD_GUARD, "  (That isn't enough. Trying to expand the sample.)");
+    entry_guards_expand_sample(gs);
+  }
+
+  if (exclude_primary && !gs->primary_guards_up_to_date && !no_update_primary)
+    entry_guards_update_primary(gs);
+
+  /* Build the set of reachable filtered guards. */
+  smartlist_t *reachable_filtered_sample = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);// redundant, but cheap.
+    if (! entry_guard_obeys_restriction(guard, rst))
       continue;
-    } else if (routerset_contains_node(options->ExcludeNodes, node)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
+    if (! guard->is_usable_filtered_guard)
       continue;
-    } else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION,
-                                             0)) {
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
+    if (exclude_confirmed && guard->confirmed_idx >= 0)
       continue;
-    } else if (! node->is_possible_guard) {
-      smartlist_add(worse_entry_nodes, (node_t*)node);
-      SMARTLIST_DEL_CURRENT(entry_nodes, node);
-    }
-  } SMARTLIST_FOREACH_END(node);
+    if (exclude_primary && guard->is_primary)
+      continue;
+    if (exclude_pending && guard->is_pending)
+      continue;
+    if (need_descriptor && !guard_has_descriptor(guard))
+      continue;
+    smartlist_add(reachable_filtered_sample, guard);
+  } SMARTLIST_FOREACH_END(guard);
 
-  /* Now build the new entry_guards list. */
-  smartlist_clear(entry_guards);
-  /* First, the previously configured guards that are in EntryNodes. */
-  smartlist_add_all(entry_guards, old_entry_guards_on_list);
-  /* Next, scramble the rest of EntryNodes, putting the guards first. */
-  smartlist_shuffle(entry_nodes);
-  smartlist_shuffle(worse_entry_nodes);
-  smartlist_add_all(entry_nodes, worse_entry_nodes);
-
-  /* Next, the rest of EntryNodes */
-  SMARTLIST_FOREACH_BEGIN(entry_nodes, const node_t *, node) {
-    add_an_entry_guard(node, 0, 0, 1, 0);
-    if (smartlist_len(entry_guards) > numentryguards * 10)
-      break;
-  } SMARTLIST_FOREACH_END(node);
-  log_notice(LD_GENERAL, "%d entries in guards", smartlist_len(entry_guards));
-  /* Finally, free the remaining previously configured guards that are not in
-   * EntryNodes. */
-  SMARTLIST_FOREACH(old_entry_guards_not_on_list, entry_guard_t *, e,
-                    entry_guard_free(e));
+  log_info(LD_GUARD, "  (After filters [%x], we have %d guards to consider.)",
+           flags, smartlist_len(reachable_filtered_sample));
 
-  update_node_guard_status();
+  if (smartlist_len(reachable_filtered_sample)) {
+    result = smartlist_choose(reachable_filtered_sample);
+    log_info(LD_GUARD, "  (Selected %s.)",
+             result ? entry_guard_describe(result) : "<null>");
+  }
+  smartlist_free(reachable_filtered_sample);
 
-  smartlist_free(entry_nodes);
-  smartlist_free(worse_entry_nodes);
-  smartlist_free(entry_fps);
-  smartlist_free(old_entry_guards_on_list);
-  smartlist_free(old_entry_guards_not_on_list);
-  entry_guards_changed();
+  return result;
 }
 
-/** Return 0 if we're fine adding arbitrary routers out of the
- * directory to our entry guard list, or return 1 if we have a
- * list already and we must stick to it.
+/**
+ * Helper: compare two entry_guard_t by their confirmed_idx values.
+ * Used to sort the confirmed list.
  */
-int
-entry_list_is_constrained(const or_options_t *options)
+static int
+compare_guards_by_confirmed_idx(const void **a_, const void **b_)
 {
-  if (options->EntryNodes)
-    return 1;
-  if (options->UseBridges)
+  const entry_guard_t *a = *a_, *b = *b_;
+  if (a->confirmed_idx < b->confirmed_idx)
+    return -1;
+  else if (a->confirmed_idx > b->confirmed_idx)
     return 1;
-  return 0;
+  else
+    return 0;
 }
 
-/** Pick a live (up and listed) entry guard from entry_guards. If
- * <b>state</b> is non-NULL, this is for a specific circuit --
- * make sure not to pick this circuit's exit or any node in the
- * exit's family. If <b>state</b> is NULL, we're looking for a random
- * guard (likely a bridge).  If <b>dirinfo</b> is not NO_DIRINFO (zero),
- * then only select from nodes that know how to answer directory questions
- * of that type. */
-const node_t *
-choose_random_entry(cpath_build_state_t *state)
+/**
+ * Find the confirmed guards from among the sampled guards in <b>gs</b>,
+ * and put them in confirmed_entry_guards in the correct
+ * order. Recalculate their indices.
+ */
+STATIC void
+entry_guards_update_confirmed(guard_selection_t *gs)
 {
-  return choose_random_entry_impl(state, 0, NO_DIRINFO, NULL);
+  smartlist_clear(gs->confirmed_entry_guards);
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx >= 0)
+      smartlist_add(gs->confirmed_entry_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_confirmed_idx);
+
+  int any_changed = 0;
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->confirmed_idx != guard_sl_idx) {
+      any_changed = 1;
+      guard->confirmed_idx = guard_sl_idx;
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  gs->next_confirmed_idx = smartlist_len(gs->confirmed_entry_guards);
+
+  if (any_changed) {
+    entry_guards_changed_for_guard_selection(gs);
+  }
 }
 
-/** Pick a live (up and listed) directory guard from entry_guards for
- * downloading information of type <b>type</b>. */
-const node_t *
-choose_random_dirguard(dirinfo_type_t type)
+/**
+ * Mark <b>guard</b> as a confirmed guard -- that is, one that we have
+ * connected to, and intend to use again.
+ */
+STATIC void
+make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard)
 {
-  return choose_random_entry_impl(NULL, 1, type, NULL);
+  if (BUG(guard->confirmed_on_date && guard->confirmed_idx >= 0))
+    return; // LCOV_EXCL_LINE
+
+  if (BUG(smartlist_contains(gs->confirmed_entry_guards, guard)))
+    return; // LCOV_EXCL_LINE
+
+  const int GUARD_LIFETIME = get_guard_lifetime();
+  guard->confirmed_on_date = randomize_time(approx_time(), GUARD_LIFETIME/10);
+
+  log_info(LD_GUARD, "Marking %s as a confirmed guard (index %d)",
+           entry_guard_describe(guard),
+           gs->next_confirmed_idx);
+
+  guard->confirmed_idx = gs->next_confirmed_idx++;
+  smartlist_add(gs->confirmed_entry_guards, guard);
+
+  // This confirmed guard might kick something else out of the primary
+  // guards.
+  gs->primary_guards_up_to_date = 0;
+
+  entry_guards_changed_for_guard_selection(gs);
 }
 
-/** Filter <b>all_entry_guards</b> for usable entry guards and put them
- * in <b>live_entry_guards</b>. We filter based on whether the node is
- * currently alive, and on whether it satisfies the restrictions
- * imposed by the other arguments of this function.
- *
- * We don't place more guards than NumEntryGuards in <b>live_entry_guards</b>.
- *
- * If <b>chosen_exit</b> is set, it contains the exit node of this
- * circuit. Make sure to not use it or its family as an entry guard.
- *
- * If <b>need_uptime</b> is set, we are looking for a stable entry guard.
- * if <b>need_capacity</b> is set, we are looking for a fast entry guard.
- *
- * The rest of the arguments are the same as in choose_random_entry_impl().
- *
- * Return 1 if we should choose a guard right away. Return 0 if we
- * should try to add more nodes to our list before deciding on a
- * guard.
+/**
+ * Recalculate the list of primary guards (the ones we'd prefer to use) from
+ * the filtered sample and the confirmed list.
  */
-STATIC int
-populate_live_entry_guards(smartlist_t *live_entry_guards,
-                           const smartlist_t *all_entry_guards,
-                           const node_t *chosen_exit,
-                           dirinfo_type_t dirinfo_type,
-                           int for_directory,
-                           int need_uptime, int need_capacity)
+STATIC void
+entry_guards_update_primary(guard_selection_t *gs)
 {
-  const or_options_t *options = get_options();
-  const node_t *node = NULL;
-  const int num_needed = decide_num_guards(options, for_directory);
-  smartlist_t *exit_family = smartlist_new();
-  int retval = 0;
-  entry_is_live_flags_t entry_flags = 0;
+  tor_assert(gs);
 
-  (void) dirinfo_type;
+  // prevent recursion. Recursion is potentially very bad here.
+  static int running = 0;
+  tor_assert(!running);
+  running = 1;
 
-  { /* Set the flags we want our entry node to have */
-    if (need_uptime) {
-      entry_flags |= ENTRY_NEED_UPTIME;
-    }
-    if (need_capacity) {
-      entry_flags |= ENTRY_NEED_CAPACITY;
-    }
-    if (!for_directory) {
-      entry_flags |= ENTRY_NEED_DESCRIPTOR;
-    }
-  }
+  const int N_PRIMARY_GUARDS = get_n_primary_guards();
 
-  tor_assert(all_entry_guards);
+  smartlist_t *new_primary_guards = smartlist_new();
+  smartlist_t *old_primary_guards = smartlist_new();
+  smartlist_add_all(old_primary_guards, gs->primary_entry_guards);
 
-  if (chosen_exit) {
-    nodelist_add_node_and_family(exit_family, chosen_exit);
+  /* Set this flag now, to prevent the calls below from recursing. */
+  gs->primary_guards_up_to_date = 1;
+
+  /* First, can we fill it up with confirmed guards? */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Can we keep any older primary guards? First remove all the ones
+   * that we already kept. */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    if (smartlist_contains(new_primary_guards, guard)) {
+      SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Now add any that are still good. */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    if (smartlist_len(new_primary_guards) >= N_PRIMARY_GUARDS)
+      break;
+    if (! guard->is_filtered_guard)
+      continue;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
+    SMARTLIST_DEL_CURRENT_KEEPORDER(old_primary_guards, guard);
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Mark the remaining previous primary guards as non-primary */
+  SMARTLIST_FOREACH_BEGIN(old_primary_guards, entry_guard_t *, guard) {
+    guard->is_primary = 0;
+  } SMARTLIST_FOREACH_END(guard);
+
+  /* Finally, fill out the list with sampled guards. */
+  while (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS) {
+    entry_guard_t *guard = sample_reachable_filtered_entry_guards(gs, NULL,
+                                            SAMPLE_EXCLUDE_CONFIRMED|
+                                            SAMPLE_EXCLUDE_PRIMARY|
+                                            SAMPLE_NO_UPDATE_PRIMARY);
+    if (!guard)
+      break;
+    guard->is_primary = 1;
+    smartlist_add(new_primary_guards, guard);
   }
 
-  SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) {
-      const char *msg;
-      node = entry_is_live(entry, entry_flags, &msg);
-      if (!node)
-        continue; /* down, no point */
-      if (for_directory) {
-        if (!entry->is_dir_cache)
-          continue; /* We need a directory and didn't get one. */
-      }
-      if (node == chosen_exit)
-        continue; /* don't pick the same node for entry and exit */
-      if (smartlist_contains(exit_family, node))
-        continue; /* avoid relays that are family members of our exit */
-      smartlist_add(live_entry_guards, (void*)node);
-      if (!entry->made_contact) {
-        /* Always start with the first not-yet-contacted entry
-         * guard. Otherwise we might add several new ones, pick
-         * the second new one, and now we've expanded our entry
-         * guard list without needing to. */
-        retval = 1;
-        goto done;
-      }
-      if (smartlist_len(live_entry_guards) >= num_needed) {
-        retval = 1;
-        goto done; /* We picked enough entry guards. Done! */
+#if 1
+  /* Debugging. */
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, guard, {
+    tor_assert_nonfatal(
+                   bool_eq(guard->is_primary,
+                           smartlist_contains(new_primary_guards, guard)));
+  });
+#endif
+
+  int any_change = 0;
+  if (smartlist_len(gs->primary_entry_guards) !=
+      smartlist_len(new_primary_guards)) {
+    any_change = 1;
+  } else {
+    SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, g) {
+      if (g != smartlist_get(new_primary_guards, g_sl_idx)) {
+        any_change = 1;
       }
-  } SMARTLIST_FOREACH_END(entry);
+    } SMARTLIST_FOREACH_END(g);
+  }
 
- done:
-  smartlist_free(exit_family);
+  if (any_change) {
+    log_info(LD_GUARD, "Primary entry guards have changed. "
+             "New primary guard list is: ");
+    int n = smartlist_len(new_primary_guards);
+    SMARTLIST_FOREACH_BEGIN(new_primary_guards, entry_guard_t *, g) {
+      log_info(LD_GUARD, "  %d/%d: %s%s%s",
+               g_sl_idx+1, n, entry_guard_describe(g),
+               g->confirmed_idx >= 0 ? " (confirmed)" : "",
+               g->is_filtered_guard ? "" : " (excluded by filter)");
+    } SMARTLIST_FOREACH_END(g);
+  }
 
-  return retval;
+  smartlist_free(old_primary_guards);
+  smartlist_free(gs->primary_entry_guards);
+  gs->primary_entry_guards = new_primary_guards;
+  gs->primary_guards_up_to_date = 1;
+  running = 0;
 }
 
-/** Pick a node to be used as the entry guard of a circuit.
- *
- * If <b>state</b> is set, it contains the information we know about
- * the upcoming circuit.
- *
- * If <b>for_directory</b> is set, we are looking for a directory guard.
- *
- * <b>dirinfo_type</b> contains the kind of directory information we
- * are looking for in our node, or NO_DIRINFO (zero) if we are not
- * looking for any particular directory information (when set to
- * NO_DIRINFO, the <b>dirinfo_type</b> filter is ignored).
- *
- * If <b>n_options_out</b> is set, we set it to the number of
- * candidate guard nodes we had before picking a specific guard node.
- *
- * On success, return the node that should be used as the entry guard
- * of the circuit.  Return NULL if no such node could be found.
- *
- * Helper for choose_random{entry,dirguard}.
-*/
-static const node_t *
-choose_random_entry_impl(cpath_build_state_t *state, int for_directory,
-                         dirinfo_type_t dirinfo_type, int *n_options_out)
+/**
+ * Return the number of seconds after the last attempt at which we should
+ * retry a guard that has been failing since <b>failing_since</b>.
+ */
+static int
+get_retry_schedule(time_t failing_since, time_t now,
+                   int is_primary)
 {
-  const or_options_t *options = get_options();
-  smartlist_t *live_entry_guards = smartlist_new();
-  const node_t *chosen_exit =
-    state?build_state_get_exit_node(state) : NULL;
-  const node_t *node = NULL;
-  int need_uptime = state ? state->need_uptime : 0;
-  int need_capacity = state ? state->need_capacity : 0;
-  int preferred_min = 0;
-  const int num_needed = decide_num_guards(options, for_directory);
-  int retval = 0;
-
-  if (n_options_out)
-    *n_options_out = 0;
-
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-
-  if (should_add_entry_nodes)
-    entry_guards_set_from_config(options);
-
-  if (!entry_list_is_constrained(options) &&
-      smartlist_len(entry_guards) < num_needed)
-    pick_entry_guards(options, for_directory);
-
- retry:
-  smartlist_clear(live_entry_guards);
-
-  /* Populate the list of live entry guards so that we pick one of
-     them. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      entry_guards,
-                                      chosen_exit,
-                                      dirinfo_type,
-                                      for_directory,
-                                      need_uptime, need_capacity);
-
-  if (retval == 1) { /* We should choose a guard right now. */
-    goto choose_and_finish;
-  }
-
-  if (entry_list_is_constrained(options)) {
-    /* If we prefer the entry nodes we've got, and we have at least
-     * one choice, that's great. Use it. */
-    preferred_min = 1;
-  } else {
-    /* Try to have at least 2 choices available. This way we don't
-     * get stuck with a single live-but-crummy entry and just keep
-     * using it.
-     * (We might get 2 live-but-crummy entry guards, but so be it.) */
-    preferred_min = 2;
-  }
-
-  if (smartlist_len(live_entry_guards) < preferred_min) {
-    if (!entry_list_is_constrained(options)) {
-      /* still no? try adding a new entry then */
-      /* XXX if guard doesn't imply fast and stable, then we need
-       * to tell add_an_entry_guard below what we want, or it might
-       * be a long time til we get it. -RD */
-      node = add_an_entry_guard(NULL, 0, 0, 1, for_directory);
-      if (node) {
-        entry_guards_changed();
-        /* XXX we start over here in case the new node we added shares
-         * a family with our exit node. There's a chance that we'll just
-         * load up on entry guards here, if the network we're using is
-         * one big family. Perhaps we should teach add_an_entry_guard()
-         * to understand nodes-to-avoid-if-possible? -RD */
-        goto retry;
-      }
-    }
-    if (!node && need_uptime) {
-      need_uptime = 0; /* try without that requirement */
-      goto retry;
-    }
-    if (!node && need_capacity) {
-      /* still no? last attempt, try without requiring capacity */
-      need_capacity = 0;
-      goto retry;
-    }
+  const unsigned SIX_HOURS = 6 * 3600;
+  const unsigned FOUR_DAYS = 4 * 86400;
+  const unsigned SEVEN_DAYS = 7 * 86400;
 
-    /* live_entry_guards may be empty below. Oh well, we tried. */
+  time_t tdiff;
+  if (now > failing_since) {
+    tdiff = now - failing_since;
+  } else {
+    tdiff = 0;
   }
 
- choose_and_finish:
-  if (entry_list_is_constrained(options)) {
-    /* We need to weight by bandwidth, because our bridges or entryguards
-     * were not already selected proportional to their bandwidth. */
-    node = node_sl_choose_by_bandwidth(live_entry_guards, WEIGHT_FOR_GUARD);
-  } else {
-    /* We choose uniformly at random here, because choose_good_entry_server()
-     * already weights its choices by bandwidth, so we don't want to
-     * *double*-weight our guard selection. */
-    node = smartlist_choose(live_entry_guards);
+  const struct {
+    time_t maximum; int primary_delay; int nonprimary_delay;
+  } delays[] = {
+    { SIX_HOURS,    10*60,  1*60*60 },
+    { FOUR_DAYS,    90*60,  4*60*60 },
+    { SEVEN_DAYS, 4*60*60, 18*60*60 },
+    { TIME_MAX,   9*60*60, 36*60*60 }
+  };
+
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(delays); ++i) {
+    if (tdiff <= delays[i].maximum) {
+      return is_primary ? delays[i].primary_delay : delays[i].nonprimary_delay;
+    }
   }
-  if (n_options_out)
-    *n_options_out = smartlist_len(live_entry_guards);
-  smartlist_free(live_entry_guards);
-  return node;
+  /* LCOV_EXCL_START -- can't reach, since delays ends with TIME_MAX. */
+  tor_assert_nonfatal_unreached();
+  return 36*60*60;
+  /* LCOV_EXCL_STOP */
 }
 
-/** Parse <b>state</b> and learn about the entry guards it describes.
- * If <b>set</b> is true, and there are no errors, replace the global
- * entry_list with what we find.
- * On success, return 0. On failure, alloc into *<b>msg</b> a string
- * describing the error, and return -1.
+/**
+ * If <b>guard</b> is unreachable, consider whether enough time has passed
+ * to consider it maybe-reachable again.
  */
-int
-entry_guards_parse_state(or_state_t *state, int set, char **msg)
+STATIC void
+entry_guard_consider_retry(entry_guard_t *guard)
 {
-  entry_guard_t *node = NULL;
-  smartlist_t *new_entry_guards = smartlist_new();
-  config_line_t *line;
-  time_t now = time(NULL);
-  const char *state_version = state->TorVersion;
-  digestmap_t *added_by = digestmap_new();
-
-  *msg = NULL;
-  for (line = state->EntryGuards; line; line = line->next) {
-    if (!strcasecmp(line->key, "EntryGuard")) {
-      smartlist_t *args = smartlist_new();
-      node = tor_malloc_zero(sizeof(entry_guard_t));
-      /* all entry guards on disk have been contacted */
-      node->made_contact = 1;
-      smartlist_add(new_entry_guards, node);
-      smartlist_split_string(args, line->value, " ",
-                             SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(args)<2) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Too few arguments to EntryGuard");
-      } else if (!is_legal_nickname(smartlist_get(args,0))) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Bad nickname for EntryGuard");
-      } else {
-        strlcpy(node->nickname, smartlist_get(args,0), MAX_NICKNAME_LEN+1);
-        if (base16_decode(node->identity, DIGEST_LEN, smartlist_get(args,1),
-                          strlen(smartlist_get(args,1))) != DIGEST_LEN) {
-          *msg = tor_strdup("Unable to parse entry nodes: "
-                            "Bad hex digest for EntryGuard");
-        }
-      }
-      if (smartlist_len(args) >= 3) {
-        const char *is_cache = smartlist_get(args, 2);
-        if (!strcasecmp(is_cache, "DirCache")) {
-          node->is_dir_cache = 1;
-        } else if (!strcasecmp(is_cache, "NoDirCache")) {
-          node->is_dir_cache = 0;
-        } else {
-          log_warn(LD_CONFIG, "Bogus third argument to EntryGuard line: %s",
-                   escaped(is_cache));
-        }
-      }
-      SMARTLIST_FOREACH(args, char*, cp, tor_free(cp));
-      smartlist_free(args);
-      if (*msg)
-        break;
-    } else if (!strcasecmp(line->key, "EntryGuardDownSince") ||
-               !strcasecmp(line->key, "EntryGuardUnlistedSince")) {
-      time_t when;
-      time_t last_try = 0;
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardDownSince/UnlistedSince without EntryGuard");
-        break;
-      }
-      if (parse_iso_time_(line->value, &when, 0)<0) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-                          "Bad time in EntryGuardDownSince/UnlistedSince");
-        break;
-      }
-      if (when > now) {
-        /* It's a bad idea to believe info in the future: you can wind
-         * up with timeouts that aren't allowed to happen for years. */
-        continue;
-      }
-      if (strlen(line->value) >= ISO_TIME_LEN+ISO_TIME_LEN+1) {
-        /* ignore failure */
-        (void) parse_iso_time(line->value+ISO_TIME_LEN+1, &last_try);
-      }
-      if (!strcasecmp(line->key, "EntryGuardDownSince")) {
-        node->unreachable_since = when;
-        node->last_attempted = last_try;
-      } else {
-        node->bad_since = when;
-      }
-    } else if (!strcasecmp(line->key, "EntryGuardAddedBy")) {
-      char d[DIGEST_LEN];
-      /* format is digest version date */
-      if (strlen(line->value) < HEX_DIGEST_LEN+1+1+1+ISO_TIME_LEN) {
-        log_warn(LD_BUG, "EntryGuardAddedBy line is not long enough.");
-        continue;
-      }
-      if (base16_decode(d, sizeof(d),
-                        line->value, HEX_DIGEST_LEN) != sizeof(d) ||
-                        line->value[HEX_DIGEST_LEN] != ' ') {
-        log_warn(LD_BUG, "EntryGuardAddedBy line %s does not begin with "
-                 "hex digest", escaped(line->value));
+  if (guard->is_reachable != GUARD_REACHABLE_NO)
+    return; /* No retry needed. */
+
+  const time_t now = approx_time();
+  const int delay =
+    get_retry_schedule(guard->failing_since, now, guard->is_primary);
+  const time_t last_attempt = guard->last_tried_to_connect;
+
+  if (BUG(last_attempt == 0) ||
+      now >= last_attempt + delay) {
+    /* We should mark this retriable. */
+    char tbuf[ISO_TIME_LEN+1];
+    format_local_iso_time(tbuf, last_attempt);
+    log_info(LD_GUARD, "Marked %s%sguard %s for possible retry, since we "
+             "haven't tried to use it since %s.",
+             guard->is_primary?"primary ":"",
+             guard->confirmed_idx>=0?"confirmed ":"",
+             entry_guard_describe(guard),
+             tbuf);
+
+    guard->is_reachable = GUARD_REACHABLE_MAYBE;
+    if (guard->is_filtered_guard)
+      guard->is_usable_filtered_guard = 1;
+  }
+}
+
+/** Tell the entry guards subsystem that we have confirmed that as of
+ * just now, we're on the internet. */
+void
+entry_guards_note_internet_connectivity(guard_selection_t *gs)
+{
+  gs->last_time_on_internet = approx_time();
+}
+
+/**
+ * Get a guard for use with a circuit.  Prefer to pick a running primary
+ * guard; then a non-pending running filtered confirmed guard; then a
+ * non-pending runnable filtered guard.  Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate.  Set <b>state_out</b> to the new guard-state
+ * of the circuit.
+ */
+STATIC entry_guard_t *
+select_entry_guard_for_circuit(guard_selection_t *gs,
+                               guard_usage_t usage,
+                               const entry_guard_restriction_t *rst,
+                               unsigned *state_out)
+{
+  const int need_descriptor = (usage == GUARD_USAGE_TRAFFIC);
+  tor_assert(gs);
+  tor_assert(state_out);
+
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  int num_entry_guards = get_n_primary_guards_to_use(usage);
+  smartlist_t *usable_primary_guards = smartlist_new();
+
+  /* "If any entry in PRIMARY_GUARDS has {is_reachable} status of
+      <maybe> or <yes>, return the first such guard." */
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    if (guard->is_reachable != GUARD_REACHABLE_NO) {
+      if (need_descriptor && !guard_has_descriptor(guard)) {
         continue;
       }
-      digestmap_set(added_by, d, tor_strdup(line->value+HEX_DIGEST_LEN+1));
-    } else if (!strcasecmp(line->key, "EntryGuardPathUseBias")) {
-      const or_options_t *options = get_options();
-      double use_cnt, success_cnt;
-
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardPathUseBias without EntryGuard");
+      *state_out = GUARD_CIRC_STATE_USABLE_ON_COMPLETION;
+      guard->last_tried_to_connect = approx_time();
+      smartlist_add(usable_primary_guards, guard);
+      if (smartlist_len(usable_primary_guards) >= num_entry_guards)
         break;
-      }
+    }
+  } SMARTLIST_FOREACH_END(guard);
+
+  if (smartlist_len(usable_primary_guards)) {
+    entry_guard_t *guard = smartlist_choose(usable_primary_guards);
+    smartlist_free(usable_primary_guards);
+    log_info(LD_GUARD, "Selected primary guard %s for circuit.",
+             entry_guard_describe(guard));
+    return guard;
+  }
+  smartlist_free(usable_primary_guards);
+
+  /* "Otherwise, if the ordered intersection of {CONFIRMED_GUARDS}
+      and {USABLE_FILTERED_GUARDS} is nonempty, return the first
+      entry in that intersection that has {is_pending} set to
+      false." */
+  SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
+    if (guard->is_primary)
+      continue; /* we already considered this one. */
+    if (! entry_guard_obeys_restriction(guard, rst))
+      continue;
+    entry_guard_consider_retry(guard);
+    if (guard->is_usable_filtered_guard && ! guard->is_pending) {
+      if (need_descriptor && !guard_has_descriptor(guard))
+        continue; /* not a bug */
+      guard->is_pending = 1;
+      guard->last_tried_to_connect = approx_time();
+      *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+      log_info(LD_GUARD, "No primary guards available. Selected confirmed "
+               "guard %s for circuit. Will try other guards before using "
+               "this circuit.",
+               entry_guard_describe(guard));
+      return guard;
+    }
+  } SMARTLIST_FOREACH_END(guard);
 
-      if (tor_sscanf(line->value, "%lf %lf",
-                     &use_cnt, &success_cnt) != 2) {
-        log_info(LD_GENERAL, "Malformed path use bias line for node %s",
-                 node->nickname);
-        continue;
-      }
+  /* "Otherwise, if there is no such entry, select a member at
+      random from {USABLE_FILTERED_GUARDS}." */
+  {
+    entry_guard_t *guard;
+    unsigned flags = 0;
+    if (need_descriptor)
+      flags |= SAMPLE_EXCLUDE_NO_DESCRIPTOR;
+    guard = sample_reachable_filtered_entry_guards(gs,
+                                                   rst,
+                                                   SAMPLE_EXCLUDE_CONFIRMED |
+                                                   SAMPLE_EXCLUDE_PRIMARY |
+                                                   SAMPLE_EXCLUDE_PENDING |
+                                                   flags);
+    if (guard == NULL) {
+      log_info(LD_GUARD, "Absolutely no sampled guards were available. "
+               "Marking all guards for retry and starting from top again.");
+      mark_all_guards_maybe_reachable(gs);
+      return NULL;
+    }
+    guard->is_pending = 1;
+    guard->last_tried_to_connect = approx_time();
+    *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+    log_info(LD_GUARD, "No primary or confirmed guards available. Selected "
+             "random guard %s for circuit. Will try other guards before "
+             "using this circuit.",
+             entry_guard_describe(guard));
+    return guard;
+  }
+}
 
-      if (use_cnt < success_cnt) {
-        int severity = LOG_INFO;
-        /* If this state file was written by a Tor that would have
-         * already fixed it, then the overcounting bug is still there.. */
-        if (tor_version_as_new_as(state_version, "0.2.4.13-alpha")) {
-          severity = LOG_NOTICE;
-        }
-        log_fn(severity, LD_BUG,
-                   "State file contains unexpectedly high usage success "
-                   "counts %lf/%lf for Guard %s ($%s)",
-                   success_cnt, use_cnt,
-                   node->nickname, hex_str(node->identity, DIGEST_LEN));
-        success_cnt = use_cnt;
-      }
+/**
+ * Note that we failed to connect to or build circuits through <b>guard</b>.
+ * Use with a guard returned by select_entry_guard_for_circuit().
+ */
+STATIC void
+entry_guards_note_guard_failure(guard_selection_t *gs,
+                                entry_guard_t *guard)
+{
+  tor_assert(gs);
 
-      node->use_attempts = use_cnt;
-      node->use_successes = success_cnt;
-
-      log_info(LD_GENERAL, "Read %f/%f path use bias for node %s",
-               node->use_successes, node->use_attempts, node->nickname);
-
-      /* Note: We rely on the < comparison here to allow us to set a 0
-       * rate and disable the feature entirely. If refactoring, don't
-       * change to <= */
-      if (pathbias_get_use_success_count(node)/node->use_attempts
-            < pathbias_get_extreme_use_rate(options) &&
-          pathbias_get_dropguards(options)) {
-        node->path_bias_disabled = 1;
-        log_info(LD_GENERAL,
-                 "Path use bias is too high (%f/%f); disabling node %s",
-                 node->circ_successes, node->circ_attempts, node->nickname);
-      }
-    } else if (!strcasecmp(line->key, "EntryGuardPathBias")) {
-      const or_options_t *options = get_options();
-      double hop_cnt, success_cnt, timeouts, collapsed, successful_closed,
-               unusable;
-
-      if (!node) {
-        *msg = tor_strdup("Unable to parse entry nodes: "
-               "EntryGuardPathBias without EntryGuard");
-        break;
-      }
+  guard->is_reachable = GUARD_REACHABLE_NO;
+  guard->is_usable_filtered_guard = 0;
 
-      /* First try 3 params, then 2. */
-      /* In the long run: circuit_success ~= successful_circuit_close +
-       *                                     collapsed_circuits +
-       *                                     unusable_circuits */
-      if (tor_sscanf(line->value, "%lf %lf %lf %lf %lf %lf",
-                  &hop_cnt, &success_cnt, &successful_closed,
-                  &collapsed, &unusable, &timeouts) != 6) {
-        int old_success, old_hops;
-        if (tor_sscanf(line->value, "%u %u", &old_success, &old_hops) != 2) {
-          continue;
-        }
-        log_info(LD_GENERAL, "Reading old-style EntryGuardPathBias %s",
-                 escaped(line->value));
-
-        success_cnt = old_success;
-        successful_closed = old_success;
-        hop_cnt = old_hops;
-        timeouts = 0;
-        collapsed = 0;
-        unusable = 0;
-      }
+  guard->is_pending = 0;
+  if (guard->failing_since == 0)
+    guard->failing_since = approx_time();
 
-      if (hop_cnt < success_cnt) {
-        int severity = LOG_INFO;
-        /* If this state file was written by a Tor that would have
-         * already fixed it, then the overcounting bug is still there.. */
-        if (tor_version_as_new_as(state_version, "0.2.4.13-alpha")) {
-          severity = LOG_NOTICE;
-        }
-        log_fn(severity, LD_BUG,
-                "State file contains unexpectedly high success counts "
-                "%lf/%lf for Guard %s ($%s)",
-                success_cnt, hop_cnt,
-                node->nickname, hex_str(node->identity, DIGEST_LEN));
-        success_cnt = hop_cnt;
-      }
+  log_info(LD_GUARD, "Recorded failure for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
+}
 
-      node->circ_attempts = hop_cnt;
-      node->circ_successes = success_cnt;
-
-      node->successful_circuits_closed = successful_closed;
-      node->timeouts = timeouts;
-      node->collapsed_circuits = collapsed;
-      node->unusable_circuits = unusable;
-
-      log_info(LD_GENERAL, "Read %f/%f path bias for node %s",
-               node->circ_successes, node->circ_attempts, node->nickname);
-      /* Note: We rely on the < comparison here to allow us to set a 0
-       * rate and disable the feature entirely. If refactoring, don't
-       * change to <= */
-      if (pathbias_get_close_success_count(node)/node->circ_attempts
-            < pathbias_get_extreme_rate(options) &&
-          pathbias_get_dropguards(options)) {
-        node->path_bias_disabled = 1;
-        log_info(LD_GENERAL,
-                 "Path bias is too high (%f/%f); disabling node %s",
-                 node->circ_successes, node->circ_attempts, node->nickname);
-      }
+/**
+ * Note that we successfully connected to, and built a circuit through
+ * <b>guard</b>. Given the old guard-state of the circuit in <b>old_state</b>,
+ * return the new guard-state of the circuit.
+ *
+ * Be aware: the circuit is only usable when its guard-state becomes
+ * GUARD_CIRC_STATE_COMPLETE.
+ **/
+STATIC unsigned
+entry_guards_note_guard_success(guard_selection_t *gs,
+                                entry_guard_t *guard,
+                                unsigned old_state)
+{
+  tor_assert(gs);
+
+  /* Save this, since we're about to overwrite it. */
+  const time_t last_time_on_internet = gs->last_time_on_internet;
+  gs->last_time_on_internet = approx_time();
+
+  guard->is_reachable = GUARD_REACHABLE_YES;
+  guard->failing_since = 0;
+  guard->is_pending = 0;
+  if (guard->is_filtered_guard)
+    guard->is_usable_filtered_guard = 1;
+
+  if (guard->confirmed_idx < 0) {
+    make_guard_confirmed(gs, guard);
+    if (!gs->primary_guards_up_to_date)
+      entry_guards_update_primary(gs);
+  }
 
-    } else {
-      log_warn(LD_BUG, "Unexpected key %s", line->key);
-    }
+  unsigned new_state;
+  switch (old_state) {
+    case GUARD_CIRC_STATE_COMPLETE:
+    case GUARD_CIRC_STATE_USABLE_ON_COMPLETION:
+      new_state = GUARD_CIRC_STATE_COMPLETE;
+      break;
+    default:
+      tor_assert_nonfatal_unreached();
+      /* Fall through. */
+    case GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD:
+      if (guard->is_primary) {
+        /* XXXX #20832 -- I don't actually like this logic. It seems to make
+         * us a little more susceptible to evil-ISP attacks.  The mitigations
+         * I'm thinking of, however, aren't local to this point, so I'll leave
+         * it alone. */
+        /* This guard may have become primary by virtue of being confirmed.
+         * If so, the circuit for it is now complete.
+         */
+        new_state = GUARD_CIRC_STATE_COMPLETE;
+      } else {
+        new_state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+      }
+      break;
   }
 
-  SMARTLIST_FOREACH_BEGIN(new_entry_guards, entry_guard_t *, e) {
-     char *sp;
-     char *val = digestmap_get(added_by, e->identity);
-     if (val && (sp = strchr(val, ' '))) {
-       time_t when;
-       *sp++ = '\0';
-       if (parse_iso_time(sp, &when)<0) {
-         log_warn(LD_BUG, "Can't read time %s in EntryGuardAddedBy", sp);
-       } else {
-         e->chosen_by_version = tor_strdup(val);
-         e->chosen_on_date = when;
-       }
-     } else {
-       if (state_version) {
-         e->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now);
-         e->chosen_by_version = tor_strdup(state_version);
-       }
-     }
-     if (e->path_bias_disabled && !e->bad_since)
-       e->bad_since = time(NULL);
+  if (! guard->is_primary) {
+    if (last_time_on_internet + get_internet_likely_down_interval()
+        < approx_time()) {
+      mark_primary_guards_maybe_reachable(gs);
     }
-  SMARTLIST_FOREACH_END(e);
+  }
 
-  if (*msg || !set) {
-    SMARTLIST_FOREACH(new_entry_guards, entry_guard_t *, e,
-                      entry_guard_free(e));
-    smartlist_free(new_entry_guards);
-  } else { /* !err && set */
-    if (entry_guards) {
-      SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e,
-                        entry_guard_free(e));
-      smartlist_free(entry_guards);
-    }
-    entry_guards = new_entry_guards;
-    entry_guards_dirty = 0;
-    /* XXX hand new_entry_guards to this func, and move it up a
-     * few lines, so we don't have to re-dirty it */
-    if (remove_obsolete_entry_guards(now))
-      entry_guards_dirty = 1;
+  log_info(LD_GUARD, "Recorded success for %s%sguard %s",
+           guard->is_primary?"primary ":"",
+           guard->confirmed_idx>=0?"confirmed ":"",
+           entry_guard_describe(guard));
 
-    update_node_guard_status();
-  }
-  digestmap_free(added_by, tor_free_);
-  return *msg ? -1 : 0;
+  return new_state;
 }
 
-/** How long will we let a change in our guard nodes stay un-saved
- * when we are trying to avoid disk writes? */
-#define SLOW_GUARD_STATE_FLUSH_TIME 600
-/** How long will we let a change in our guard nodes stay un-saved
- * when we are not trying to avoid disk writes? */
-#define FAST_GUARD_STATE_FLUSH_TIME 30
-
-/** Our list of entry guards has changed, or some element of one
- * of our entry guards has changed. Write the changes to disk within
- * the next few minutes.
+/**
+ * Helper: Return true iff <b>a</b> has higher priority than <b>b</b>.
  */
-void
-entry_guards_changed(void)
+STATIC int
+entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b)
 {
-  time_t when;
-  entry_guards_dirty = 1;
+  tor_assert(a && b);
+  if (a == b)
+    return 0;
 
-  if (get_options()->AvoidDiskWrites)
-    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
-  else
-    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
+  /* Confirmed is always better than unconfirmed; lower index better
+     than higher */
+  if (a->confirmed_idx < 0) {
+    if (b->confirmed_idx >= 0)
+      return 0;
+  } else {
+    if (b->confirmed_idx < 0)
+      return 1;
 
-  /* or_state_save() will call entry_guards_update_state(). */
-  or_state_mark_dirty(get_or_state(), when);
+    /* Lower confirmed_idx is better than higher. */
+    return (a->confirmed_idx < b->confirmed_idx);
+  }
+
+  /* If we reach this point, both are unconfirmed. If one is pending, it
+   * has higher priority. */
+  if (a->is_pending) {
+    if (! b->is_pending)
+      return 1;
+
+    /* Both are pending: earlier last_tried_connect wins. */
+    return a->last_tried_to_connect < b->last_tried_to_connect;
+  } else {
+    if (b->is_pending)
+      return 0;
+
+    /* Neither is pending: priorities are equal. */
+    return 0;
+  }
 }
 
-/** If the entry guard info has not changed, do nothing and return.
- * Otherwise, free the EntryGuards piece of <b>state</b> and create
- * a new one out of the global entry_guards list, and then mark
- * <b>state</b> dirty so it will get saved to disk.
+/** Release all storage held in <b>restriction</b> */
+STATIC void
+entry_guard_restriction_free(entry_guard_restriction_t *rst)
+{
+  tor_free(rst);
+}
+
+/**
+ * Release all storage held in <b>state</b>.
  */
 void
-entry_guards_update_state(or_state_t *state)
+circuit_guard_state_free(circuit_guard_state_t *state)
 {
-  config_line_t **next, *line;
-  if (! entry_guards_dirty)
+  if (!state)
     return;
+  entry_guard_restriction_free(state->restrictions);
+  entry_guard_handle_free(state->guard);
+  tor_free(state);
+}
 
-  config_free_lines(state->EntryGuards);
-  next = &state->EntryGuards;
-  *next = NULL;
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-      char dbuf[HEX_DIGEST_LEN+1];
-      if (!e->made_contact)
-        continue; /* don't write this one to disk */
-      *next = line = tor_malloc_zero(sizeof(config_line_t));
-      line->key = tor_strdup("EntryGuard");
-      base16_encode(dbuf, sizeof(dbuf), e->identity, DIGEST_LEN);
-      tor_asprintf(&line->value, "%s %s %sDirCache", e->nickname, dbuf,
-                   e->is_dir_cache ? "" : "No");
-      next = &(line->next);
-      if (e->unreachable_since) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardDownSince");
-        line->value = tor_malloc(ISO_TIME_LEN+1+ISO_TIME_LEN+1);
-        format_iso_time(line->value, e->unreachable_since);
-        if (e->last_attempted) {
-          line->value[ISO_TIME_LEN] = ' ';
-          format_iso_time(line->value+ISO_TIME_LEN+1, e->last_attempted);
-        }
-        next = &(line->next);
-      }
-      if (e->bad_since) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardUnlistedSince");
-        line->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(line->value, e->bad_since);
-        next = &(line->next);
-      }
-      if (e->chosen_on_date && e->chosen_by_version &&
-          !strchr(e->chosen_by_version, ' ')) {
-        char d[HEX_DIGEST_LEN+1];
-        char t[ISO_TIME_LEN+1];
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardAddedBy");
-        base16_encode(d, sizeof(d), e->identity, DIGEST_LEN);
-        format_iso_time(t, e->chosen_on_date);
-        tor_asprintf(&line->value, "%s %s %s",
-                     d, e->chosen_by_version, t);
-        next = &(line->next);
-      }
-      if (e->circ_attempts > 0) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardPathBias");
-        /* In the long run: circuit_success ~= successful_circuit_close +
-         *                                     collapsed_circuits +
-         *                                     unusable_circuits */
-        tor_asprintf(&line->value, "%f %f %f %f %f %f",
-                     e->circ_attempts, e->circ_successes,
-                     pathbias_get_close_success_count(e),
-                     e->collapsed_circuits,
-                     e->unusable_circuits, e->timeouts);
-        next = &(line->next);
-      }
-      if (e->use_attempts > 0) {
-        *next = line = tor_malloc_zero(sizeof(config_line_t));
-        line->key = tor_strdup("EntryGuardPathUseBias");
-
-        tor_asprintf(&line->value, "%f %f",
-                     e->use_attempts,
-                     pathbias_get_use_success_count(e));
-        next = &(line->next);
-      }
+/** Allocate and return a new circuit_guard_state_t to track the result
+ * of using <b>guard</b> for a given operation. */
+static circuit_guard_state_t *
+circuit_guard_state_new(entry_guard_t *guard, unsigned state,
+                        entry_guard_restriction_t *rst)
+{
+  circuit_guard_state_t *result;
 
-  } SMARTLIST_FOREACH_END(e);
-  if (!get_options()->AvoidDiskWrites)
-    or_state_mark_dirty(get_or_state(), 0);
-  entry_guards_dirty = 0;
+  result = tor_malloc_zero(sizeof(circuit_guard_state_t));
+  result->guard = entry_guard_handle_new(guard);
+  result->state = state;
+  result->state_set_at = approx_time();
+  result->restrictions = rst;
+
+  return result;
 }
 
-/** If <b>question</b> is the string "entry-guards", then dump
- * to *<b>answer</b> a newly allocated string describing all of
- * the nodes in the global entry_guards list. See control-spec.txt
- * for details.
- * For backward compatibility, we also handle the string "helper-nodes".
- * */
+/**
+ * Pick a suitable entry guard for a circuit in, and place that guard
+ * in *<b>chosen_node_out</b>. Set *<b>guard_state_out</b> to an opaque
+ * state object that will record whether the circuit is ready to be used
+ * or not. Return 0 on success; on failure, return -1.
+ *
+ * If a restriction is provided in <b>rst</b>, do not return any guards that
+ * violate it, and remember that restriction in <b>guard_state_out</b> for
+ * later use. (Takes ownership of the <b>rst</b> object.)
+ */
 int
-getinfo_helper_entry_guards(control_connection_t *conn,
-                            const char *question, char **answer,
-                            const char **errmsg)
-{
-  (void) conn;
-  (void) errmsg;
+entry_guard_pick_for_circuit(guard_selection_t *gs,
+                             guard_usage_t usage,
+                             entry_guard_restriction_t *rst,
+                             const node_t **chosen_node_out,
+                             circuit_guard_state_t **guard_state_out)
+{
+  tor_assert(gs);
+  tor_assert(chosen_node_out);
+  tor_assert(guard_state_out);
+  *chosen_node_out = NULL;
+  *guard_state_out = NULL;
+
+  unsigned state = 0;
+  entry_guard_t *guard =
+    select_entry_guard_for_circuit(gs, usage, rst, &state);
+  if (! guard)
+    goto fail;
+  if (BUG(state == 0))
+    goto fail;
+  const node_t *node = node_get_by_id(guard->identity);
+  // XXXX #20827 check Ed ID.
+  if (! node)
+    goto fail;
+  if (BUG(usage != GUARD_USAGE_DIRGUARD && !node_has_descriptor(node)))
+    goto fail;
+
+  *chosen_node_out = node;
+  *guard_state_out = circuit_guard_state_new(guard, state, rst);
 
-  if (!strcmp(question,"entry-guards") ||
-      !strcmp(question,"helper-nodes")) {
-    smartlist_t *sl = smartlist_new();
-    char tbuf[ISO_TIME_LEN+1];
-    char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
-    if (!entry_guards)
-      entry_guards = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-        const char *status = NULL;
-        time_t when = 0;
-        const node_t *node;
-
-        if (!e->made_contact) {
-          status = "never-connected";
-        } else if (e->bad_since) {
-          when = e->bad_since;
-          status = "unusable";
-        } else if (e->unreachable_since) {
-          when = e->unreachable_since;
-          status = "down";
-        } else {
-          status = "up";
-        }
-
-        node = node_get_by_id(e->identity);
-        if (node) {
-          node_get_verbose_nickname(node, nbuf);
-        } else {
-          nbuf[0] = '$';
-          base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
-          /* e->nickname field is not very reliable if we don't know about
-           * this router any longer; don't include it. */
-        }
-
-        if (when) {
-          format_iso_time(tbuf, when);
-          smartlist_add_asprintf(sl, "%s %s %s\n", nbuf, status, tbuf);
-        } else {
-          smartlist_add_asprintf(sl, "%s %s\n", nbuf, status);
-        }
-    } SMARTLIST_FOREACH_END(e);
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  }
   return 0;
+ fail:
+  entry_guard_restriction_free(rst);
+  return -1;
 }
 
-/** Return 0 if we should apply guardfraction information found in the
- *  consensus. A specific consensus can be specified with the
- *  <b>ns</b> argument, if NULL the most recent one will be picked.*/
-int
-should_apply_guardfraction(const networkstatus_t *ns)
+/**
+ * Called by the circuit building module when a circuit has succeeded: informs
+ * the guards code that the guard in *<b>guard_state_p</b> is working, and
+ * advances the state of the guard module.  On a GUARD_USABLE_NEVER return
+ * value, the circuit is broken and should not be used.  On a GUARD_USABLE_NOW
+ * return value, the circuit is ready to use.  On a GUARD_MAYBE_USABLE_LATER
+ * return value, the circuit should not be used until we find out whether
+ * preferred guards will work for us.
+ */
+guard_usable_t
+entry_guard_succeeded(circuit_guard_state_t **guard_state_p)
 {
-  /* We need to check the corresponding torrc option and the consensus
-   * parameter if we need to. */
-  const or_options_t *options = get_options();
+  if (BUG(*guard_state_p == NULL))
+    return GUARD_USABLE_NEVER;
 
-  /* If UseGuardFraction is 'auto' then check the same-named consensus
-   * parameter. If the consensus parameter is not present, default to
-   * "off". */
-  if (options->UseGuardFraction == -1) {
-    return networkstatus_get_param(ns, "UseGuardFraction",
-                                   0, /* default to "off" */
-                                   0, 1);
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return GUARD_USABLE_NEVER;
+
+  unsigned newstate =
+    entry_guards_note_guard_success(guard->in_selection, guard,
+                                    (*guard_state_p)->state);
+
+  (*guard_state_p)->state = newstate;
+  (*guard_state_p)->state_set_at = approx_time();
+
+  if (newstate == GUARD_CIRC_STATE_COMPLETE) {
+    return GUARD_USABLE_NOW;
+  } else {
+    return GUARD_MAYBE_USABLE_LATER;
   }
+}
 
-  return options->UseGuardFraction;
+/** Cancel the selection of *<b>guard_state_p</b> without declaring
+ * success or failure. It is safe to call this function if success or
+ * failure _has_ already been declared. */
+void
+entry_guard_cancel(circuit_guard_state_t **guard_state_p)
+{
+  if (BUG(*guard_state_p == NULL))
+    return;
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard)
+    return;
+
+  /* XXXX prop271 -- last_tried_to_connect_at will be erroneous here, but this
+   * function will only get called in "bug" cases anyway. */
+  guard->is_pending = 0;
+  circuit_guard_state_free(*guard_state_p);
+  *guard_state_p = NULL;
 }
 
-/* Given the original bandwidth of a guard and its guardfraction,
- * calculate how much bandwidth the guard should have as a guard and
- * as a non-guard.
- *
- * Quoting from proposal236:
- *
- *   Let Wpf denote the weight from the 'bandwidth-weights' line a
- *   client would apply to N for position p if it had the guard
- *   flag, Wpn the weight if it did not have the guard flag, and B the
- *   measured bandwidth of N in the consensus.  Then instead of choosing
- *   N for position p proportionally to Wpf*B or Wpn*B, clients should
- *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
- *
- * This function fills the <b>guardfraction_bw</b> structure. It sets
- * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
+/**
+ * Called by the circuit building module when a circuit has succeeded:
+ * informs the guards code that the guard in *<b>guard_state_p</b> is
+ * not working, and advances the state of the guard module.
  */
 void
-guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
-                                  int orig_bandwidth,
-                                  uint32_t guardfraction_percentage)
+entry_guard_failed(circuit_guard_state_t **guard_state_p)
 {
-  double guardfraction_fraction;
-
-  /* Turn the percentage into a fraction. */
-  tor_assert(guardfraction_percentage <= 100);
-  guardfraction_fraction = guardfraction_percentage / 100.0;
+  if (BUG(*guard_state_p == NULL))
+    return;
 
-  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
-  tor_assert(guard_bw <= INT_MAX);
+  entry_guard_t *guard = entry_guard_handle_get((*guard_state_p)->guard);
+  if (! guard || BUG(guard->in_selection == NULL))
+    return;
 
-  guardfraction_bw->guard_bw = (int) guard_bw;
+  entry_guards_note_guard_failure(guard->in_selection, guard);
 
-  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
+  (*guard_state_p)->state = GUARD_CIRC_STATE_DEAD;
+  (*guard_state_p)->state_set_at = approx_time();
 }
 
-/** A list of configured bridges. Whenever we actually get a descriptor
- * for one, we add it as an entry guard.  Note that the order of bridges
- * in this list does not necessarily correspond to the order of bridges
- * in the torrc. */
-static smartlist_t *bridge_list = NULL;
-
-/** Mark every entry of the bridge list to be removed on our next call to
- * sweep_bridge_list unless it has first been un-marked. */
+/**
+ * Run the entry_guard_failed() function on every circuit that is
+ * pending on <b>chan</b>.
+ */
 void
-mark_bridge_list(void)
+entry_guard_chan_failed(channel_t *chan)
 {
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b,
-                    b->marked_for_removal = 1);
-}
+  if (!chan)
+    return;
 
-/** Remove every entry of the bridge list that was marked with
- * mark_bridge_list if it has not subsequently been un-marked. */
-void
-sweep_bridge_list(void)
-{
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-    if (b->marked_for_removal) {
-      SMARTLIST_DEL_CURRENT(bridge_list, b);
-      bridge_free(b);
+  smartlist_t *pending = smartlist_new();
+  circuit_get_all_pending_on_channel(pending, chan);
+  SMARTLIST_FOREACH_BEGIN(pending, circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ);
+    if (origin_circ->guard_state) {
+      /* We might have no guard state if we didn't use a guard on this
+       * circuit (eg it's for a fallback directory). */
+      entry_guard_failed(&origin_circ->guard_state);
     }
-  } SMARTLIST_FOREACH_END(b);
+  } SMARTLIST_FOREACH_END(circ);
+  smartlist_free(pending);
 }
 
-/** Initialize the bridge list to empty, creating it if needed. */
-static void
-clear_bridge_list(void)
+/**
+ * Return true iff every primary guard in <b>gs</b> is believed to
+ * be unreachable.
+ */
+STATIC int
+entry_guards_all_primary_guards_are_down(guard_selection_t *gs)
+{
+  tor_assert(gs);
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable != GUARD_REACHABLE_NO)
+      return 0;
+  } SMARTLIST_FOREACH_END(guard);
+  return 1;
+}
+
+/** Wrapper for entry_guard_has_higher_priority that compares the
+ * guard-priorities of a pair of circuits. Return 1 if <b>a</b> has higher
+ * priority than <b>b</b>.
+ *
+ * If a restriction is provided in <b>rst</b>, then do not consider
+ * <b>a</b> to have higher priority if it violates the restriction.
+ */
+static int
+circ_state_has_higher_priority(origin_circuit_t *a,
+                               const entry_guard_restriction_t *rst,
+                               origin_circuit_t *b)
 {
-  if (!bridge_list)
-    bridge_list = smartlist_new();
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, b, bridge_free(b));
-  smartlist_clear(bridge_list);
-}
+  circuit_guard_state_t *state_a = origin_circuit_get_guard_state(a);
+  circuit_guard_state_t *state_b = origin_circuit_get_guard_state(b);
 
-/** Free the bridge <b>bridge</b>. */
-static void
-bridge_free(bridge_info_t *bridge)
-{
-  if (!bridge)
-    return;
+  tor_assert(state_a);
+  tor_assert(state_b);
 
-  tor_free(bridge->transport_name);
-  if (bridge->socks_args) {
-    SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s));
-    smartlist_free(bridge->socks_args);
-  }
+  entry_guard_t *guard_a = entry_guard_handle_get(state_a->guard);
+  entry_guard_t *guard_b = entry_guard_handle_get(state_b->guard);
 
-  tor_free(bridge);
+  if (! guard_a) {
+    /* Unknown guard -- never higher priority. */
+    return 0;
+  } else if (! guard_b) {
+    /* Known guard -- higher priority than any unknown guard. */
+    return 1;
+  } else  if (! entry_guard_obeys_restriction(guard_a, rst)) {
+    /* Restriction violated; guard_a cannot have higher priority. */
+    return 0;
+  } else {
+    /* Both known -- compare.*/
+    return entry_guard_has_higher_priority(guard_a, guard_b);
+  }
 }
 
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches any of the
- * tor_addr_port_t's in <b>orports</b>, return that bridge.  Else return
- * NULL. */
-static bridge_info_t *
-get_configured_bridge_by_orports_digest(const char *digest,
-                                        const smartlist_t *orports)
-{
-  if (!bridge_list)
-    return NULL;
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if (tor_digest_is_zero(bridge->identity)) {
-        SMARTLIST_FOREACH_BEGIN(orports, tor_addr_port_t *, ap)
-          {
-            if (tor_addr_compare(&bridge->addr, &ap->addr, CMP_EXACT) == 0 &&
-                bridge->port == ap->port)
-              return bridge;
-          }
-        SMARTLIST_FOREACH_END(ap);
+/**
+ * Look at all of the origin_circuit_t * objects in <b>all_circuits_in</b>,
+ * and see if any of them that were previously not ready to use for
+ * guard-related reasons are now ready to use. Place those circuits
+ * in <b>newly_complete_out</b>, and mark them COMPLETE.
+ *
+ * Return 1 if we upgraded any circuits, and 0 otherwise.
+ */
+int
+entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                      const smartlist_t *all_circuits_in,
+                                      smartlist_t *newly_complete_out)
+{
+  tor_assert(gs);
+  tor_assert(all_circuits_in);
+  tor_assert(newly_complete_out);
+
+  if (! entry_guards_all_primary_guards_are_down(gs)) {
+    /* We only upgrade a waiting circuit if the primary guards are all
+     * down. */
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but not all primary guards were definitely down.");
+    return 0;
+  }
+
+  int n_waiting = 0;
+  int n_complete = 0;
+  int n_complete_blocking = 0;
+  origin_circuit_t *best_waiting_circuit = NULL;
+  smartlist_t *all_circuits = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(all_circuits_in, origin_circuit_t *, circ) {
+    // We filter out circuits that aren't ours, or which we can't
+    // reason about.
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (state == NULL)
+      continue;
+    entry_guard_t *guard = entry_guard_handle_get(state->guard);
+    if (!guard || guard->in_selection != gs)
+      continue;
+
+    smartlist_add(all_circuits, circ);
+  } SMARTLIST_FOREACH_END(circ);
+
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+
+    if (state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD) {
+      ++n_waiting;
+      if (! best_waiting_circuit ||
+          circ_state_has_higher_priority(circ, NULL, best_waiting_circuit)) {
+        best_waiting_circuit = circ;
       }
-      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
     }
-  SMARTLIST_FOREACH_END(bridge);
-  return NULL;
-}
+  } SMARTLIST_FOREACH_END(circ);
 
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
- * return that bridge.  Else return NULL. If <b>digest</b> is NULL, check for
- * address/port matches only. */
-static bridge_info_t *
-get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr,
-                                          uint16_t port,
-                                          const char *digest)
-{
-  if (!bridge_list)
-    return NULL;
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if ((tor_digest_is_zero(bridge->identity) || digest == NULL) &&
-          !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) &&
-          bridge->port == port)
-        return bridge;
-      if (digest && tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
+  if (! best_waiting_circuit) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits, "
+              "but didn't find any.");
+    goto no_change;
+  }
+
+  /* We'll need to keep track of what restrictions were used when picking this
+   * circuit, so that we don't allow any circuit without those restrictions to
+   * block it. */
+  const entry_guard_restriction_t *rst_on_best_waiting =
+    origin_circuit_get_guard_state(best_waiting_circuit)->restrictions;
+
+  /* First look at the complete circuits: Do any block this circuit? */
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    /* "C2 "blocks" C1 if:
+        * C2 obeys all the restrictions that C1 had to obey, AND
+        * C2 has higher priority than C1, AND
+        * Either C2 is <complete>, or C2 is <waiting_for_better_guard>,
+          or C2 has been <usable_if_no_better_guard> for no more than
+          {NONPRIMARY_GUARD_CONNECT_TIMEOUT} seconds."
+    */
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if BUG((state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_COMPLETE)
+      continue;
+    ++n_complete;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_complete_blocking;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_complete_blocking) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d complete and %d guard-stalled. At least one complete "
+              "circuit had higher priority, so not upgrading.",
+              n_complete, n_waiting);
+    goto no_change;
+  }
+
+  /* " * If any circuit C1 is <waiting_for_better_guard>, AND:
+          * All primary guards have reachable status of <no>.
+          * There is no circuit C2 that "blocks" C1.
+         Then, upgrade C1 to <complete>.""
+  */
+  int n_blockers_found = 0;
+  const time_t state_set_at_cutoff =
+    approx_time() - get_nonprimary_guard_connect_timeout();
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (state->state != GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD)
+      continue;
+    if (state->state_set_at <= state_set_at_cutoff)
+      continue;
+    if (circ_state_has_higher_priority(circ, rst_on_best_waiting,
+                                       best_waiting_circuit))
+      ++n_blockers_found;
+  } SMARTLIST_FOREACH_END(circ);
+
+  if (n_blockers_found) {
+    log_debug(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+              "%d guard-stalled, but %d pending circuit(s) had higher "
+              "guard priority, so not upgrading.",
+              n_waiting, n_blockers_found);
+    goto no_change;
+  }
+
+  /* Okay. We have a best waiting circuit, and we aren't waiting for
+     anything better.  Add all circuits with that priority to the
+     list, and call them COMPLETE. */
+  int n_succeeded = 0;
+  SMARTLIST_FOREACH_BEGIN(all_circuits, origin_circuit_t *, circ) {
+    circuit_guard_state_t *state = origin_circuit_get_guard_state(circ);
+    if (BUG(state == NULL))
+      continue;
+    if (circ != best_waiting_circuit && rst_on_best_waiting) {
+      /* Can't upgrade other circ with same priority as best; might
+         be blocked. */
+      continue;
     }
-  SMARTLIST_FOREACH_END(bridge);
-  return NULL;
-}
+    if (state->state != GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD)
+      continue;
+    if (circ_state_has_higher_priority(best_waiting_circuit, NULL, circ))
+      continue;
 
-/** If we have a bridge configured whose digest matches <b>digest</b>, or a
- * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>,
- * return 1.  Else return 0. If <b>digest</b> is NULL, check for
- * address/port matches only. */
-int
-addr_is_a_configured_bridge(const tor_addr_t *addr,
-                                uint16_t port,
-                                const char *digest)
-{
-  tor_assert(addr);
-  return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0;
-}
+    state->state = GUARD_CIRC_STATE_COMPLETE;
+    state->state_set_at = approx_time();
+    smartlist_add(newly_complete_out, circ);
+    ++n_succeeded;
+  } SMARTLIST_FOREACH_END(circ);
 
-/** If we have a bridge configured whose digest matches
- * <b>ei->identity_digest</b>, or a bridge with no known digest whose address
- * matches <b>ei->addr</b>:<b>ei->port</b>, return 1.  Else return 0.
- * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */
-int
-extend_info_is_a_configured_bridge(const extend_info_t *ei)
-{
-  const char *digest = ei->onion_key ? ei->identity_digest : NULL;
-  return addr_is_a_configured_bridge(&ei->addr, ei->port, digest);
-}
+  log_info(LD_GUARD, "Considered upgrading guard-stalled circuits: found "
+           "%d guard-stalled, %d complete. %d of the guard-stalled "
+           "circuit(s) had high enough priority to upgrade.",
+           n_waiting, n_complete, n_succeeded);
 
-/** Wrapper around get_configured_bridge_by_addr_port_digest() to look
- * it up via router descriptor <b>ri</b>. */
-static bridge_info_t *
-get_configured_bridge_by_routerinfo(const routerinfo_t *ri)
-{
-  bridge_info_t *bi = NULL;
-  smartlist_t *orports = router_get_all_orports(ri);
-  bi = get_configured_bridge_by_orports_digest(ri->cache_info.identity_digest,
-                                               orports);
-  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
-  smartlist_free(orports);
-  return bi;
+  tor_assert_nonfatal(n_succeeded >= 1);
+  smartlist_free(all_circuits);
+  return 1;
+
+ no_change:
+  smartlist_free(all_circuits);
+  return 0;
 }
 
-/** Return 1 if <b>ri</b> is one of our known bridges, else 0. */
+/**
+ * Return true iff the circuit whose state is <b>guard_state</b> should
+ * expire.
+ */
 int
-routerinfo_is_a_configured_bridge(const routerinfo_t *ri)
+entry_guard_state_should_expire(circuit_guard_state_t *guard_state)
 {
-  return get_configured_bridge_by_routerinfo(ri) ? 1 : 0;
+  if (guard_state == NULL)
+    return 0;
+  const time_t expire_if_waiting_since =
+    approx_time() - get_nonprimary_guard_idle_timeout();
+  return (guard_state->state == GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD
+          && guard_state->state_set_at < expire_if_waiting_since);
 }
 
-/** Return 1 if <b>node</b> is one of our configured bridges, else 0. */
+/**
+ * Update all derived pieces of the guard selection state in <b>gs</b>.
+ * Return true iff we should stop using all previously generated circuits.
+ */
 int
-node_is_a_configured_bridge(const node_t *node)
+entry_guards_update_all(guard_selection_t *gs)
 {
-  int retval = 0;
-  smartlist_t *orports = node_get_all_orports(node);
-  retval = get_configured_bridge_by_orports_digest(node->identity,
-                                                   orports) != NULL;
-  SMARTLIST_FOREACH(orports, tor_addr_port_t *, p, tor_free(p));
-  smartlist_free(orports);
-  return retval;
+  sampled_guards_update_from_consensus(gs);
+  entry_guards_update_filtered_sets(gs);
+  entry_guards_update_confirmed(gs);
+  entry_guards_update_primary(gs);
+  return 0;
 }
 
-/** We made a connection to a router at <b>addr</b>:<b>port</b>
- * without knowing its digest. Its digest turned out to be <b>digest</b>.
- * If it was a bridge, and we still don't know its digest, record it.
+/**
+ * Return a newly allocated string for encoding the persistent parts of
+ * <b>guard</b> to the state file.
  */
-void
-learned_router_identity(const tor_addr_t *addr, uint16_t port,
-                        const char *digest)
-{
-  bridge_info_t *bridge =
-    get_configured_bridge_by_addr_port_digest(addr, port, digest);
-  if (bridge && tor_digest_is_zero(bridge->identity)) {
-    char *transport_info = NULL;
-    const char *transport_name =
-      find_transport_name_by_bridge_addrport(addr, port);
-    if (transport_name)
-      tor_asprintf(&transport_info, " (with transport '%s')", transport_name);
+STATIC char *
+entry_guard_encode_for_state(entry_guard_t *guard)
+{
+  /*
+   * The meta-format we use is K=V K=V K=V... where K can be any
+   * characters excepts space and =, and V can be any characters except
+   * space.  The order of entries is not allowed to matter.
+   * Unrecognized K=V entries are persisted; recognized but erroneous
+   * entries are corrected.
+   */
+
+  smartlist_t *result = smartlist_new();
+  char tbuf[ISO_TIME_LEN+1];
+
+  tor_assert(guard);
 
-    memcpy(bridge->identity, digest, DIGEST_LEN);
-    log_notice(LD_DIR, "Learned fingerprint %s for bridge %s%s.",
-               hex_str(digest, DIGEST_LEN), fmt_addrport(addr, port),
-               transport_info ? transport_info : "");
-    tor_free(transport_info);
+  smartlist_add_asprintf(result, "in=%s", guard->selection_name);
+  smartlist_add_asprintf(result, "rsa_id=%s",
+                         hex_str(guard->identity, DIGEST_LEN));
+  if (guard->bridge_addr) {
+    smartlist_add_asprintf(result, "bridge_addr=%s:%d",
+                           fmt_and_decorate_addr(&guard->bridge_addr->addr),
+                           guard->bridge_addr->port);
+  }
+  if (strlen(guard->nickname) && is_legal_nickname(guard->nickname)) {
+    smartlist_add_asprintf(result, "nickname=%s", guard->nickname);
   }
-}
 
-/** Return true if <b>bridge</b> has the same identity digest as
- *  <b>digest</b>. If <b>digest</b> is NULL, it matches
- *  bridges with unspecified identity digests. */
-static int
-bridge_has_digest(const bridge_info_t *bridge, const char *digest)
-{
-  if (digest)
-    return tor_memeq(digest, bridge->identity, DIGEST_LEN);
-  else
-    return tor_digest_is_zero(bridge->identity);
+  format_iso_time_nospace(tbuf, guard->sampled_on_date);
+  smartlist_add_asprintf(result, "sampled_on=%s", tbuf);
+
+  if (guard->sampled_by_version) {
+    smartlist_add_asprintf(result, "sampled_by=%s",
+                           guard->sampled_by_version);
+  }
+
+  if (guard->unlisted_since_date > 0) {
+    format_iso_time_nospace(tbuf, guard->unlisted_since_date);
+    smartlist_add_asprintf(result, "unlisted_since=%s", tbuf);
+  }
+
+  smartlist_add_asprintf(result, "listed=%d",
+                         (int)guard->currently_listed);
+
+  if (guard->confirmed_idx >= 0) {
+    format_iso_time_nospace(tbuf, guard->confirmed_on_date);
+    smartlist_add_asprintf(result, "confirmed_on=%s", tbuf);
+
+    smartlist_add_asprintf(result, "confirmed_idx=%d", guard->confirmed_idx);
+  }
+
+  const double EPSILON = 1.0e-6;
+
+  /* Make a copy of the pathbias object, since we will want to update
+     some of them */
+  guard_pathbias_t *pb = tor_memdup(&guard->pb, sizeof(*pb));
+  pb->use_successes = pathbias_get_use_success_count(guard);
+  pb->successful_circuits_closed = pathbias_get_close_success_count(guard);
+
+  #define PB_FIELD(field) do {                                          \
+      if (pb->field >= EPSILON) {                                       \
+        smartlist_add_asprintf(result, "pb_" #field "=%f", pb->field);  \
+      }                                                                 \
+    } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+  tor_free(pb);
+#undef PB_FIELD
+
+  if (guard->extra_state_fields)
+    smartlist_add_strdup(result, guard->extra_state_fields);
+
+  char *joined = smartlist_join_strings(result, " ", 0, NULL);
+  SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+  smartlist_free(result);
+
+  return joined;
 }
 
-/** We are about to add a new bridge at <b>addr</b>:<b>port</b>, with optional
- * <b>digest</b> and <b>transport_name</b>. Mark for removal any previously
- * existing bridge with the same address and port, and warn the user as
- * appropriate.
+/**
+ * Given a string generated by entry_guard_encode_for_state(), parse it
+ * (if possible) and return an entry_guard_t object for it.  Return NULL
+ * on complete failure.
  */
-static void
-bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port,
-                         const char *digest, const char *transport_name)
-{
-  /* Iterate the already-registered bridge list:
+STATIC entry_guard_t *
+entry_guard_parse_from_state(const char *s)
+{
+  /* Unrecognized entries get put in here. */
+  smartlist_t *extra = smartlist_new();
+
+  /* These fields get parsed from the string. */
+  char *in = NULL;
+  char *rsa_id = NULL;
+  char *nickname = NULL;
+  char *sampled_on = NULL;
+  char *sampled_by = NULL;
+  char *unlisted_since = NULL;
+  char *listed  = NULL;
+  char *confirmed_on = NULL;
+  char *confirmed_idx = NULL;
+  char *bridge_addr = NULL;
+
+  // pathbias
+  char *pb_use_attempts = NULL;
+  char *pb_use_successes = NULL;
+  char *pb_circ_attempts = NULL;
+  char *pb_circ_successes = NULL;
+  char *pb_successful_circuits_closed = NULL;
+  char *pb_collapsed_circuits = NULL;
+  char *pb_unusable_circuits = NULL;
+  char *pb_timeouts = NULL;
+
+  /* Split up the entries.  Put the ones we know about in strings and the
+   * rest in "extra". */
+  {
+    smartlist_t *entries = smartlist_new();
+
+    strmap_t *vals = strmap_new(); // Maps keyword to location
+#define FIELD(f) \
+    strmap_set(vals, #f, &f);
+    FIELD(in);
+    FIELD(rsa_id);
+    FIELD(nickname);
+    FIELD(sampled_on);
+    FIELD(sampled_by);
+    FIELD(unlisted_since);
+    FIELD(listed);
+    FIELD(confirmed_on);
+    FIELD(confirmed_idx);
+    FIELD(bridge_addr);
+    FIELD(pb_use_attempts);
+    FIELD(pb_use_successes);
+    FIELD(pb_circ_attempts);
+    FIELD(pb_circ_successes);
+    FIELD(pb_successful_circuits_closed);
+    FIELD(pb_collapsed_circuits);
+    FIELD(pb_unusable_circuits);
+    FIELD(pb_timeouts);
+#undef FIELD
+
+    smartlist_split_string(entries, s, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
+      const char *eq = strchr(entry, '=');
+      if (!eq) {
+        smartlist_add(extra, entry);
+        continue;
+      }
+      char *key = tor_strndup(entry, eq-entry);
+      char **target = strmap_get(vals, key);
+      if (target == NULL || *target != NULL) {
+        /* unrecognized or already set */
+        smartlist_add(extra, entry);
+        tor_free(key);
+        continue;
+      }
 
-     If you find a bridge with the same adress and port, mark it for
-     removal. It doesn't make sense to have two active bridges with
-     the same IP:PORT. If the bridge in question has a different
-     digest or transport than <b>digest</b>/<b>transport_name</b>,
-     it's probably a misconfiguration and we should warn the user.
-  */
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) {
-    if (bridge->marked_for_removal)
-      continue;
+      *target = tor_strdup(eq+1);
+      tor_free(key);
+      tor_free(entry);
+    } SMARTLIST_FOREACH_END(entry);
 
-    if (tor_addr_eq(&bridge->addr, addr) && (bridge->port == port)) {
-
-      bridge->marked_for_removal = 1;
-
-      if (!bridge_has_digest(bridge, digest) ||
-          strcmp_opt(bridge->transport_name, transport_name)) {
-        /* warn the user */
-        char *bridge_description_new, *bridge_description_old;
-        tor_asprintf(&bridge_description_new, "%s:%s:%s",
-                     fmt_addrport(addr, port),
-                     digest ? hex_str(digest, DIGEST_LEN) : "",
-                     transport_name ? transport_name : "");
-        tor_asprintf(&bridge_description_old, "%s:%s:%s",
-                     fmt_addrport(&bridge->addr, bridge->port),
-                     tor_digest_is_zero(bridge->identity) ?
-                     "" : hex_str(bridge->identity,DIGEST_LEN),
-                     bridge->transport_name ? bridge->transport_name : "");
-
-        log_warn(LD_GENERAL,"Tried to add bridge '%s', but we found a conflict"
-                 " with the already registered bridge '%s'. We will discard"
-                 " the old bridge and keep '%s'. If this is not what you"
-                 " wanted, please change your configuration file accordingly.",
-                 bridge_description_new, bridge_description_old,
-                 bridge_description_new);
-
-        tor_free(bridge_description_new);
-        tor_free(bridge_description_old);
-      }
-    }
-  } SMARTLIST_FOREACH_END(bridge);
-}
+    smartlist_free(entries);
+    strmap_free(vals, NULL);
+  }
 
-/** Return True if we have a bridge that uses a transport with name
- *  <b>transport_name</b>. */
-MOCK_IMPL(int,
-transport_is_needed, (const char *transport_name))
-{
-  if (!bridge_list)
-    return 0;
+  entry_guard_t *guard = tor_malloc_zero(sizeof(entry_guard_t));
+  guard->is_persistent = 1;
 
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (bridge->transport_name &&
-        !strcmp(bridge->transport_name, transport_name))
-      return 1;
-  } SMARTLIST_FOREACH_END(bridge);
+  if (in == NULL) {
+    log_warn(LD_CIRC, "Guard missing 'in' field");
+    goto err;
+  }
 
-  return 0;
-}
+  guard->selection_name = in;
+  in = NULL;
 
-/** Register the bridge information in <b>bridge_line</b> to the
- *  bridge subsystem. Steals reference of <b>bridge_line</b>. */
-void
-bridge_add_from_config(bridge_line_t *bridge_line)
-{
-  bridge_info_t *b;
+  if (rsa_id == NULL) {
+    log_warn(LD_CIRC, "Guard missing RSA ID field");
+    goto err;
+  }
 
-  { /* Log the bridge we are about to register: */
-    log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)",
-              fmt_addrport(&bridge_line->addr, bridge_line->port),
-              bridge_line->transport_name ?
-              bridge_line->transport_name : "no transport",
-              tor_digest_is_zero(bridge_line->digest) ?
-              "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN));
+  /* Process the identity and nickname. */
+  if (base16_decode(guard->identity, sizeof(guard->identity),
+                    rsa_id, strlen(rsa_id)) != DIGEST_LEN) {
+    log_warn(LD_CIRC, "Unable to decode guard identity %s", escaped(rsa_id));
+    goto err;
+  }
 
-    if (bridge_line->socks_args) { /* print socks arguments */
-      int i = 0;
+  if (nickname) {
+    strlcpy(guard->nickname, nickname, sizeof(guard->nickname));
+  } else {
+    guard->nickname[0]='$';
+    base16_encode(guard->nickname+1, sizeof(guard->nickname)-1,
+                  guard->identity, DIGEST_LEN);
+  }
 
-      tor_assert(smartlist_len(bridge_line->socks_args) > 0);
+  if (bridge_addr) {
+    tor_addr_port_t res;
+    memset(&res, 0, sizeof(res));
+    int r = tor_addr_port_parse(LOG_WARN, bridge_addr,
+                                &res.addr, &res.port, -1);
+    if (r == 0)
+      guard->bridge_addr = tor_memdup(&res, sizeof(res));
+    /* On error, we already warned. */
+  }
 
-      log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:",
-                smartlist_len(bridge_line->socks_args));
-      SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg,
-                        log_debug(LD_CONFIG, "%d: %s", ++i, arg));
+  /* Process the various time fields. */
+
+#define HANDLE_TIME(field) do {                                 \
+    if (field) {                                                \
+      int r = parse_iso_time_nospace(field, &field ## _time);   \
+      if (r < 0) {                                              \
+        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
+                 #field, escaped(field));                       \
+        field##_time = -1;                                      \
+      }                                                         \
+    }                                                           \
+  } while (0)
+
+  time_t sampled_on_time = 0;
+  time_t unlisted_since_time = 0;
+  time_t confirmed_on_time = 0;
+
+  HANDLE_TIME(sampled_on);
+  HANDLE_TIME(unlisted_since);
+  HANDLE_TIME(confirmed_on);
+
+  if (sampled_on_time <= 0)
+    sampled_on_time = approx_time();
+  if (unlisted_since_time < 0)
+    unlisted_since_time = 0;
+  if (confirmed_on_time < 0)
+    confirmed_on_time = 0;
+
+  #undef HANDLE_TIME
+
+  guard->sampled_on_date = sampled_on_time;
+  guard->unlisted_since_date = unlisted_since_time;
+  guard->confirmed_on_date = confirmed_on_time;
+
+  /* Take sampled_by_version verbatim. */
+  guard->sampled_by_version = sampled_by;
+  sampled_by = NULL; /* prevent free */
+
+  /* Listed is a boolean */
+  if (listed && strcmp(listed, "0"))
+    guard->currently_listed = 1;
+
+  /* The index is a nonnegative integer. */
+  guard->confirmed_idx = -1;
+  if (confirmed_idx) {
+    int ok=1;
+    long idx = tor_parse_long(confirmed_idx, 10, 0, INT_MAX, &ok, NULL);
+    if (! ok) {
+      log_warn(LD_GUARD, "Guard has invalid confirmed_idx %s",
+               escaped(confirmed_idx));
+    } else {
+      guard->confirmed_idx = (int)idx;
     }
   }
 
-  bridge_resolve_conflicts(&bridge_line->addr,
-                           bridge_line->port,
-                           bridge_line->digest,
-                           bridge_line->transport_name);
+  /* Anything we didn't recognize gets crammed together */
+  if (smartlist_len(extra) > 0) {
+    guard->extra_state_fields = smartlist_join_strings(extra, " ", 0, NULL);
+  }
 
-  b = tor_malloc_zero(sizeof(bridge_info_t));
-  tor_addr_copy(&b->addr, &bridge_line->addr);
-  b->port = bridge_line->port;
-  memcpy(b->identity, bridge_line->digest, DIGEST_LEN);
-  if (bridge_line->transport_name)
-    b->transport_name = bridge_line->transport_name;
-  b->fetch_status.schedule = DL_SCHED_BRIDGE;
-  b->fetch_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL;
-  b->socks_args = bridge_line->socks_args;
-  if (!bridge_list)
-    bridge_list = smartlist_new();
+  /* initialize non-persistent fields */
+  guard->is_reachable = GUARD_REACHABLE_MAYBE;
+
+#define PB_FIELD(field)                                                 \
+  do {                                                                  \
+    if (pb_ ## field) {                                                 \
+      int ok = 1;                                                       \
+      double r = tor_parse_double(pb_ ## field, 0.0, 1e9, &ok, NULL);   \
+      if (! ok) {                                                       \
+        log_warn(LD_CIRC, "Guard has invalid pb_%s %s",                 \
+                 #field, pb_ ## field);                                 \
+      } else {                                                          \
+        guard->pb.field = r;                                            \
+      }                                                                 \
+    }                                                                   \
+  } while (0)
+  PB_FIELD(use_attempts);
+  PB_FIELD(use_successes);
+  PB_FIELD(circ_attempts);
+  PB_FIELD(circ_successes);
+  PB_FIELD(successful_circuits_closed);
+  PB_FIELD(collapsed_circuits);
+  PB_FIELD(unusable_circuits);
+  PB_FIELD(timeouts);
+#undef PB_FIELD
+
+  pathbias_check_use_success_count(guard);
+  pathbias_check_close_success_count(guard);
+
+  /* We update everything on this guard later, after we've parsed
+   * everything.  */
+
+  goto done;
+
+ err:
+  // only consider it an error if the guard state was totally unparseable.
+  entry_guard_free(guard);
+  guard = NULL;
 
-  tor_free(bridge_line); /* Deallocate bridge_line now. */
+ done:
+  tor_free(in);
+  tor_free(rsa_id);
+  tor_free(nickname);
+  tor_free(sampled_on);
+  tor_free(sampled_by);
+  tor_free(unlisted_since);
+  tor_free(listed);
+  tor_free(confirmed_on);
+  tor_free(confirmed_idx);
+  tor_free(bridge_addr);
+  tor_free(pb_use_attempts);
+  tor_free(pb_use_successes);
+  tor_free(pb_circ_attempts);
+  tor_free(pb_circ_successes);
+  tor_free(pb_successful_circuits_closed);
+  tor_free(pb_collapsed_circuits);
+  tor_free(pb_unusable_circuits);
+  tor_free(pb_timeouts);
+
+  SMARTLIST_FOREACH(extra, char *, cp, tor_free(cp));
+  smartlist_free(extra);
+
+  return guard;
+}
 
-  smartlist_add(bridge_list, b);
+/**
+ * Replace the Guards entries in <b>state</b> with a list of all our sampled
+ * guards.
+ */
+static void
+entry_guards_update_guards_in_state(or_state_t *state)
+{
+  if (!guard_contexts)
+    return;
+  config_line_t *lines = NULL;
+  config_line_t **nextline = &lines;
+
+  SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+      if (guard->is_persistent == 0)
+        continue;
+      *nextline = tor_malloc_zero(sizeof(config_line_t));
+      (*nextline)->key = tor_strdup("Guard");
+      (*nextline)->value = entry_guard_encode_for_state(guard);
+      nextline = &(*nextline)->next;
+    } SMARTLIST_FOREACH_END(guard);
+  } SMARTLIST_FOREACH_END(gs);
+
+  config_free_lines(state->Guard);
+  state->Guard = lines;
 }
 
-/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+/**
+ * Replace our sampled guards from the Guards entries in <b>state</b>. Return 0
+ * on success, -1 on failure. (If <b>set</b> is true, replace nothing -- only
+ * check whether replacing would work.)
+ */
 static int
-routerset_contains_bridge(const routerset_t *routerset,
-                          const bridge_info_t *bridge)
+entry_guards_load_guards_from_state(or_state_t *state, int set)
+{
+  const config_line_t *line = state->Guard;
+  int n_errors = 0;
+
+  if (!guard_contexts)
+    guard_contexts = smartlist_new();
+
+  /* Wipe all our existing guard info. (we shouldn't have any, but
+   * let's be safe.) */
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+      if (curr_guard_context == gs)
+        curr_guard_context = NULL;
+      SMARTLIST_DEL_CURRENT(guard_contexts, gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+
+  for ( ; line != NULL; line = line->next) {
+    entry_guard_t *guard = entry_guard_parse_from_state(line->value);
+    if (guard == NULL) {
+      ++n_errors;
+      continue;
+    }
+    tor_assert(guard->selection_name);
+    if (!strcmp(guard->selection_name, "legacy")) {
+      ++n_errors;
+      entry_guard_free(guard);
+      continue;
+    }
+
+    if (set) {
+      guard_selection_t *gs;
+      gs = get_guard_selection_by_name(guard->selection_name,
+                                       GS_TYPE_INFER, 1);
+      tor_assert(gs);
+      smartlist_add(gs->sampled_entry_guards, guard);
+      guard->in_selection = gs;
+    } else {
+      entry_guard_free(guard);
+    }
+  }
+
+  if (set) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      entry_guards_update_all(gs);
+    } SMARTLIST_FOREACH_END(gs);
+  }
+  return n_errors ? -1 : 0;
+}
+
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the provided guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest_for_guard_selection(guard_selection_t *gs,
+                                                 const char *digest)
 {
-  int result;
-  extend_info_t *extinfo;
-  tor_assert(bridge);
-  if (!routerset)
-    return 0;
+  return get_sampled_guard_with_id(gs, (const uint8_t*)digest);
+}
 
-  extinfo = extend_info_new(
-         NULL, bridge->identity, NULL, NULL, &bridge->addr, bridge->port);
-  result = routerset_contains_extendinfo(routerset, extinfo);
-  extend_info_free(extinfo);
-  return result;
+/** Return the node_t associated with a single entry_guard_t. May
+ * return NULL if the guard is not currently in the consensus. */
+const node_t *
+entry_guard_find_node(const entry_guard_t *guard)
+{
+  tor_assert(guard);
+  return node_get_by_id(guard->identity);
 }
 
-/** If <b>digest</b> is one of our known bridges, return it. */
-static bridge_info_t *
-find_bridge_by_digest(const char *digest)
+/** If <b>digest</b> matches the identity of any node in the
+ * entry_guards list for the default guard selection state,
+ return that node. Else return NULL. */
+entry_guard_t *
+entry_guard_get_by_id_digest(const char *digest)
 {
-  SMARTLIST_FOREACH(bridge_list, bridge_info_t *, bridge,
-    {
-      if (tor_memeq(bridge->identity, digest, DIGEST_LEN))
-        return bridge;
-    });
-  return NULL;
+  return entry_guard_get_by_id_digest_for_guard_selection(
+      get_guard_selection_info(), digest);
 }
 
-/** Given the <b>addr</b> and <b>port</b> of a bridge, if that bridge
- *  supports a pluggable transport, return its name. Otherwise, return
- *  NULL. */
-const char *
-find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+/** We are about to connect to bridge with identity <b>digest</b> to fetch its
+ *  descriptor. Create a new guard state for this connection and return it. */
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest)
 {
-  if (!bridge_list)
+  circuit_guard_state_t *guard_state = NULL;
+  entry_guard_t *guard = NULL;
+
+  guard = entry_guard_get_by_id_digest_for_guard_selection(
+                                    get_guard_selection_info(), digest);
+  if (!guard) {
     return NULL;
+  }
 
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (tor_addr_eq(&bridge->addr, addr) &&
-        (bridge->port == port))
-      return bridge->transport_name;
-  } SMARTLIST_FOREACH_END(bridge);
+  /* Update the guard last_tried_to_connect time since it's checked by the
+   * guard susbsystem. */
+  guard->last_tried_to_connect = approx_time();
 
-  return NULL;
+  /* Create the guard state */
+  guard_state = circuit_guard_state_new(guard,
+                                        GUARD_CIRC_STATE_USABLE_ON_COMPLETION,
+                                        NULL);
+
+  return guard_state;
 }
 
-/** If <b>addr</b> and <b>port</b> match the address and port of a
- * bridge of ours that uses pluggable transports, place its transport
- * in <b>transport</b>.
- *
- * Return 0 on success (found a transport, or found a bridge with no
- * transport, or found no bridge); return -1 if we should be using a
- * transport, but the transport could not be found.
+/** Release all storage held by <b>e</b>. */
+STATIC void
+entry_guard_free(entry_guard_t *e)
+{
+  if (!e)
+    return;
+  entry_guard_handles_clear(e);
+  tor_free(e->sampled_by_version);
+  tor_free(e->extra_state_fields);
+  tor_free(e->selection_name);
+  tor_free(e->bridge_addr);
+  tor_free(e);
+}
+
+/** Return 0 if we're fine adding arbitrary routers out of the
+ * directory to our entry guard list, or return 1 if we have a
+ * list already and we must stick to it.
  */
 int
-get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
-                                  const transport_t **transport)
+entry_list_is_constrained(const or_options_t *options)
 {
-  *transport = NULL;
-  if (!bridge_list)
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) {
-    if (tor_addr_eq(&bridge->addr, addr) &&
-        (bridge->port == port)) { /* bridge matched */
-      if (bridge->transport_name) { /* it also uses pluggable transports */
-        *transport = transport_get_by_name(bridge->transport_name);
-        if (*transport == NULL) { /* it uses pluggable transports, but
-                                     the transport could not be found! */
-          return -1;
-        }
-        return 0;
-      } else { /* bridge matched, but it doesn't use transports. */
-        break;
-      }
-    }
-  } SMARTLIST_FOREACH_END(bridge);
-
-  *transport = NULL;
+  // XXXX #21425 look at the current selection.
+  if (options->EntryNodes)
+    return 1;
+  if (options->UseBridges)
+    return 1;
   return 0;
 }
 
-/** Return a smartlist containing all the SOCKS arguments that we
- *  should pass to the SOCKS proxy. */
-const smartlist_t *
-get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port)
+/** Return the number of bridges that have descriptors that are marked with
+ * purpose 'bridge' and are running.
+ */
+int
+num_bridges_usable(void)
 {
-  bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr,
-                                                                    port,
-                                                                    NULL);
-  return bridge ? bridge->socks_args : NULL;
+  int n_options = 0;
+
+  tor_assert(get_options()->UseBridges);
+  guard_selection_t *gs  = get_guard_selection_info();
+  tor_assert(gs->type == GS_TYPE_BRIDGE);
+
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    if (tor_digest_is_zero(guard->identity))
+      continue;
+    const node_t *node = node_get_by_id(guard->identity);
+    if (node && node->ri)
+      ++n_options;
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_options;
 }
 
-/** We need to ask <b>bridge</b> for its server descriptor. */
+/** Check the pathbias use success count of <b>node</b> and disable it if it
+ *  goes over our thresholds. */
 static void
-launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge)
+pathbias_check_use_success_count(entry_guard_t *node)
 {
   const or_options_t *options = get_options();
-
-  if (connection_get_by_type_addr_port_purpose(
-      CONN_TYPE_DIR, &bridge->addr, bridge->port,
-      DIR_PURPOSE_FETCH_SERVERDESC))
-    return; /* it's already on the way */
-
-  if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
-    download_status_mark_impossible(&bridge->fetch_status);
-    log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
-             safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
-    return;
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.use_attempts > EPSILON &&
+      pathbias_get_use_success_count(node)/node->pb.use_attempts
+      < pathbias_get_extreme_use_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path use bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
   }
+}
 
-  /* Until we get a descriptor for the bridge, we only know one address for
-   * it. */
-  if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
-                                            FIREWALL_OR_CONNECTION, 0, 0)) {
-    log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a "
-               "bridge, but that bridge is not reachable through our "
-               "firewall.");
-    return;
+/** Check the pathbias close count of <b>node</b> and disable it if it goes
+ *  over our thresholds. */
+static void
+pathbias_check_close_success_count(entry_guard_t *node)
+{
+  const or_options_t *options = get_options();
+  const double EPSILON = 1.0e-9;
+
+  /* Note: We rely on the < comparison here to allow us to set a 0
+   * rate and disable the feature entirely. If refactoring, don't
+   * change to <= */
+  if (node->pb.circ_attempts > EPSILON &&
+      pathbias_get_close_success_count(node)/node->pb.circ_attempts
+      < pathbias_get_extreme_rate(options) &&
+      pathbias_get_dropguards(options)) {
+    node->pb.path_bias_disabled = 1;
+    log_info(LD_GENERAL,
+             "Path bias is too high (%f/%f); disabling node %s",
+             node->pb.circ_successes, node->pb.circ_attempts,
+             node->nickname);
   }
-
-  directory_initiate_command(&bridge->addr, bridge->port,
-                             NULL, 0, /*no dirport*/
-                             bridge->identity,
-                             DIR_PURPOSE_FETCH_SERVERDESC,
-                             ROUTER_PURPOSE_BRIDGE,
-                             DIRIND_ONEHOP, "authority.z", NULL, 0, 0);
 }
 
-/** Fetching the bridge descriptor from the bridge authority returned a
- * "not found". Fall back to trying a direct fetch. */
-void
-retry_bridge_descriptor_fetch_directly(const char *digest)
+/** Parse <b>state</b> and learn about the entry guards it describes.
+ * If <b>set</b> is true, and there are no errors, replace the guard
+ * list in the default guard selection context with what we find.
+ * On success, return 0. On failure, alloc into *<b>msg</b> a string
+ * describing the error, and return -1.
+ */
+int
+entry_guards_parse_state(or_state_t *state, int set, char **msg)
 {
-  bridge_info_t *bridge = find_bridge_by_digest(digest);
-  if (!bridge)
-    return; /* not found? oh well. */
+  entry_guards_dirty = 0;
+  int r1 = entry_guards_load_guards_from_state(state, set);
+  entry_guards_dirty = 0;
 
-  launch_direct_bridge_descriptor_fetch(bridge);
+  if (r1 < 0) {
+    if (msg && *msg == NULL) {
+      *msg = tor_strdup("parsing error");
+    }
+    return -1;
+  }
+  return 0;
 }
 
-/** For each bridge in our list for which we don't currently have a
- * descriptor, fetch a new copy of its descriptor -- either directly
- * from the bridge or via a bridge authority. */
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are trying to avoid disk writes? */
+#define SLOW_GUARD_STATE_FLUSH_TIME 600
+/** How long will we let a change in our guard nodes stay un-saved
+ * when we are not trying to avoid disk writes? */
+#define FAST_GUARD_STATE_FLUSH_TIME 30
+
+/** Our list of entry guards has changed for a particular guard selection
+ * context, or some element of one of our entry guards has changed for one.
+ * Write the changes to disk within the next few minutes.
+ */
 void
-fetch_bridge_descriptors(const or_options_t *options, time_t now)
+entry_guards_changed_for_guard_selection(guard_selection_t *gs)
 {
-  int num_bridge_auths = get_n_authorities(BRIDGE_DIRINFO);
-  int ask_bridge_directly;
-  int can_use_bridge_authority;
+  time_t when;
 
-  if (!bridge_list)
-    return;
+  tor_assert(gs != NULL);
 
-  /* If we still have unconfigured managed proxies, don't go and
-     connect to a bridge. */
-  if (pt_proxies_configuration_pending())
-    return;
+  entry_guards_dirty = 1;
 
-  SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge)
-    {
-      if (!download_status_is_ready(&bridge->fetch_status, now,
-                                    IMPOSSIBLE_TO_DOWNLOAD))
-        continue; /* don't bother, no need to retry yet */
-      if (routerset_contains_bridge(options->ExcludeNodes, bridge)) {
-        download_status_mark_impossible(&bridge->fetch_status);
-        log_warn(LD_APP, "Not using bridge at %s: it is in ExcludeNodes.",
-                 safe_str_client(fmt_and_decorate_addr(&bridge->addr)));
-        continue;
-      }
+  if (get_options()->AvoidDiskWrites)
+    when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME;
+  else
+    when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME;
 
-      /* schedule another fetch as if this one will fail, in case it does */
-      download_status_failed(&bridge->fetch_status, 0);
-
-      can_use_bridge_authority = !tor_digest_is_zero(bridge->identity) &&
-                                 num_bridge_auths;
-      ask_bridge_directly = !can_use_bridge_authority ||
-                            !options->UpdateBridgesFromAuthority;
-      log_debug(LD_DIR, "ask_bridge_directly=%d (%d, %d, %d)",
-                ask_bridge_directly, tor_digest_is_zero(bridge->identity),
-                !options->UpdateBridgesFromAuthority, !num_bridge_auths);
-
-      if (ask_bridge_directly &&
-          !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port,
-                                                FIREWALL_OR_CONNECTION, 0,
-                                                0)) {
-        log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our "
-                   "firewall policy. %s.",
-                   fmt_addrport(&bridge->addr, bridge->port),
-                   can_use_bridge_authority ?
-                     "Asking bridge authority instead" : "Skipping");
-        if (can_use_bridge_authority)
-          ask_bridge_directly = 0;
-        else
-          continue;
-      }
+  /* or_state_save() will call entry_guards_update_state() and
+     entry_guards_update_guards_in_state()
+  */
+  or_state_mark_dirty(get_or_state(), when);
+}
 
-      if (ask_bridge_directly) {
-        /* we need to ask the bridge itself for its descriptor. */
-        launch_direct_bridge_descriptor_fetch(bridge);
-      } else {
-        /* We have a digest and we want to ask an authority. We could
-         * combine all the requests into one, but that may give more
-         * hints to the bridge authority than we want to give. */
-        char resource[10 + HEX_DIGEST_LEN];
-        memcpy(resource, "fp/", 3);
-        base16_encode(resource+3, HEX_DIGEST_LEN+1,
-                      bridge->identity, DIGEST_LEN);
-        memcpy(resource+3+HEX_DIGEST_LEN, ".z", 3);
-        log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.",
-                 resource);
-        directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC,
-                ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY);
-      }
-    }
-  SMARTLIST_FOREACH_END(bridge);
+/** Our list of entry guards has changed for the default guard selection
+ * context, or some element of one of our entry guards has changed. Write
+ * the changes to disk within the next few minutes.
+ */
+void
+entry_guards_changed(void)
+{
+  entry_guards_changed_for_guard_selection(get_guard_selection_info());
 }
 
-/** If our <b>bridge</b> is configured to be a different address than
- * the bridge gives in <b>node</b>, rewrite the routerinfo
- * we received to use the address we meant to use. Now we handle
- * multihomed bridges better.
+/** If the entry guard info has not changed, do nothing and return.
+ * Otherwise, free the EntryGuards piece of <b>state</b> and create
+ * a new one out of the global entry_guards list, and then mark
+ * <b>state</b> dirty so it will get saved to disk.
  */
-static void
-rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
+void
+entry_guards_update_state(or_state_t *state)
 {
-  /* XXXX move this function. */
-  /* XXXX overridden addresses should really live in the node_t, so that the
-   *   routerinfo_t and the microdesc_t can be immutable.  But we can only
-   *   do that safely if we know that no function that connects to an OR
-   *   does so through an address from any source other than node_get_addr().
-   */
-  tor_addr_t addr;
-  const or_options_t *options = get_options();
+  entry_guards_dirty = 0;
 
-  if (node->ri) {
-    routerinfo_t *ri = node->ri;
-    tor_addr_from_ipv4h(&addr, ri->addr);
+  // Handles all guard info.
+  entry_guards_update_guards_in_state(state);
 
-    if ((!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
-         bridge->port == ri->or_port) ||
-        (!tor_addr_compare(&bridge->addr, &ri->ipv6_addr, CMP_EXACT) &&
-         bridge->port == ri->ipv6_orport)) {
-      /* they match, so no need to do anything */
-    } else {
-      if (tor_addr_family(&bridge->addr) == AF_INET) {
-        ri->addr = tor_addr_to_ipv4h(&bridge->addr);
-        ri->or_port = bridge->port;
-        log_info(LD_DIR,
-                 "Adjusted bridge routerinfo for '%s' to match configured "
-                 "address %s:%d.",
-                 ri->nickname, fmt_addr32(ri->addr), ri->or_port);
-      } else if (tor_addr_family(&bridge->addr) == AF_INET6) {
-        tor_addr_copy(&ri->ipv6_addr, &bridge->addr);
-        ri->ipv6_orport = bridge->port;
-        log_info(LD_DIR,
-                 "Adjusted bridge routerinfo for '%s' to match configured "
-                 "address %s.",
-                 ri->nickname, fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
-      } else {
-        log_err(LD_BUG, "Address family not supported: %d.",
-                tor_addr_family(&bridge->addr));
-        return;
-      }
-    }
+  entry_guards_dirty = 0;
 
-    if (options->ClientPreferIPv6ORPort == -1) {
-      /* Mark which address to use based on which bridge_t we got. */
-      node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
-                              !tor_addr_is_null(&node->ri->ipv6_addr));
-    } else {
-      /* Mark which address to use based on user preference */
-      node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) &&
-                              !tor_addr_is_null(&node->ri->ipv6_addr));
-    }
+  if (!get_options()->AvoidDiskWrites)
+    or_state_mark_dirty(get_or_state(), 0);
+  entry_guards_dirty = 0;
+}
 
-    /* XXXipv6 we lack support for falling back to another address for
-       the same relay, warn the user */
-    if (!tor_addr_is_null(&ri->ipv6_addr)) {
-      tor_addr_port_t ap;
-      node_get_pref_orport(node, &ap);
-      log_notice(LD_CONFIG,
-                 "Bridge '%s' has both an IPv4 and an IPv6 address.  "
-                 "Will prefer using its %s address (%s) based on %s.",
-                 ri->nickname,
-                 node->ipv6_preferred ? "IPv6" : "IPv4",
-                 fmt_addrport(&ap.addr, ap.port),
-                 options->ClientPreferIPv6ORPort == -1 ?
-                 "the configured Bridge address" :
-                 "ClientPreferIPv6ORPort");
-    }
+/**
+ * Format a single entry guard in the format expected by the controller.
+ * Return a newly allocated string.
+ */
+STATIC char *
+getinfo_helper_format_single_entry_guard(const entry_guard_t *e)
+{
+  const char *status = NULL;
+  time_t when = 0;
+  const node_t *node;
+  char tbuf[ISO_TIME_LEN+1];
+  char nbuf[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  /* This is going to be a bit tricky, since the status
+   * codes weren't really intended for prop271 guards.
+   *
+   * XXXX use a more appropriate format for exporting this information
+   */
+  if (e->confirmed_idx < 0) {
+    status = "never-connected";
+  } else if (! e->currently_listed) {
+    when = e->unlisted_since_date;
+    status = "unusable";
+  } else if (! e->is_filtered_guard) {
+    status = "unusable";
+  } else if (e->is_reachable == GUARD_REACHABLE_NO) {
+    when = e->failing_since;
+    status = "down";
+  } else {
+    status = "up";
   }
-  if (node->rs) {
-    routerstatus_t *rs = node->rs;
-    tor_addr_from_ipv4h(&addr, rs->addr);
 
-    if (!tor_addr_compare(&bridge->addr, &addr, CMP_EXACT) &&
-        bridge->port == rs->or_port) {
-      /* they match, so no need to do anything */
-    } else {
-      rs->addr = tor_addr_to_ipv4h(&bridge->addr);
-      rs->or_port = bridge->port;
-      log_info(LD_DIR,
-               "Adjusted bridge routerstatus for '%s' to match "
-               "configured address %s.",
-               rs->nickname, fmt_addrport(&bridge->addr, rs->or_port));
-    }
+  node = entry_guard_find_node(e);
+  if (node) {
+    node_get_verbose_nickname(node, nbuf);
+  } else {
+    nbuf[0] = '$';
+    base16_encode(nbuf+1, sizeof(nbuf)-1, e->identity, DIGEST_LEN);
+    /* e->nickname field is not very reliable if we don't know about
+     * this router any longer; don't include it. */
   }
-}
 
-/** We just learned a descriptor for a bridge. See if that
- * digest is in our entry guard list, and add it if not. */
-void
-learned_bridge_descriptor(routerinfo_t *ri, int from_cache)
-{
-  tor_assert(ri);
-  tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE);
-  if (get_options()->UseBridges) {
-    int first = num_bridges_usable() <= 1;
-    bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri);
-    time_t now = time(NULL);
-    router_set_status(ri->cache_info.identity_digest, 1);
-
-    if (bridge) { /* if we actually want to use this one */
-      node_t *node;
-      /* it's here; schedule its re-fetch for a long time from now. */
-      if (!from_cache)
-        download_status_reset(&bridge->fetch_status);
-
-      node = node_get_mutable_by_id(ri->cache_info.identity_digest);
-      tor_assert(node);
-      rewrite_node_address_for_bridge(bridge, node);
-      if (tor_digest_is_zero(bridge->identity)) {
-        memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN);
-        log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d",
-                   hex_str(bridge->identity, DIGEST_LEN),
-                   fmt_and_decorate_addr(&bridge->addr),
-                   (int) bridge->port);
-      }
-      add_an_entry_guard(node, 1, 1, 0, 0);
-
-      log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname,
-                 from_cache ? "cached" : "fresh", router_describe(ri));
-      /* set entry->made_contact so if it goes down we don't drop it from
-       * our entry node list */
-      entry_guard_register_connect_status(ri->cache_info.identity_digest,
-                                          1, 0, now);
-      if (first) {
-        routerlist_retry_directory_downloads(now);
-      }
-    }
+  char *result = NULL;
+  if (when) {
+    format_iso_time(tbuf, when);
+    tor_asprintf(&result, "%s %s %s\n", nbuf, status, tbuf);
+  } else {
+    tor_asprintf(&result, "%s %s\n", nbuf, status);
   }
+  return result;
 }
 
-/** Return the number of bridges that have descriptors that
- * are marked with purpose 'bridge' and are running.
+/** If <b>question</b> is the string "entry-guards", then dump
+ * to *<b>answer</b> a newly allocated string describing all of
+ * the nodes in the global entry_guards list. See control-spec.txt
+ * for details.
+ * For backward compatibility, we also handle the string "helper-nodes".
  *
- * We use this function to decide if we're ready to start building
- * circuits through our bridges, or if we need to wait until the
- * directory "server/authority" requests finish. */
+ * XXX this should be totally redesigned after prop 271 too, and that's
+ * going to take some control spec work.
+ * */
 int
-any_bridge_descriptors_known(void)
+getinfo_helper_entry_guards(control_connection_t *conn,
+                            const char *question, char **answer,
+                            const char **errmsg)
 {
-  tor_assert(get_options()->UseBridges);
-  return choose_random_entry(NULL) != NULL;
+  guard_selection_t *gs = get_guard_selection_info();
+
+  tor_assert(gs != NULL);
+
+  (void) conn;
+  (void) errmsg;
+
+  if (!strcmp(question,"entry-guards") ||
+      !strcmp(question,"helper-nodes")) {
+    const smartlist_t *guards;
+    guards = gs->sampled_entry_guards;
+
+    smartlist_t *sl = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(guards, const entry_guard_t *, e) {
+      char *cp = getinfo_helper_format_single_entry_guard(e);
+      smartlist_add(sl, cp);
+    } SMARTLIST_FOREACH_END(e);
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  }
+  return 0;
 }
 
-/** Return the number of bridges that have descriptors that are marked with
- * purpose 'bridge' and are running.
+/* Given the original bandwidth of a guard and its guardfraction,
+ * calculate how much bandwidth the guard should have as a guard and
+ * as a non-guard.
+ *
+ * Quoting from proposal236:
+ *
+ *   Let Wpf denote the weight from the 'bandwidth-weights' line a
+ *   client would apply to N for position p if it had the guard
+ *   flag, Wpn the weight if it did not have the guard flag, and B the
+ *   measured bandwidth of N in the consensus.  Then instead of choosing
+ *   N for position p proportionally to Wpf*B or Wpn*B, clients should
+ *   choose N proportionally to F*Wpf*B + (1-F)*Wpn*B.
+ *
+ * This function fills the <b>guardfraction_bw</b> structure. It sets
+ * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B.
  */
-static int
-num_bridges_usable(void)
+void
+guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
+                                  int orig_bandwidth,
+                                  uint32_t guardfraction_percentage)
 {
-  int n_options = 0;
-  tor_assert(get_options()->UseBridges);
-  (void) choose_random_entry_impl(NULL, 0, 0, &n_options);
-  return n_options;
+  double guardfraction_fraction;
+
+  /* Turn the percentage into a fraction. */
+  tor_assert(guardfraction_percentage <= 100);
+  guardfraction_fraction = guardfraction_percentage / 100.0;
+
+  long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth);
+  tor_assert(guard_bw <= INT_MAX);
+
+  guardfraction_bw->guard_bw = (int) guard_bw;
+
+  guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw;
 }
 
-/** Return a smartlist containing all bridge identity digests */
-MOCK_IMPL(smartlist_t *,
-list_bridge_identities, (void))
+/** Helper: Update the status of all entry guards, in whatever algorithm
+ * is used. Return true if we should stop using all previously generated
+ * circuits, by calling circuit_mark_all_unused_circs() and
+ * circuit_mark_all_dirty_circs_as_unusable().
+ */
+int
+guards_update_all(void)
 {
-  smartlist_t *result = NULL;
-  char *digest_tmp;
+  int mark_circuits = 0;
+  if (update_guard_selection_choice(get_options()))
+    mark_circuits = 1;
 
-  if (get_options()->UseBridges && bridge_list) {
-    result = smartlist_new();
+  tor_assert(curr_guard_context);
 
-    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-      digest_tmp = tor_malloc(DIGEST_LEN);
-      memcpy(digest_tmp, b->identity, DIGEST_LEN);
-      smartlist_add(result, digest_tmp);
-    } SMARTLIST_FOREACH_END(b);
-  }
+  if (entry_guards_update_all(curr_guard_context))
+    mark_circuits = 1;
 
-  return result;
+  return mark_circuits;
+}
+
+/** Helper: pick a guard for a circuit, with whatever algorithm is
+    used. */
+const node_t *
+guards_choose_guard(cpath_build_state_t *state,
+                   circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  const uint8_t *exit_id = NULL;
+  entry_guard_restriction_t *rst = NULL;
+  if (state && (exit_id = build_state_get_exit_rsa_id(state))) {
+    /* We're building to a targeted exit node, so that node can't be
+     * chosen as our guard for this circuit.  Remember that fact in a
+     * restriction. */
+    rst = guard_create_exit_restriction(exit_id);
+    tor_assert(rst);
+  }
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_TRAFFIC,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
 }
 
-/** Get the download status for a bridge descriptor given its identity */
-MOCK_IMPL(download_status_t *,
-get_bridge_dl_status_by_id, (const char *digest))
+/** Remove all currently listed entry guards for a given guard selection
+ * context.  This frees and replaces <b>gs</b>, so don't use <b>gs</b>
+ * after calling this function. */
+void
+remove_all_entry_guards_for_guard_selection(guard_selection_t *gs)
 {
-  download_status_t *dl = NULL;
+  // This function shouldn't exist. XXXX
+  tor_assert(gs != NULL);
+  char *old_name = tor_strdup(gs->name);
+  guard_selection_type_t old_type = gs->type;
 
-  if (digest && get_options()->UseBridges && bridge_list) {
-    SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) {
-      if (tor_memeq(digest, b->identity, DIGEST_LEN)) {
-        dl = &(b->fetch_status);
-        break;
-      }
-    } SMARTLIST_FOREACH_END(b);
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, entry, {
+    control_event_guard(entry->nickname, entry->identity, "DROPPED");
+  });
+
+  if (gs == curr_guard_context) {
+    curr_guard_context = NULL;
   }
 
-  return dl;
+  smartlist_remove(guard_contexts, gs);
+  guard_selection_free(gs);
+
+  gs = get_guard_selection_by_name(old_name, old_type, 1);
+  entry_guards_changed_for_guard_selection(gs);
+  tor_free(old_name);
 }
 
-/** Return 1 if we have at least one descriptor for an entry guard
- * (bridge or member of EntryNodes) and all descriptors we know are
- * down. Else return 0. If <b>act</b> is 1, then mark the down guards
- * up; else just observe and report. */
-static int
-entries_retry_helper(const or_options_t *options, int act)
+/** Remove all currently listed entry guards, so new ones will be chosen.
+ *
+ * XXXX This function shouldn't exist -- it's meant to support the DROPGUARDS
+ * command, which is deprecated.
+ */
+void
+remove_all_entry_guards(void)
 {
-  const node_t *node;
-  int any_known = 0;
-  int any_running = 0;
-  int need_bridges = options->UseBridges != 0;
-  if (!entry_guards)
-    entry_guards = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-      node = node_get_by_id(e->identity);
-      if (node && node_has_descriptor(node) &&
-          node_is_bridge(node) == need_bridges &&
-          (!need_bridges || (!e->bad_since &&
-                             node_is_a_configured_bridge(node)))) {
-        any_known = 1;
-        if (node->is_running)
-          any_running = 1; /* some entry is both known and running */
-        else if (act) {
-          /* Mark all current connections to this OR as unhealthy, since
-           * otherwise there could be one that started 30 seconds
-           * ago, and in 30 seconds it will time out, causing us to mark
-           * the node down and undermine the retry attempt. We mark even
-           * the established conns, since if the network just came back
-           * we'll want to attach circuits to fresh conns. */
-          connection_or_set_bad_connections(node->identity, 1);
-
-          /* mark this entry node for retry */
-          router_set_status(node->identity, 1);
-          e->can_retry = 1;
-          e->bad_since = 0;
-        }
-      }
-  } SMARTLIST_FOREACH_END(e);
-  log_debug(LD_DIR, "%d: any_known %d, any_running %d",
-            act, any_known, any_running);
-  return any_known && !any_running;
+  remove_all_entry_guards_for_guard_selection(get_guard_selection_info());
 }
 
-/** Do we know any descriptors for our bridges / entrynodes, and are
- * all the ones we have descriptors for down? */
+/** Helper: pick a directory guard, with whatever algorithm is used. */
+const node_t *
+guards_choose_dirguard(uint8_t dir_purpose,
+                       circuit_guard_state_t **guard_state_out)
+{
+  const node_t *r = NULL;
+  entry_guard_restriction_t *rst = NULL;
+
+  /* If we are fetching microdescs, don't query outdated dirservers. */
+  if (dir_purpose == DIR_PURPOSE_FETCH_MICRODESC) {
+    rst = guard_create_dirserver_md_restriction();
+  }
+
+  if (entry_guard_pick_for_circuit(get_guard_selection_info(),
+                                   GUARD_USAGE_DIRGUARD,
+                                   rst,
+                                   &r,
+                                   guard_state_out) < 0) {
+    tor_assert(r == NULL);
+  }
+  return r;
+}
+
+/**
+ * If we're running with a constrained guard set, then maybe mark our guards
+ * usable.  Return 1 if we do; 0 if we don't.
+ */
 int
-entries_known_but_down(const or_options_t *options)
+guards_retry_optimistic(const or_options_t *options)
 {
-  tor_assert(entry_list_is_constrained(options));
-  return entries_retry_helper(options, 0);
+  if (! entry_list_is_constrained(options))
+    return 0;
+
+  mark_primary_guards_maybe_reachable(get_guard_selection_info());
+
+  return 1;
 }
 
-/** Mark all down known bridges / entrynodes up. */
-void
-entries_retry_all(const or_options_t *options)
+/**
+ * Return true iff we know enough directory information to construct
+ * circuits through all of the primary guards we'd currently use.
+ */
+int
+guard_selection_have_enough_dir_info_to_build_circuits(guard_selection_t *gs)
 {
-  tor_assert(entry_list_is_constrained(options));
-  entries_retry_helper(options, 1);
+  if (!gs->primary_guards_up_to_date)
+    entry_guards_update_primary(gs);
+
+  int n_missing_descriptors = 0;
+  int n_considered = 0;
+  int num_primary_to_check;
+
+  /* We want to check for the descriptor of at least the first two primary
+   * guards in our list, since these are the guards that we typically use for
+   * circuits. */
+  num_primary_to_check = get_n_primary_guards_to_use(GUARD_USAGE_TRAFFIC);
+  num_primary_to_check++;
+
+  SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
+    entry_guard_consider_retry(guard);
+    if (guard->is_reachable == GUARD_REACHABLE_NO)
+      continue;
+    n_considered++;
+    if (!guard_has_descriptor(guard))
+      n_missing_descriptors++;
+    if (n_considered >= num_primary_to_check)
+      break;
+  } SMARTLIST_FOREACH_END(guard);
+
+  return n_missing_descriptors == 0;
 }
 
-/** Return true if at least one of our bridges runs a Tor version that can
- * provide microdescriptors to us. If not, we'll fall back to asking for
- * full descriptors. */
+/** As guard_selection_have_enough_dir_info_to_build_circuits, but uses
+ * the default guard selection. */
 int
-any_bridge_supports_microdescriptors(void)
+entry_guards_have_enough_dir_info_to_build_circuits(void)
 {
-  const node_t *node;
-  if (!get_options()->UseBridges || !entry_guards)
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) {
-    node = node_get_by_id(e->identity);
-    if (node && node->is_running &&
-        node_is_bridge(node) && node_is_a_configured_bridge(node)) {
-      /* This is one of our current bridges, and we know enough about
-       * it to know that it will be able to answer our questions. */
-       return 1;
-    }
-  } SMARTLIST_FOREACH_END(e);
-  return 0;
+  return guard_selection_have_enough_dir_info_to_build_circuits(
+                                        get_guard_selection_info());
+}
+
+/** Free one guard selection context */
+STATIC void
+guard_selection_free(guard_selection_t *gs)
+{
+  if (!gs) return;
+
+  tor_free(gs->name);
+
+  if (gs->sampled_entry_guards) {
+    SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, e,
+                      entry_guard_free(e));
+    smartlist_free(gs->sampled_entry_guards);
+    gs->sampled_entry_guards = NULL;
+  }
+
+  smartlist_free(gs->confirmed_entry_guards);
+  smartlist_free(gs->primary_entry_guards);
+
+  tor_free(gs);
 }
 
 /** Release all storage held by the list of entry guards and related
@@ -2547,15 +3606,16 @@ any_bridge_supports_microdescriptors(void)
 void
 entry_guards_free_all(void)
 {
-  if (entry_guards) {
-    SMARTLIST_FOREACH(entry_guards, entry_guard_t *, e,
-                      entry_guard_free(e));
-    smartlist_free(entry_guards);
-    entry_guards = NULL;
+  /* Null out the default */
+  curr_guard_context = NULL;
+  /* Free all the guard contexts */
+  if (guard_contexts != NULL) {
+    SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      guard_selection_free(gs);
+    } SMARTLIST_FOREACH_END(gs);
+    smartlist_free(guard_contexts);
+    guard_contexts = NULL;
   }
-  clear_bridge_list();
-  smartlist_free(bridge_list);
-  bridge_list = NULL;
   circuit_build_times_free_timeouts(get_circuit_build_times_mutable());
 }
 
diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h
index 1021e67d43..39bff14381 100644
--- a/src/or/entrynodes.h
+++ b/src/or/entrynodes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,25 +12,27 @@
 #ifndef TOR_ENTRYNODES_H
 #define TOR_ENTRYNODES_H
 
-#if 1
-/* XXXX NM I would prefer that all of this stuff be private to
- * entrynodes.c. */
+#include "handles.h"
 
-/** An entry_guard_t represents our information about a chosen long-term
- * first hop, known as a "helper" node in the literature. We can't just
- * use a node_t, since we want to remember these even when we
- * don't have any directory info. */
-typedef struct entry_guard_t {
-  char nickname[MAX_NICKNAME_LEN+1];
-  char identity[DIGEST_LEN];
-  time_t chosen_on_date; /**< Approximately when was this guard added?
-                          * "0" if we don't know. */
-  char *chosen_by_version; /**< What tor version added this guard? NULL
-                            * if we don't know. */
-  unsigned int made_contact : 1; /**< 0 if we have never connected to this
-                                  * router, 1 if we have. */
-  unsigned int can_retry : 1; /**< Should we retry connecting to this entry,
-                               * in spite of having it marked as unreachable?*/
+/* Forward declare for guard_selection_t; entrynodes.c has the real struct */
+typedef struct guard_selection_s guard_selection_t;
+
+/* Forward declare for entry_guard_t; the real declaration is private. */
+typedef struct entry_guard_t entry_guard_t;
+
+/* Forward declaration for circuit_guard_state_t; the real declaration is
+   private. */
+typedef struct circuit_guard_state_t circuit_guard_state_t;
+
+/* Forward declaration for entry_guard_restriction_t; the real declaration is
+   private. */
+typedef struct entry_guard_restriction_t entry_guard_restriction_t;
+
+/* Information about a guard's pathbias status.
+ * These fields are used in circpathbias.c to try to detect entry
+ * nodes that are failing circuits at a suspicious frequency.
+ */
+typedef struct guard_pathbias_t {
   unsigned int path_bias_noticed : 1; /**< Did we alert the user about path
                                        * bias for this node already? */
   unsigned int path_bias_warned : 1; /**< Did we alert the user about path bias
@@ -43,15 +45,6 @@ typedef struct entry_guard_t {
                                        * use bias for this node already? */
   unsigned int path_bias_use_extreme : 1; /**< Did we alert the user about path
                                        * use bias for this node already? */
-  unsigned int is_dir_cache : 1; /**< Is this node a directory cache? */
-  time_t bad_since; /**< 0 if this guard is currently usable, or the time at
-                      * which it was observed to become (according to the
-                      * directory or the user configuration) unusable. */
-  time_t unreachable_since; /**< 0 if we can connect to this guard, or the
-                             * time at which we first noticed we couldn't
-                             * connect to it. */
-  time_t last_attempted; /**< 0 if we can connect to this guard, or the time
-                          * at which we last failed to connect to it. */
 
   double circ_attempts; /**< Number of circuits this guard has "attempted" */
   double circ_successes; /**< Number of successfully built circuits using
@@ -68,98 +61,540 @@ typedef struct entry_guard_t {
   double use_attempts; /**< Number of circuits we tried to use with streams */
   double use_successes; /**< Number of successfully used circuits using
                                * this guard as first hop. */
-} entry_guard_t;
+} guard_pathbias_t;
+
+#if defined(ENTRYNODES_PRIVATE)
+/**
+ * @name values for entry_guard_t.is_reachable.
+ *
+ * See entry_guard_t.is_reachable for more information.
+ */
+/**@{*/
+#define GUARD_REACHABLE_NO    0
+#define GUARD_REACHABLE_YES   1
+#define GUARD_REACHABLE_MAYBE 2
+/**@}*/
+
+/** An entry_guard_t represents our information about a chosen long-term
+ * first hop, known as a "helper" node in the literature. We can't just
+ * use a node_t, since we want to remember these even when we
+ * don't have any directory info. */
+struct entry_guard_t {
+  HANDLE_ENTRY(entry_guard, entry_guard_t);
+
+  char nickname[MAX_HEX_NICKNAME_LEN+1];
+  char identity[DIGEST_LEN];
+  ed25519_public_key_t ed_id;
+
+  /**
+   * @name new guard selection algorithm fields.
+   *
+   * Only the new (prop271) algorithm uses these.  For a more full
+   * description of the algorithm, see the module documentation for
+   * entrynodes.c
+   */
+  /**@{*/
+
+  /* == Persistent fields, present for all sampled guards. */
+  /** When was this guard added to the sample? */
+  time_t sampled_on_date;
+  /** Since what date has this guard been "unlisted"?  A guard counts as
+   * unlisted if we have a live consensus that does not include it, or
+   * if we have a live consensus that does not include it as a usable
+   * guard.  This field is zero when the guard is listed. */
+  time_t unlisted_since_date; // can be zero
+  /** What version of Tor added this guard to the sample? */
+  char *sampled_by_version;
+  /** Is this guard listed right now? If this is set, then
+   * unlisted_since_date should be set too. */
+  unsigned currently_listed : 1;
+
+  /* == Persistent fields, for confirmed guards only */
+  /** When was this guard confirmed? (That is, when did we first use it
+   * successfully and decide to keep it?) This field is zero if this is not a
+   * confirmed guard. */
+  time_t confirmed_on_date; /* 0 if not confirmed */
+  /**
+   * In what order was this guard confirmed? Guards with lower indices
+   * appear earlier on the confirmed list.  If the confirmed list is compacted,
+   * this field corresponds to the index of this guard on the confirmed list.
+   *
+   * This field is set to -1 if this guard is not confirmed.
+   */
+  int confirmed_idx; /* -1 if not confirmed; otherwise the order that this
+                      * item should occur in the CONFIRMED_GUARDS ordered
+                      * list */
+
+  /**
+   * Which selection does this guard belong to?
+   */
+  char *selection_name;
 
+  /** Bridges only: address of the bridge. */
+  tor_addr_port_t *bridge_addr;
+
+  /* ==== Non-persistent fields. */
+  /* == These are used by sampled guards */
+  /** When did we last decide to try using this guard for a circuit? 0 for
+   * "not since we started up." */
+  time_t last_tried_to_connect;
+  /** How reachable do we consider this guard to be? One of
+   * GUARD_REACHABLE_NO, GUARD_REACHABLE_YES, or GUARD_REACHABLE_MAYBE. */
+  unsigned is_reachable : 2;
+  /** Boolean: true iff this guard is pending. A pending guard is one
+   * that we have an in-progress circuit through, and which we do not plan
+   * to try again until it either succeeds or fails. Primary guards can
+   * never be pending. */
+  unsigned is_pending : 1;
+  /** If true, don't write this guard to disk. (Used for bridges with unknown
+   * identities) */
+  unsigned is_persistent : 1;
+  /** When did we get the earliest connection failure for this guard?
+   * We clear this field on a successful connect.  We do _not_ clear it
+   * when we mark the guard as "MAYBE" reachable.
+   */
+  time_t failing_since;
+
+  /* == Set inclusion flags. */
+  /** If true, this guard is in the filtered set.  The filtered set includes
+   * all sampled guards that our configuration allows us to use. */
+  unsigned is_filtered_guard : 1;
+  /** If true, this guard is in the usable filtered set. The usable filtered
+   * set includes all filtered guards that are not believed to be
+   * unreachable. (That is, those for which is_reachable is not
+   * GUARD_REACHABLE_NO) */
+  unsigned is_usable_filtered_guard : 1;
+  unsigned is_primary:1;
+
+  /** This string holds any fields that we are maintaining because
+   * we saw them in the state, even if we don't understand them. */
+  char *extra_state_fields;
+
+  /** Backpointer to the guard selection that this guard belongs to.
+   * The entry_guard_t must never outlive its guard_selection. */
+  guard_selection_t *in_selection;
+  /**@}*/
+
+  /** Path bias information for this guard. */
+  guard_pathbias_t pb;
+};
+
+/**
+ * Possible rules for a guard selection to follow
+ */
+typedef enum guard_selection_type_t {
+  /** Infer the type of this selection from its name. */
+  GS_TYPE_INFER=0,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * complete list of guards in the consensus. */
+  GS_TYPE_NORMAL=1,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * configured bridges, and allowing it to grow as large as all the configured
+   * bridges */
+  GS_TYPE_BRIDGE,
+  /** Use the normal guard selection algorithm, taking our sample from the
+   * set of filtered nodes. */
+  GS_TYPE_RESTRICTED,
+} guard_selection_type_t;
+
+/**
+ * All of the the context for guard selection on a particular client.
+ *
+ * We maintain multiple guard selection contexts for a client, depending
+ * aspects on its current configuration -- whether an extremely
+ * restrictive EntryNodes is used, whether UseBridges is enabled, and so
+ * on.)
+ *
+ * See the module documentation for entrynodes.c for more information
+ * about guard selection algorithms.
+ */
+struct guard_selection_s {
+  /**
+   * The name for this guard-selection object. (Must not contain spaces).
+   */
+  char *name;
+
+  /**
+   * What rules does this guard-selection object follow?
+   */
+  guard_selection_type_t type;
+
+  /**
+   * A value of 1 means that primary_entry_guards is up-to-date; 0
+   * means we need to recalculate it before using primary_entry_guards
+   * or the is_primary flag on any guard.
+   */
+  int primary_guards_up_to_date;
+
+  /**
+   * A list of the sampled entry guards, as entry_guard_t structures.
+   * Not in any particular order.  When we 'sample' a guard, we are
+   * noting it as a possible guard to pick in the future. The use of
+   * sampling here prevents us from being forced by an attacker to try
+   * every guard on the network. This list is persistent.
+   */
+  smartlist_t *sampled_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * have successfully contacted and decided to use. Every member of
+   * this list is a member of sampled_entry_guards. Every member should
+   * have confirmed_on_date set, and have confirmed_idx greater than
+   * any earlier member of the list.
+   *
+   * This list is persistent. It is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *confirmed_entry_guards;
+
+  /**
+   * Ordered list (from highest to lowest priority) of guards that we
+   * are willing to use the most happily.  These guards may or may not
+   * yet be confirmed yet.  If we can use one of these guards, we are
+   * probably not on a network that is trying to restrict our guard
+   * choices.
+   *
+   * This list is a subset of the elements in
+   * sampled_entry_guards, and its pointers point to elements of
+   * sampled_entry_guards.
+   */
+  smartlist_t *primary_entry_guards;
+
+  /** When did we last successfully build a circuit or use a circuit? */
+  time_t last_time_on_internet;
+
+  /** What confirmed_idx value should the next-added member of
+   * confirmed_entry_guards receive? */
+  int next_confirmed_idx;
+
+};
+
+struct entry_guard_handle_t;
+
+/** Types of restrictions we impose when picking guard nodes */
+typedef enum guard_restriction_type_t {
+  /* Don't pick the same guard node as our exit node (or its family) */
+  RST_EXIT_NODE = 0,
+  /* Don't pick dirguards that have previously shown to be outdated */
+  RST_OUTDATED_MD_DIRSERVER = 1
+} guard_restriction_type_t;
+
+/**
+ * A restriction to remember which entry guards are off-limits for a given
+ * circuit.
+ *
+ * Note: This mechanism is NOT for recording which guards are never to be
+ * used: only which guards cannot be used on <em>one particular circuit</em>.
+ */
+struct entry_guard_restriction_t {
+  /* What type of restriction are we imposing? */
+  guard_restriction_type_t type;
+
+  /* In case of restriction type RST_EXIT_NODE, the guard's RSA identity
+   * digest must not equal this; and it must not be in the same family as any
+   * node with this digest. */
+  uint8_t exclude_id[DIGEST_LEN];
+};
+
+/**
+ * Per-circuit state to track whether we'll be able to use the circuit.
+ */
+struct circuit_guard_state_t {
+  /** Handle to the entry guard object for this circuit. */
+  struct entry_guard_handle_t *guard;
+  /** The time at which <b>state</b> last changed. */
+  time_t state_set_at;
+  /** One of GUARD_CIRC_STATE_* */
+  uint8_t state;
+
+  /**
+   * A set of restrictions that were placed on this guard when we selected it
+   * for this particular circuit.  We need to remember the restrictions here,
+   * since any guard that breaks these restrictions will not block this
+   * circuit from becoming COMPLETE.
+   */
+  entry_guard_restriction_t *restrictions;
+};
+#endif
+
+/* Common entry points for old and new guard code */
+int guards_update_all(void);
+const node_t *guards_choose_guard(cpath_build_state_t *state,
+                                  circuit_guard_state_t **guard_state_out);
+const node_t *guards_choose_dirguard(uint8_t dir_purpose,
+                                     circuit_guard_state_t **guard_state_out);
+
+#if 1
+/* XXXX NM I would prefer that all of this stuff be private to
+ * entrynodes.c. */
+entry_guard_t *entry_guard_get_by_id_digest_for_guard_selection(
+    guard_selection_t *gs, const char *digest);
 entry_guard_t *entry_guard_get_by_id_digest(const char *digest);
+
+circuit_guard_state_t *
+get_guard_state_for_bridge_desc_fetch(const char *digest);
+
+void entry_guards_changed_for_guard_selection(guard_selection_t *gs);
 void entry_guards_changed(void);
-const smartlist_t *get_entry_guards(void);
+guard_selection_t * get_guard_selection_info(void);
+int num_live_entry_guards_for_guard_selection(
+    guard_selection_t *gs,
+    int for_directory);
 int num_live_entry_guards(int for_directory);
-
 #endif
 
-#ifdef ENTRYNODES_PRIVATE
-STATIC const node_t *add_an_entry_guard(const node_t *chosen,
-                                        int reset_status, int prepend,
-                                        int for_discovery, int for_directory);
-
-STATIC int populate_live_entry_guards(smartlist_t *live_entry_guards,
-                                      const smartlist_t *all_entry_guards,
-                                      const node_t *chosen_exit,
-                                      dirinfo_type_t dirinfo_type,
-                                      int for_directory,
-                                      int need_uptime, int need_capacity);
-STATIC int decide_num_guards(const or_options_t *options, int for_directory);
-
-STATIC void entry_guards_set_from_config(const or_options_t *options);
-
-/** Flags to be passed to entry_is_live() to indicate what kind of
- * entry nodes we are looking for. */
+const node_t *entry_guard_find_node(const entry_guard_t *guard);
+const char *entry_guard_get_rsa_id_digest(const entry_guard_t *guard);
+const char *entry_guard_describe(const entry_guard_t *guard);
+guard_pathbias_t *entry_guard_get_pathbias_state(entry_guard_t *guard);
+
+/** Enum to specify how we're going to use a given guard, when we're picking
+ * one for immediate use. */
 typedef enum {
-  ENTRY_NEED_UPTIME = 1<<0,
-  ENTRY_NEED_CAPACITY = 1<<1,
-  ENTRY_ASSUME_REACHABLE = 1<<2,
-  ENTRY_NEED_DESCRIPTOR = 1<<3,
-} entry_is_live_flags_t;
+  GUARD_USAGE_TRAFFIC = 0,
+  GUARD_USAGE_DIRGUARD = 1
+} guard_usage_t;
 
-STATIC const node_t *entry_is_live(const entry_guard_t *e,
-                                   entry_is_live_flags_t flags,
-                                   const char **msg);
+void circuit_guard_state_free(circuit_guard_state_t *state);
+int entry_guard_pick_for_circuit(guard_selection_t *gs,
+                                 guard_usage_t usage,
+                                 entry_guard_restriction_t *rst,
+                                 const node_t **chosen_node_out,
+                                 circuit_guard_state_t **guard_state_out);
 
-STATIC int entry_is_time_to_retry(const entry_guard_t *e, time_t now);
+/* We just connected to an entry guard. What should we do with the circuit? */
+typedef enum {
+  GUARD_USABLE_NEVER = -1, /* Never use the circuit */
+  GUARD_MAYBE_USABLE_LATER = 0, /* Keep it. We might use it in the future */
+  GUARD_USABLE_NOW = 1, /* Use it right now */
+} guard_usable_t;
 
-#endif
+guard_usable_t entry_guard_succeeded(circuit_guard_state_t **guard_state_p);
+void entry_guard_failed(circuit_guard_state_t **guard_state_p);
+void entry_guard_cancel(circuit_guard_state_t **guard_state_p);
+void entry_guard_chan_failed(channel_t *chan);
+int entry_guards_update_all(guard_selection_t *gs);
+int entry_guards_upgrade_waiting_circuits(guard_selection_t *gs,
+                                          const smartlist_t *all_circuits,
+                                          smartlist_t *newly_complete_out);
+int entry_guard_state_should_expire(circuit_guard_state_t *guard_state);
+void entry_guards_note_internet_connectivity(guard_selection_t *gs);
+
+int update_guard_selection_choice(const or_options_t *options);
+
+/* Used by bridges.c only. */
+int num_bridges_usable(void);
+
+#ifdef ENTRYNODES_PRIVATE
+/**
+ * @name Default values for the parameters for the new (prop271) entry guard
+ * algorithm.
+ */
+/**@{*/
+/**
+ * We never let our sampled guard set grow larger than this percentage
+ * of the guards on the network.
+ */
+#define DFLT_MAX_SAMPLE_THRESHOLD_PERCENT 20
+/**
+ * We never let our sampled guard set grow larger than this number of
+ * guards.
+ */
+#define DFLT_MAX_SAMPLE_SIZE 60
+/**
+ * We always try to make our sample contain at least this many guards.
+ */
+#define DFLT_MIN_FILTERED_SAMPLE_SIZE 20
+/**
+ * If a guard is unlisted for this many days in a row, we remove it.
+ */
+#define DFLT_REMOVE_UNLISTED_GUARDS_AFTER_DAYS 20
+/**
+ * We remove unconfirmed guards from the sample after this many days,
+ * regardless of whether they are listed or unlisted.
+ */
+#define DFLT_GUARD_LIFETIME_DAYS 120
+/**
+ * We remove confirmed guards from the sample if they were sampled
+ * GUARD_LIFETIME_DAYS ago and confirmed this many days ago.
+ */
+#define DFLT_GUARD_CONFIRMED_MIN_LIFETIME_DAYS 60
+/**
+ * How many guards do we try to keep on our primary guard list?
+ */
+#define DFLT_N_PRIMARY_GUARDS 3
+/**
+ * Of the live guards on the primary guard list, how many do we consider when
+ * choosing a guard to use?
+ */
+#define DFLT_N_PRIMARY_GUARDS_TO_USE 1
+/**
+ * As DFLT_N_PRIMARY_GUARDS, but for choosing which directory guard to use.
+ */
+#define DFLT_N_PRIMARY_DIR_GUARDS_TO_USE 3
+/**
+ * If we haven't successfully built or used a circuit in this long, then
+ * consider that the internet is probably down.
+ */
+#define DFLT_INTERNET_LIKELY_DOWN_INTERVAL (10*60)
+/**
+ * If we're trying to connect to a nonprimary guard for at least this
+ * many seconds, and we haven't gotten the connection to work, we will treat
+ * lower-priority guards as usable.
+ */
+#define DFLT_NONPRIMARY_GUARD_CONNECT_TIMEOUT 15
+/**
+ * If a circuit has been sitting around in 'waiting for better guard' state
+ * for at least this long, we'll expire it.
+ */
+#define DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT (10*60)
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in a restricted setting.
+ */
+#define DFLT_MEANINGFUL_RESTRICTION_PERCENT 20
+/**
+ * If our configuration retains fewer than this fraction of guards from the
+ * torrc, we are in an extremely restricted setting, and should warn.
+ */
+#define DFLT_EXTREME_RESTRICTION_PERCENT 1
+/**@}*/
+
+STATIC double get_max_sample_threshold(void);
+STATIC int get_max_sample_size_absolute(void);
+STATIC int get_min_filtered_sample_size(void);
+STATIC int get_remove_unlisted_guards_after_days(void);
+STATIC int get_guard_lifetime(void);
+STATIC int get_guard_confirmed_min_lifetime(void);
+STATIC int get_n_primary_guards(void);
+STATIC int get_n_primary_guards_to_use(guard_usage_t usage);
+STATIC int get_internet_likely_down_interval(void);
+STATIC int get_nonprimary_guard_connect_timeout(void);
+STATIC int get_nonprimary_guard_idle_timeout(void);
+STATIC double get_meaningful_restriction_threshold(void);
+STATIC double get_extreme_restriction_threshold(void);
+
+HANDLE_DECL(entry_guard, entry_guard_t, STATIC)
+STATIC guard_selection_type_t guard_selection_infer_type(
+                           guard_selection_type_t type_in,
+                           const char *name);
+STATIC guard_selection_t *guard_selection_new(const char *name,
+                                              guard_selection_type_t type);
+STATIC guard_selection_t *get_guard_selection_by_name(
+          const char *name, guard_selection_type_t type, int create_if_absent);
+STATIC void guard_selection_free(guard_selection_t *gs);
+MOCK_DECL(STATIC int, entry_guard_is_listed,
+          (guard_selection_t *gs, const entry_guard_t *guard));
+STATIC const char *choose_guard_selection(const or_options_t *options,
+                                        const networkstatus_t *ns,
+                                        const guard_selection_t *old_selection,
+                                        guard_selection_type_t *type_out);
+STATIC entry_guard_t *get_sampled_guard_with_id(guard_selection_t *gs,
+                                                const uint8_t *rsa_id);
+
+MOCK_DECL(STATIC time_t, randomize_time, (time_t now, time_t max_backdate));
 
+STATIC entry_guard_t *entry_guard_add_to_sample(guard_selection_t *gs,
+                                                const node_t *node);
+STATIC entry_guard_t *entry_guards_expand_sample(guard_selection_t *gs);
+STATIC char *entry_guard_encode_for_state(entry_guard_t *guard);
+STATIC entry_guard_t *entry_guard_parse_from_state(const char *s);
+STATIC void entry_guard_free(entry_guard_t *e);
+STATIC void entry_guards_update_filtered_sets(guard_selection_t *gs);
+STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
+/**
+ * @name Flags for sample_reachable_filtered_entry_guards()
+ */
+/**@{*/
+#define SAMPLE_EXCLUDE_CONFIRMED   (1u<<0)
+#define SAMPLE_EXCLUDE_PRIMARY     (1u<<1)
+#define SAMPLE_EXCLUDE_PENDING     (1u<<2)
+#define SAMPLE_NO_UPDATE_PRIMARY   (1u<<3)
+#define SAMPLE_EXCLUDE_NO_DESCRIPTOR (1u<<4)
+/**@}*/
+STATIC entry_guard_t *sample_reachable_filtered_entry_guards(
+                                    guard_selection_t *gs,
+                                    const entry_guard_restriction_t *rst,
+                                    unsigned flags);
+STATIC void entry_guard_consider_retry(entry_guard_t *guard);
+STATIC void make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard);
+STATIC void entry_guards_update_confirmed(guard_selection_t *gs);
+STATIC void entry_guards_update_primary(guard_selection_t *gs);
+STATIC int num_reachable_filtered_guards(const guard_selection_t *gs,
+                                         const entry_guard_restriction_t *rst);
+STATIC void sampled_guards_update_from_consensus(guard_selection_t *gs);
+/**
+ * @name Possible guard-states for a circuit.
+ */
+/**@{*/
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic as soon as it is successfully
+ * opened. */
+#define GUARD_CIRC_STATE_USABLE_ON_COMPLETION 1
+/** State for an non-open circuit that we shouldn't use for actual
+ * traffic, when it completes, unless other circuits to preferable
+ * guards fail. */
+#define GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD 2
+/** State for an open circuit that we shouldn't use for actual traffic
+ * unless other circuits to preferable guards fail. */
+#define GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD 3
+/** State for a circuit that can (so far as the guard subsystem is
+ * concerned) be used for actual traffic. */
+#define GUARD_CIRC_STATE_COMPLETE 4
+/** State for a circuit that is unusable, and will not become usable. */
+#define GUARD_CIRC_STATE_DEAD 5
+/**@}*/
+STATIC void entry_guards_note_guard_failure(guard_selection_t *gs,
+                                            entry_guard_t *guard);
+STATIC entry_guard_t *select_entry_guard_for_circuit(guard_selection_t *gs,
+                                          guard_usage_t usage,
+                                          const entry_guard_restriction_t *rst,
+                                          unsigned *state_out);
+STATIC void mark_primary_guards_maybe_reachable(guard_selection_t *gs);
+STATIC unsigned entry_guards_note_guard_success(guard_selection_t *gs,
+                                                entry_guard_t *guard,
+                                                unsigned old_state);
+STATIC int entry_guard_has_higher_priority(entry_guard_t *a, entry_guard_t *b);
+STATIC char *getinfo_helper_format_single_entry_guard(const entry_guard_t *e);
+
+STATIC entry_guard_restriction_t *
+guard_create_exit_restriction(const uint8_t *exit_id);
+
+STATIC entry_guard_restriction_t *
+guard_create_dirserver_md_restriction(void);
+
+STATIC void
+entry_guard_restriction_free(entry_guard_restriction_t *rst);
+
+#endif /* defined(ENTRYNODES_PRIVATE) */
+
+void remove_all_entry_guards_for_guard_selection(guard_selection_t *gs);
 void remove_all_entry_guards(void);
 
-void entry_guards_compute_status(const or_options_t *options, time_t now);
-int entry_guard_register_connect_status(const char *digest, int succeeded,
-                                        int mark_relay_status, time_t now);
-void entry_nodes_should_be_added(void);
+struct bridge_info_t;
+void entry_guard_learned_bridge_identity(const tor_addr_port_t *addrport,
+                                         const uint8_t *rsa_id_digest);
+
 int entry_list_is_constrained(const or_options_t *options);
-const node_t *choose_random_entry(cpath_build_state_t *state);
-const node_t *choose_random_dirguard(dirinfo_type_t t);
+int guards_retry_optimistic(const or_options_t *options);
+int entry_guards_parse_state_for_guard_selection(
+    guard_selection_t *gs, or_state_t *state, int set, char **msg);
 int entry_guards_parse_state(or_state_t *state, int set, char **msg);
 void entry_guards_update_state(or_state_t *state);
 int getinfo_helper_entry_guards(control_connection_t *conn,
                                 const char *question, char **answer,
                                 const char **errmsg);
 
-void mark_bridge_list(void);
-void sweep_bridge_list(void);
-
-int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port,
-                                const char *digest);
-int extend_info_is_a_configured_bridge(const extend_info_t *ei);
-int routerinfo_is_a_configured_bridge(const routerinfo_t *ri);
-int node_is_a_configured_bridge(const node_t *node);
-void learned_router_identity(const tor_addr_t *addr, uint16_t port,
-                             const char *digest);
-struct bridge_line_t;
-void bridge_add_from_config(struct bridge_line_t *bridge_line);
-void retry_bridge_descriptor_fetch_directly(const char *digest);
-void fetch_bridge_descriptors(const or_options_t *options, time_t now);
-void learned_bridge_descriptor(routerinfo_t *ri, int from_cache);
-int any_bridge_descriptors_known(void);
 int entries_known_but_down(const or_options_t *options);
 void entries_retry_all(const or_options_t *options);
 
-int any_bridge_supports_microdescriptors(void);
-const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr,
-                                                     uint16_t port);
-
-int any_bridges_dont_support_microdescriptors(void);
+int guard_selection_have_enough_dir_info_to_build_circuits(
+                                                    guard_selection_t *gs);
+int entry_guards_have_enough_dir_info_to_build_circuits(void);
 
 void entry_guards_free_all(void);
 
-const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr,
-                                                   uint16_t port);
-struct transport_t;
-int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port,
-                                      const struct transport_t **transport);
-
-MOCK_DECL(int, transport_is_needed, (const char *transport_name));
-int validate_pluggable_transports_config(void);
-
 double pathbias_get_close_success_count(entry_guard_t *guard);
 double pathbias_get_use_success_count(entry_guard_t *guard);
 
@@ -179,9 +614,5 @@ guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw,
                                   int orig_bandwidth,
                                   uint32_t guardfraction_percentage);
 
-MOCK_DECL(smartlist_t *, list_bridge_identities, (void));
-MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id,
-          (const char *digest));
-
 #endif
 
diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c
index 676adfd8bf..b60d2e55c8 100644
--- a/src/or/ext_orport.c
+++ b/src/or/ext_orport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h
index 33d954e8d0..b2cd05db8f 100644
--- a/src/or/ext_orport.h
+++ b/src/or/ext_orport.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef EXT_ORPORT_H
diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c
index eeeb0f1de3..f730106d06 100644
--- a/src/or/fp_pair.c
+++ b/src/or/fp_pair.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/fp_pair.h b/src/or/fp_pair.h
index b1466581d2..4cea3eda6d 100644
--- a/src/or/fp_pair.h
+++ b/src/or/fp_pair.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/geoip.c b/src/or/geoip.c
index a39366ed13..ff46990de6 100644
--- a/src/or/geoip.c
+++ b/src/or/geoip.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -1017,7 +1017,7 @@ geoip_get_transport_history(void)
 
     /* If it's the first time we see this transport, note it. */
     if (val == 1)
-      smartlist_add(transports_used, tor_strdup(transport_name));
+      smartlist_add_strdup(transports_used, transport_name);
 
     log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
               "I've now seen %d clients.",
diff --git a/src/or/geoip.h b/src/or/geoip.h
index c8ea9f85ea..773525ccfe 100644
--- a/src/or/geoip.h
+++ b/src/or/geoip.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/hibernate.c b/src/or/hibernate.c
index e3c80b5f14..8c48a6f47d 100644
--- a/src/or/hibernate.c
+++ b/src/or/hibernate.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -424,8 +424,8 @@ configure_accounting(time_t now)
     if (-0.50 <= delta && delta <= 0.50) {
       /* The start of the period is now a little later or earlier than we
        * remembered.  That's fine; we might lose some bytes we could otherwise
-       * have written, but better to err on the side of obeying people's
-       * accounting settings. */
+       * have written, but better to err on the side of obeying accounting
+       * settings. */
       log_info(LD_ACCT, "Accounting interval moved by %.02f%%; "
                "that's fine.", delta*100);
       interval_end_time = start_of_accounting_period_after(now);
@@ -587,7 +587,10 @@ accounting_set_wakeup_time(void)
     char buf[ISO_TIME_LEN+1];
     format_iso_time(buf, interval_start_time);
 
-    crypto_pk_get_digest(get_server_identity_key(), digest);
+    if (crypto_pk_get_digest(get_server_identity_key(), digest) < 0) {
+      log_err(LD_BUG, "Error getting our key's digest.");
+      tor_assert(0);
+    }
 
     d_env = crypto_digest_new();
     crypto_digest_add_bytes(d_env, buf, ISO_TIME_LEN);
@@ -896,7 +899,7 @@ hibernate_go_dormant(time_t now)
   log_notice(LD_ACCT,"Going dormant. Blowing away remaining connections.");
 
   /* Close all OR/AP/exit conns. Leave dir conns because we still want
-   * to be able to upload server descriptors so people know we're still
+   * to be able to upload server descriptors so clients know we're still
    * running, and download directories so we can detect if we're obsolete.
    * Leave control conns because we still want to be controllable.
    */
diff --git a/src/or/hibernate.h b/src/or/hibernate.h
index fa9da6de39..8bdb65a927 100644
--- a/src/or/hibernate.h
+++ b/src/or/hibernate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c
new file mode 100644
index 0000000000..29681b42b5
--- /dev/null
+++ b/src/or/hs_cache.c
@@ -0,0 +1,398 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.c
+ * \brief Handle hidden service descriptor caches.
+ **/
+
+/* For unit tests.*/
+#define HS_CACHE_PRIVATE
+
+#include "hs_cache.h"
+
+#include "or.h"
+#include "config.h"
+#include "hs_common.h"
+#include "hs_descriptor.h"
+#include "networkstatus.h"
+#include "rendcache.h"
+
+/* Directory descriptor cache. Map indexed by blinded key. */
+static digest256map_t *hs_cache_v3_dir;
+
+/* Remove a given descriptor from our cache. */
+static void
+remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_remove(hs_cache_v3_dir, desc->key);
+}
+
+/* Store a given descriptor in our cache. */
+static void
+store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  tor_assert(desc);
+  digest256map_set(hs_cache_v3_dir, desc->key, desc);
+}
+
+/* Query our cache and return the entry or NULL if not found. */
+static hs_cache_dir_descriptor_t *
+lookup_v3_desc_as_dir(const uint8_t *key)
+{
+  tor_assert(key);
+  return digest256map_get(hs_cache_v3_dir, key);
+}
+
+/* Free a directory descriptor object. */
+static void
+cache_dir_desc_free(hs_cache_dir_descriptor_t *desc)
+{
+  if (desc == NULL) {
+    return;
+  }
+  hs_desc_plaintext_data_free(desc->plaintext_data);
+  tor_free(desc->encoded_desc);
+  tor_free(desc);
+}
+
+/* Helper function: Use by the free all function using the digest256map
+ * interface to cache entries. */
+static void
+cache_dir_desc_free_(void *ptr)
+{
+  hs_cache_dir_descriptor_t *desc = ptr;
+  cache_dir_desc_free(desc);
+}
+
+/* Create a new directory cache descriptor object from a encoded descriptor.
+ * On success, return the heap-allocated cache object, otherwise return NULL if
+ * we can't decode the descriptor. */
+static hs_cache_dir_descriptor_t *
+cache_dir_desc_new(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc;
+
+  tor_assert(desc);
+
+  dir_desc = tor_malloc_zero(sizeof(hs_cache_dir_descriptor_t));
+  dir_desc->plaintext_data =
+    tor_malloc_zero(sizeof(hs_desc_plaintext_data_t));
+  dir_desc->encoded_desc = tor_strdup(desc);
+
+  if (hs_desc_decode_plaintext(desc, dir_desc->plaintext_data) < 0) {
+    log_debug(LD_DIR, "Unable to decode descriptor. Rejecting.");
+    goto err;
+  }
+
+  /* The blinded pubkey is the indexed key. */
+  dir_desc->key = dir_desc->plaintext_data->blinded_pubkey.pubkey;
+  dir_desc->created_ts = time(NULL);
+  return dir_desc;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return NULL;
+}
+
+/* Return the size of a cache entry in bytes. */
+static size_t
+cache_get_entry_size(const hs_cache_dir_descriptor_t *entry)
+{
+  return (sizeof(*entry) + hs_desc_plaintext_obj_size(entry->plaintext_data)
+          + strlen(entry->encoded_desc));
+}
+
+/* Try to store a valid version 3 descriptor in the directory cache. Return 0
+ * on success else a negative value is returned indicating that we have a
+ * newer version in our cache. On error, caller is responsible to free the
+ * given descriptor desc. */
+static int
+cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
+{
+  hs_cache_dir_descriptor_t *cache_entry;
+
+  tor_assert(desc);
+
+  /* Verify if we have an entry in the cache for that key and if yes, check
+   * if we should replace it? */
+  cache_entry = lookup_v3_desc_as_dir(desc->key);
+  if (cache_entry != NULL) {
+    /* Only replace descriptor if revision-counter is greater than the one
+     * in our cache */
+    if (cache_entry->plaintext_data->revision_counter >=
+        desc->plaintext_data->revision_counter) {
+      log_info(LD_REND, "Descriptor revision counter in our cache is "
+                        "greater or equal than the one we received. "
+                        "Rejecting!");
+      goto err;
+    }
+    /* We now know that the descriptor we just received is a new one so
+     * remove the entry we currently have from our cache so we can then
+     * store the new one. */
+    remove_v3_desc_as_dir(cache_entry);
+    rend_cache_decrement_allocation(cache_get_entry_size(cache_entry));
+    cache_dir_desc_free(cache_entry);
+  }
+  /* Store the descriptor we just got. We are sure here that either we
+   * don't have the entry or we have a newer descriptor and the old one
+   * has been removed from the cache. */
+  store_v3_desc_as_dir(desc);
+
+  /* Update our total cache size with this entry for the OOM. This uses the
+   * old HS protocol cache subsystem for which we are tied with. */
+  rend_cache_increment_allocation(cache_get_entry_size(desc));
+
+  /* XXX: Update HS statistics. We should have specific stats for v3. */
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Using the query which is the base64 encoded blinded key of a version 3
+ * descriptor, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being the
+ * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
+ * On error, a negative value is returned and desc_out is untouched. */
+static int
+cache_lookup_v3_as_dir(const char *query, const char **desc_out)
+{
+  int found = 0;
+  ed25519_public_key_t blinded_key;
+  const hs_cache_dir_descriptor_t *entry;
+
+  tor_assert(query);
+
+  /* Decode blinded key using the given query value. */
+  if (ed25519_public_from_base64(&blinded_key, query) < 0) {
+    log_info(LD_REND, "Unable to decode the v3 HSDir query %s.",
+             safe_str_client(query));
+    goto err;
+  }
+
+  entry = lookup_v3_desc_as_dir(blinded_key.pubkey);
+  if (entry != NULL) {
+    found = 1;
+    if (desc_out) {
+      *desc_out = entry->encoded_desc;
+    }
+  }
+
+  return found;
+
+ err:
+  return -1;
+}
+
+/* Clean the v3 cache by removing any entry that has expired using the
+ * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
+ * process will use the lifetime found in the plaintext data section. Return
+ * the number of bytes cleaned. */
+STATIC size_t
+cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
+{
+  size_t bytes_removed = 0;
+
+  /* Code flow error if this ever happens. */
+  tor_assert(global_cutoff >= 0);
+
+  if (!hs_cache_v3_dir) { /* No cache to clean. Just return. */
+    return 0;
+  }
+
+  DIGEST256MAP_FOREACH_MODIFY(hs_cache_v3_dir, key,
+                              hs_cache_dir_descriptor_t *, entry) {
+    size_t entry_size;
+    time_t cutoff = global_cutoff;
+    if (!cutoff) {
+      /* Cutoff is the lifetime of the entry found in the descriptor. */
+      cutoff = now - entry->plaintext_data->lifetime_sec;
+    }
+
+    /* If the entry has been created _after_ the cutoff, not expired so
+     * continue to the next entry in our v3 cache. */
+    if (entry->created_ts > cutoff) {
+      continue;
+    }
+    /* Here, our entry has expired, remove and free. */
+    MAP_DEL_CURRENT(key);
+    entry_size = cache_get_entry_size(entry);
+    bytes_removed += entry_size;
+    /* Entry is not in the cache anymore, destroy it. */
+    cache_dir_desc_free(entry);
+    /* Update our cache entry allocation size for the OOM. */
+    rend_cache_decrement_allocation(entry_size);
+    /* Logging. */
+    {
+      char key_b64[BASE64_DIGEST256_LEN + 1];
+      base64_encode(key_b64, sizeof(key_b64), (const char *) key,
+                    DIGEST256_LEN, 0);
+      log_info(LD_REND, "Removing v3 descriptor '%s' from HSDir cache",
+               safe_str_client(key_b64));
+    }
+  } DIGEST256MAP_FOREACH_END;
+
+  return bytes_removed;
+}
+
+/* Given an encoded descriptor, store it in the directory cache depending on
+ * which version it is. Return a negative value on error. On success, 0 is
+ * returned. */
+int
+hs_cache_store_as_dir(const char *desc)
+{
+  hs_cache_dir_descriptor_t *dir_desc = NULL;
+
+  tor_assert(desc);
+
+  /* Create a new cache object. This can fail if the descriptor plaintext data
+   * is unparseable which in this case a log message will be triggered. */
+  dir_desc = cache_dir_desc_new(desc);
+  if (dir_desc == NULL) {
+    goto err;
+  }
+
+  /* Call the right function against the descriptor version. At this point,
+   * we are sure that the descriptor's version is supported else the
+   * decoding would have failed. */
+  switch (dir_desc->plaintext_data->version) {
+  case HS_VERSION_THREE:
+  default:
+    if (cache_store_v3_as_dir(dir_desc) < 0) {
+      goto err;
+    }
+    break;
+  }
+  return 0;
+
+ err:
+  cache_dir_desc_free(dir_desc);
+  return -1;
+}
+
+/* Using the query, lookup in our directory cache the entry. If found, 1 is
+ * returned and desc_out is populated with a newly allocated string being
+ * the encoded descriptor. If not found, 0 is returned and desc_out is
+ * untouched. On error, a negative value is returned and desc_out is
+ * untouched. */
+int
+hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                       const char **desc_out)
+{
+  int found;
+
+  tor_assert(query);
+  /* This should never be called with an unsupported version. */
+  tor_assert(hs_desc_is_supported_version(version));
+
+  switch (version) {
+  case HS_VERSION_THREE:
+  default:
+    found = cache_lookup_v3_as_dir(query, desc_out);
+    break;
+  }
+
+  return found;
+}
+
+/* Clean all directory caches using the current time now. */
+void
+hs_cache_clean_as_dir(time_t now)
+{
+  time_t cutoff;
+
+  /* Start with v2 cache cleaning. */
+  cutoff = now - rend_cache_max_entry_lifetime();
+  rend_cache_clean_v2_descs_as_dir(cutoff);
+
+  /* Now, clean the v3 cache. Set the cutoff to 0 telling the cleanup function
+   * to compute the cutoff by itself using the lifetime value. */
+  cache_clean_v3_as_dir(now, 0);
+}
+
+/* Do a round of OOM cleanup on all directory caches. Return the amount of
+ * removed bytes. It is possible that the returned value is lower than
+ * min_remove_bytes if the caches get emptied out so the caller should be
+ * aware of this. */
+size_t
+hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t k;
+  size_t bytes_removed = 0;
+
+  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+  tor_assert(min_remove_bytes != 0);
+
+  /* The algorithm is as follow. K is the oldest expected descriptor age.
+   *
+   *   1) Deallocate all entries from v2 cache that are older than K hours.
+   *      1.1) If the amount of remove bytes has been reached, stop.
+   *   2) Deallocate all entries from v3 cache that are older than K hours
+   *      2.1) If the amount of remove bytes has been reached, stop.
+   *   3) Set K = K - RendPostPeriod and repeat process until K is < 0.
+   *
+   * This ends up being O(Kn).
+   */
+
+  /* Set K to the oldest expected age in seconds which is the maximum
+   * lifetime of a cache entry. We'll use the v2 lifetime because it's much
+   * bigger than the v3 thus leading to cleaning older descriptors. */
+  k = rend_cache_max_entry_lifetime();
+
+  do {
+    time_t cutoff;
+
+    /* If K becomes negative, it means we've empty the caches so stop and
+     * return what we were able to cleanup. */
+    if (k < 0) {
+      break;
+    }
+    /* Compute a cutoff value with K and the current time. */
+    cutoff = now - k;
+
+    /* Start by cleaning the v2 cache with that cutoff. */
+    bytes_removed += rend_cache_clean_v2_descs_as_dir(cutoff);
+
+    if (bytes_removed < min_remove_bytes) {
+      /* We haven't remove enough bytes so clean v3 cache. */
+      bytes_removed += cache_clean_v3_as_dir(now, cutoff);
+      /* Decrement K by a post period to shorten the cutoff. */
+      k -= get_options()->RendPostPeriod;
+    }
+  } while (bytes_removed < min_remove_bytes);
+
+  return bytes_removed;
+}
+
+/**
+ * Return the maximum size of an HS descriptor we are willing to accept as an
+ * HSDir.
+ */
+unsigned int
+hs_cache_get_max_descriptor_size(void)
+{
+  return (unsigned) networkstatus_get_param(NULL,
+                                            "HSV3MaxDescriptorSize",
+                                            HS_DESC_MAX_LEN, 1, INT32_MAX);
+}
+
+/* Initialize the hidden service cache subsystem. */
+void
+hs_cache_init(void)
+{
+  /* Calling this twice is very wrong code flow. */
+  tor_assert(!hs_cache_v3_dir);
+  hs_cache_v3_dir = digest256map_new();
+}
+
+/* Cleanup the hidden service cache subsystem. */
+void
+hs_cache_free_all(void)
+{
+  digest256map_free(hs_cache_v3_dir, cache_dir_desc_free_);
+  hs_cache_v3_dir = NULL;
+}
+
diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h
new file mode 100644
index 0000000000..ed00424234
--- /dev/null
+++ b/src/or/hs_cache.h
@@ -0,0 +1,63 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_cache.h
+ * \brief Header file for hs_cache.c
+ **/
+
+#ifndef TOR_HS_CACHE_H
+#define TOR_HS_CACHE_H
+
+#include <stdint.h>
+
+#include "crypto.h"
+#include "crypto_ed25519.h"
+#include "hs_common.h"
+#include "hs_descriptor.h"
+#include "torcert.h"
+
+/* Descriptor representation on the directory side which is a subset of
+ * information that the HSDir can decode and serve it. */
+typedef struct hs_cache_dir_descriptor_t {
+  /* This object is indexed using the blinded pubkey located in the plaintext
+   * data which is populated only once the descriptor has been successfully
+   * decoded and validated. This simply points to that pubkey. */
+  const uint8_t *key;
+
+  /* When does this entry has been created. Used to expire entries. */
+  time_t created_ts;
+
+  /* Descriptor plaintext information. Obviously, we can't decrypt the
+   * encrypted part of the descriptor. */
+  hs_desc_plaintext_data_t *plaintext_data;
+
+  /* Encoded descriptor which is basically in text form. It's a NUL terminated
+   * string thus safe to strlen(). */
+  char *encoded_desc;
+} hs_cache_dir_descriptor_t;
+
+/* Public API */
+
+void hs_cache_init(void);
+void hs_cache_free_all(void);
+void hs_cache_clean_as_dir(time_t now);
+size_t hs_cache_handle_oom(time_t now, size_t min_remove_bytes);
+
+unsigned int hs_cache_get_max_descriptor_size(void);
+
+/* Store and Lookup function. They are version agnostic that is depending on
+ * the requested version of the descriptor, it will be re-routed to the
+ * right function. */
+int hs_cache_store_as_dir(const char *desc);
+int hs_cache_lookup_as_dir(uint32_t version, const char *query,
+                           const char **desc_out);
+
+#ifdef HS_CACHE_PRIVATE
+
+STATIC size_t cache_clean_v3_as_dir(time_t now, time_t global_cutoff);
+
+#endif /* HS_CACHE_PRIVATE */
+
+#endif /* TOR_HS_CACHE_H */
+
diff --git a/src/or/hs_circuitmap.c b/src/or/hs_circuitmap.c
new file mode 100644
index 0000000000..ea66fb5194
--- /dev/null
+++ b/src/or/hs_circuitmap.c
@@ -0,0 +1,490 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.c
+ *
+ * \brief Hidden service circuitmap: A hash table that maps binary tokens to
+ *  introduction and rendezvous circuits; it's used both by relays acting as
+ *  intro points and rendezvous points, and also by hidden services themselves.
+ **/
+
+#define HS_CIRCUITMAP_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "hs_circuitmap.h"
+
+/************************** HS circuitmap code *******************************/
+
+/* This is the hidden service circuitmap. It's a hash table that maps
+   introduction and rendezvous tokens to specific circuits such that given a
+   token it's easy to find the corresponding circuit. */
+static struct hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+/* This is a helper function used by the hash table code (HT_). It returns 1 if
+ * two circuits have the same HS token. */
+static int
+hs_circuits_have_same_token(const circuit_t *first_circuit,
+                            const circuit_t *second_circuit)
+{
+  const hs_token_t *first_token;
+  const hs_token_t *second_token;
+
+  tor_assert(first_circuit);
+  tor_assert(second_circuit);
+
+  first_token = first_circuit->hs_token;
+  second_token = second_circuit->hs_token;
+
+  /* Both circs must have a token */
+  if (BUG(!first_token) || BUG(!second_token)) {
+    return 0;
+  }
+
+  if (first_token->type != second_token->type) {
+    return 0;
+  }
+
+  if (first_token->token_len != second_token->token_len)
+    return 0;
+
+  return tor_memeq(first_token->token,
+                   second_token->token,
+                   first_token->token_len);
+}
+
+/* This is a helper function for the hash table code (HT_). It hashes a circuit
+ * HS token into an unsigned int for use as a key by the hash table routines.*/
+static inline unsigned int
+hs_circuit_hash_token(const circuit_t *circuit)
+{
+  tor_assert(circuit->hs_token);
+
+  return (unsigned) siphash24g(circuit->hs_token->token,
+                               circuit->hs_token->token_len);
+}
+
+/* Register the circuitmap hash table */
+HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct
+             circuit_t,    // The name of the element struct,
+             hs_circuitmap_node,        // The name of HT_ENTRY member
+             hs_circuit_hash_token, hs_circuits_have_same_token)
+
+HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node,
+             hs_circuit_hash_token, hs_circuits_have_same_token,
+             0.6, tor_reallocarray, tor_free_)
+
+#ifdef TOR_UNIT_TESTS
+
+/* Return the global HS circuitmap. Used by unittests. */
+hs_circuitmap_ht *
+get_hs_circuitmap(void)
+{
+  return the_hs_circuitmap;
+}
+
+#endif
+
+/****************** HS circuitmap utility functions **************************/
+
+/** Return a new HS token of type <b>type</b> containing <b>token</b>. */
+static hs_token_t *
+hs_token_new(hs_token_type_t type, size_t token_len,
+             const uint8_t *token)
+{
+  tor_assert(token);
+
+  hs_token_t *hs_token = tor_malloc_zero(sizeof(hs_token_t));
+  hs_token->type = type;
+  hs_token->token_len = token_len;
+  hs_token->token = tor_memdup(token, token_len);
+
+  return hs_token;
+}
+
+/** Free memory allocated by this <b>hs_token</b>. */
+static void
+hs_token_free(hs_token_t *hs_token)
+{
+  if (!hs_token) {
+    return;
+  }
+
+  tor_free(hs_token->token);
+  tor_free(hs_token);
+}
+
+/** Return the circuit from the circuitmap with token <b>search_token</b>. */
+static circuit_t *
+get_circuit_with_token(hs_token_t *search_token)
+{
+  tor_assert(the_hs_circuitmap);
+
+  /* We use a dummy circuit object for the hash table search routine. */
+  circuit_t search_circ;
+  search_circ.hs_token = search_token;
+  return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ);
+}
+
+/* Helper function that registers <b>circ</b> with <b>token</b> on the HS
+   circuitmap. This function steals reference of <b>token</b>. */
+static void
+hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token)
+{
+  tor_assert(circ);
+  tor_assert(token);
+  tor_assert(the_hs_circuitmap);
+
+  /* If this circuit already has a token, clear it. */
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+
+  /* Kill old circuits with the same token. We want new intro/rend circuits to
+     take precedence over old ones, so that HSes and clients and reestablish
+     killed circuits without changing the HS token. */
+  {
+    circuit_t *found_circ;
+    found_circ = get_circuit_with_token(token);
+    if (found_circ) {
+      hs_circuitmap_remove_circuit(found_circ);
+      if (!found_circ->marked_for_close) {
+        circuit_mark_for_close(found_circ, END_CIRC_REASON_FINISHED);
+      }
+    }
+  }
+
+  /* Register circuit and token to circuitmap. */
+  circ->hs_token = token;
+  HT_INSERT(hs_circuitmap_ht, the_hs_circuitmap, circ);
+}
+
+/** Helper function: Register <b>circ</b> of <b>type</b> on the HS
+ *  circuitmap. Use the HS <b>token</b> as the key to the hash table.  If
+ *  <b>token</b> is not set, clear the circuit of any HS tokens. */
+static void
+hs_circuitmap_register_circuit(circuit_t *circ,
+                               hs_token_type_t type, size_t token_len,
+                               const uint8_t *token)
+{
+  hs_token_t *hs_token = NULL;
+
+  /* Create a new token and register it to the circuitmap */
+  tor_assert(token);
+  hs_token = hs_token_new(type, token_len, token);
+  tor_assert(hs_token);
+  hs_circuitmap_register_impl(circ, hs_token);
+}
+
+/* Helper function for hs_circuitmap_get_origin_circuit() and
+ * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the
+ * circuitmap, this function returns object type so the specialized functions
+ * using this helper can upcast it to the right type.
+ *
+ * Return NULL if not such circuit is found. */
+static circuit_t *
+hs_circuitmap_get_circuit_impl(hs_token_type_t type,
+                               size_t token_len,
+                               const uint8_t *token,
+                               uint8_t wanted_circ_purpose)
+{
+  circuit_t *found_circ = NULL;
+
+  tor_assert(the_hs_circuitmap);
+
+  /* Check the circuitmap if we have a circuit with this token */
+  {
+    hs_token_t *search_hs_token = hs_token_new(type, token_len, token);
+    tor_assert(search_hs_token);
+    found_circ = get_circuit_with_token(search_hs_token);
+    hs_token_free(search_hs_token);
+  }
+
+  /* Check that the circuit is useful to us */
+  if (!found_circ ||
+      found_circ->purpose != wanted_circ_purpose ||
+      found_circ->marked_for_close) {
+    return NULL;
+  }
+
+  return found_circ;
+}
+
+/* Helper function: Query circuitmap for origin circuit with <b>token</b> of
+ * size <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose
+ * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked
+ * for close. Return NULL if no such circuit is found. */
+static origin_circuit_t *
+hs_circuitmap_get_origin_circuit(hs_token_type_t type,
+                                 size_t token_len,
+                                 const uint8_t *token,
+                                 uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  return TO_ORIGIN_CIRCUIT(circ);
+}
+
+/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size
+ * <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose equal
+ * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for
+ * close. Return NULL if no such circuit is found. */
+static or_circuit_t *
+hs_circuitmap_get_or_circuit(hs_token_type_t type,
+                             size_t token_len,
+                             const uint8_t *token,
+                             uint8_t wanted_circ_purpose)
+{
+  circuit_t *circ;
+  tor_assert(token);
+  tor_assert(!CIRCUIT_PURPOSE_IS_ORIGIN(wanted_circ_purpose));
+
+  circ = hs_circuitmap_get_circuit_impl(type, token_len, token,
+                                        wanted_circ_purpose);
+  if (!circ) {
+    return NULL;
+  }
+
+  tor_assert(CIRCUIT_IS_ORCIRC(circ));
+  return TO_OR_CIRCUIT(circ);
+}
+
+/************** Public circuitmap API ****************************************/
+
+/**** Public relay-side getters: */
+
+/* Public function: Return a v3 introduction circuit to this relay with
+ * <b>auth_key</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(
+                                          const ed25519_public_key_t *auth_key)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                      ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return v2 introduction circuit to this relay with
+ * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                      REND_TOKEN_LEN, digest,
+                                      CIRCUIT_PURPOSE_INTRO_POINT);
+}
+
+/* Public function: Return rendezvous circuit to this relay with rendezvous
+ * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */
+or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
+{
+  return hs_circuitmap_get_or_circuit(HS_TOKEN_REND_RELAY_SIDE,
+                                      REND_TOKEN_LEN, cookie,
+                                      CIRCUIT_PURPOSE_REND_POINT_WAITING);
+}
+
+/** Public relay-side setters: */
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
+                                            const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_RELAY_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
+                                                const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_RELAY_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_RELAY_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/**** Public servide-side getters: */
+
+/* Public function: Return v3 introduction circuit with <b>auth_key</b>
+ * originating from this hidden service. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                          ED25519_PUBKEY_LEN, auth_key->pubkey,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return v2 introduction circuit originating from this hidden
+ * service with <b>digest</b>. Return NULL if no such circuit is found in the
+ * circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Check first for established intro circuits */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_INTRO);
+  if (circ) {
+    return circ;
+  }
+
+  /* ...if nothing found, check for pending intro circs */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, digest,
+                                          CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
+
+  return circ;
+}
+
+/* Public function: Return rendezvous circuit originating from this hidden
+ * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is
+ * found in the circuitmap. */
+origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie)
+{
+  origin_circuit_t *circ = NULL;
+
+  /* Try to check if we have a connecting circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_CONNECT_REND);
+  if (circ) {
+    return circ;
+  }
+
+  /* Then try for connected circuit. */
+  circ = hs_circuitmap_get_origin_circuit(HS_TOKEN_REND_SERVICE_SIDE,
+                                          REND_TOKEN_LEN, cookie,
+                                          CIRCUIT_PURPOSE_S_REND_JOINED);
+  return circ;
+}
+
+/**** Public servide-side setters: */
+
+/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
+                                                  const uint8_t *digest)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, digest);
+}
+
+/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
+                                          const ed25519_public_key_t *auth_key)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+                                 ED25519_PUBKEY_LEN, auth_key->pubkey);
+}
+
+/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+ * circuitmap. */
+void
+hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
+                                              const uint8_t *cookie)
+{
+  hs_circuitmap_register_circuit(TO_CIRCUIT(circ),
+                                 HS_TOKEN_REND_SERVICE_SIDE,
+                                 REND_TOKEN_LEN, cookie);
+}
+
+/**** Misc public functions: */
+
+/** Public function: Remove this circuit from the HS circuitmap. Clear its HS
+ *  token, and remove it from the hashtable. */
+void
+hs_circuitmap_remove_circuit(circuit_t *circ)
+{
+  tor_assert(the_hs_circuitmap);
+
+  if (!circ || !circ->hs_token) {
+    return;
+  }
+
+  /* Remove circ from circuitmap */
+  circuit_t *tmp;
+  tmp = HT_REMOVE(hs_circuitmap_ht, the_hs_circuitmap, circ);
+  /* ... and ensure the removal was successful. */
+  if (tmp) {
+    tor_assert(tmp == circ);
+  } else {
+    log_warn(LD_BUG, "Could not find circuit (%u) in circuitmap.",
+             circ->n_circ_id);
+  }
+
+  /* Clear token from circ */
+  hs_token_free(circ->hs_token);
+  circ->hs_token = NULL;
+}
+
+/* Public function: Initialize the global HS circuitmap. */
+void
+hs_circuitmap_init(void)
+{
+  tor_assert(!the_hs_circuitmap);
+
+  the_hs_circuitmap = tor_malloc_zero(sizeof(struct hs_circuitmap_ht));
+  HT_INIT(hs_circuitmap_ht, the_hs_circuitmap);
+}
+
+/* Public function: Free all memory allocated by the global HS circuitmap. */
+void
+hs_circuitmap_free_all(void)
+{
+  if (the_hs_circuitmap) {
+    HT_CLEAR(hs_circuitmap_ht, the_hs_circuitmap);
+    tor_free(the_hs_circuitmap);
+  }
+}
+
diff --git a/src/or/hs_circuitmap.h b/src/or/hs_circuitmap.h
new file mode 100644
index 0000000000..33d5b64117
--- /dev/null
+++ b/src/or/hs_circuitmap.h
@@ -0,0 +1,103 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_circuitmap.h
+ * \brief Header file for hs_circuitmap.c.
+ **/
+
+#ifndef TOR_HS_CIRCUITMAP_H
+#define TOR_HS_CIRCUITMAP_H
+
+typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht;
+
+typedef struct hs_token_s hs_token_t;
+struct or_circuit_t;
+struct origin_circuit_t;
+
+/** Public HS circuitmap API: */
+
+/** Public relay-side API: */
+
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v3_relay_side(const
+                                           ed25519_public_key_t *auth_key);
+struct or_circuit_t *
+hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest);
+struct or_circuit_t *
+hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ,
+                                                 const uint8_t *cookie);
+void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
+                                                     const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
+                                         const ed25519_public_key_t *auth_key);
+
+/** Public service-side API: */
+
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v3_service_side(const
+                                             ed25519_public_key_t *auth_key);
+struct origin_circuit_t *
+hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest);
+struct origin_circuit_t *
+hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie);
+
+void hs_circuitmap_register_intro_circ_v2_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *digest);
+void hs_circuitmap_register_intro_circ_v3_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const ed25519_public_key_t *auth_key);
+void hs_circuitmap_register_rend_circ_service_side(
+                                        struct origin_circuit_t *circ,
+                                        const uint8_t *cookie);
+
+void hs_circuitmap_remove_circuit(struct circuit_t *circ);
+
+void hs_circuitmap_init(void);
+void hs_circuitmap_free_all(void);
+
+#ifdef HS_CIRCUITMAP_PRIVATE
+
+/** Represents the type of HS token. */
+typedef enum {
+  /** A rendezvous cookie on a relay (128bit)*/
+  HS_TOKEN_REND_RELAY_SIDE,
+  /** A v2 introduction point pubkey on a relay (160bit) */
+  HS_TOKEN_INTRO_V2_RELAY_SIDE,
+  /** A v3 introduction point pubkey on a relay (256bit) */
+  HS_TOKEN_INTRO_V3_RELAY_SIDE,
+
+  /** A rendezvous cookie on a hidden service (128bit)*/
+  HS_TOKEN_REND_SERVICE_SIDE,
+  /** A v2 introduction point pubkey on a hidden service (160bit) */
+  HS_TOKEN_INTRO_V2_SERVICE_SIDE,
+  /** A v3 introduction point pubkey on a hidden service (256bit) */
+  HS_TOKEN_INTRO_V3_SERVICE_SIDE,
+} hs_token_type_t;
+
+/** Represents a token used in the HS protocol. Each such token maps to a
+ *  specific introduction or rendezvous circuit. */
+struct hs_token_s {
+  /* Type of HS token. */
+  hs_token_type_t type;
+
+  /* The size of the token (depends on the type). */
+  size_t token_len;
+
+  /* The token itself. Memory allocated at runtime. */
+  uint8_t *token;
+};
+
+#endif /* HS_CIRCUITMAP_PRIVATE */
+
+#ifdef TOR_UNIT_TESTS
+
+hs_circuitmap_ht *get_hs_circuitmap(void);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* TOR_HS_CIRCUITMAP_H */
+
diff --git a/src/or/hs_common.c b/src/or/hs_common.c
new file mode 100644
index 0000000000..c9af3f6887
--- /dev/null
+++ b/src/or/hs_common.c
@@ -0,0 +1,363 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.c
+ * \brief Contains code shared between different HS protocol version as well
+ *        as useful data structures and accessors used by other subsystems.
+ *        The rendcommon.c should only contains code relating to the v2
+ *        protocol.
+ **/
+
+#define HS_COMMON_PRIVATE
+
+#include "or.h"
+
+#include "config.h"
+#include "networkstatus.h"
+#include "hs_common.h"
+#include "rendcommon.h"
+
+/* Make sure that the directory for <b>service</b> is private, using the config
+ * <b>username</b>.
+ * If <b>create</b> is true:
+ *  - if the directory exists, change permissions if needed,
+ *  - if the directory does not exist, create it with the correct permissions.
+ * If <b>create</b> is false:
+ *  - if the directory exists, check permissions,
+ *  - if the directory does not exist, check if we think we can create it.
+ * Return 0 on success, -1 on failure. */
+int
+hs_check_service_private_dir(const char *username, const char *path,
+                             unsigned int dir_group_readable,
+                             unsigned int create)
+{
+  cpd_check_t check_opts = CPD_NONE;
+
+  tor_assert(path);
+
+  if (create) {
+    check_opts |= CPD_CREATE;
+  } else {
+    check_opts |= CPD_CHECK_MODE_ONLY;
+    check_opts |= CPD_CHECK;
+  }
+  if (dir_group_readable) {
+    check_opts |= CPD_GROUP_READ;
+  }
+  /* Check/create directory */
+  if (check_private_dir(path, check_opts, username) < 0) {
+    return -1;
+  }
+  return 0;
+}
+
+/** Get the default HS time period length in minutes from the consensus. */
+STATIC uint64_t
+get_time_period_length(void)
+{
+  int32_t time_period_length = networkstatus_get_param(NULL, "hsdir_interval",
+                                             HS_TIME_PERIOD_LENGTH_DEFAULT,
+                                             HS_TIME_PERIOD_LENGTH_MIN,
+                                             HS_TIME_PERIOD_LENGTH_MAX);
+  /* Make sure it's a positive value. */
+  tor_assert(time_period_length >= 0);
+  /* uint64_t will always be able to contain a int32_t */
+  return (uint64_t) time_period_length;
+}
+
+/** Get the HS time period number at time <b>now</b> */
+STATIC uint64_t
+get_time_period_num(time_t now)
+{
+  uint64_t time_period_num;
+  uint64_t time_period_length = get_time_period_length();
+  uint64_t minutes_since_epoch = now / 60;
+
+  /* Now subtract half a day to fit the prop224 time period schedule (see
+   * section [TIME-PERIODS]). */
+  tor_assert(minutes_since_epoch > HS_TIME_PERIOD_ROTATION_OFFSET);
+  minutes_since_epoch -= HS_TIME_PERIOD_ROTATION_OFFSET;
+
+  /* Calculate the time period */
+  time_period_num = minutes_since_epoch / time_period_length;
+  return time_period_num;
+}
+
+/** Get the number of the _upcoming_ HS time period, given that the current
+ *  time is <b>now</b>. */
+uint64_t
+hs_get_next_time_period_num(time_t now)
+{
+  return get_time_period_num(now) + 1;
+}
+
+/* Create a new rend_data_t for a specific given <b>version</b>.
+ * Return a pointer to the newly allocated data structure. */
+static rend_data_t *
+rend_data_alloc(uint32_t version)
+{
+  rend_data_t *rend_data = NULL;
+
+  switch (version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2 = tor_malloc_zero(sizeof(*v2));
+    v2->base_.version = HS_VERSION_TWO;
+    v2->base_.hsdirs_fp = smartlist_new();
+    rend_data = &v2->base_;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return rend_data;
+}
+
+/** Free all storage associated with <b>data</b> */
+void
+rend_data_free(rend_data_t *data)
+{
+  if (!data) {
+    return;
+  }
+  /* By using our allocation function, this should always be set. */
+  tor_assert(data->hsdirs_fp);
+  /* Cleanup the HSDir identity digest. */
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(data->hsdirs_fp);
+  /* Depending on the version, cleanup. */
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(data);
+    tor_free(v2_data);
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+}
+
+/* Allocate and return a deep copy of <b>data</b>. */
+rend_data_t *
+rend_data_dup(const rend_data_t *data)
+{
+  rend_data_t *data_dup = NULL;
+  smartlist_t *hsdirs_fp = smartlist_new();
+
+  tor_assert(data);
+  tor_assert(data->hsdirs_fp);
+
+  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
+                    smartlist_add(hsdirs_fp, tor_memdup(fp, DIGEST_LEN)));
+
+  switch (data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = tor_memdup(TO_REND_DATA_V2(data),
+                                         sizeof(*v2_data));
+    data_dup = &v2_data->base_;
+    data_dup->hsdirs_fp = hsdirs_fp;
+    break;
+  }
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  return data_dup;
+}
+
+/* Compute the descriptor ID for each HS descriptor replica and save them. A
+ * valid onion address must be present in the <b>rend_data</b>.
+ *
+ * Return 0 on success else -1. */
+static int
+compute_desc_id(rend_data_t *rend_data)
+{
+  int ret = 0;
+  unsigned replica;
+  time_t now = time(NULL);
+
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    /* Compute descriptor ID for each replicas. */
+    for (replica = 0; replica < ARRAY_LENGTH(v2_data->descriptor_id);
+         replica++) {
+      ret = rend_compute_v2_desc_id(v2_data->descriptor_id[replica],
+                                    v2_data->onion_address,
+                                    v2_data->descriptor_cookie,
+                                    now, replica);
+      if (ret < 0) {
+        goto end;
+      }
+    }
+    break;
+  }
+  default:
+    tor_assert(0);
+  }
+
+ end:
+  return ret;
+}
+
+/* Allocate and initialize a rend_data_t object for a service using the
+ * provided arguments. All arguments are optional (can be NULL), except from
+ * <b>onion_address</b> which MUST be set. The <b>pk_digest</b> is the hash of
+ * the service private key. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation this service is configured with.
+ *
+ * Return a valid rend_data_t pointer. This only returns a version 2 object of
+ * rend_data_t. */
+rend_data_t *
+rend_data_service_create(const char *onion_address, const char *pk_digest,
+                         const uint8_t *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL);
+
+  if (pk_digest) {
+    memcpy(v2->rend_pk_digest, pk_digest, sizeof(v2->rend_pk_digest));
+  }
+  if (cookie) {
+    memcpy(rend_data->rend_cookie, cookie, sizeof(rend_data->rend_cookie));
+  }
+
+  strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+  v2->auth_type = auth_type;
+
+  return rend_data;
+}
+
+/* Allocate and initialize a rend_data_t object for a client request using the
+ * given arguments. Either an onion address or a descriptor ID is needed. Both
+ * can be given but in this case only the onion address will be used to make
+ * the descriptor fetch. The <b>cookie</b> is the rendezvous cookie and
+ * <b>auth_type</b> is which authentiation the service is configured with.
+ *
+ * Return a valid rend_data_t pointer or NULL on error meaning the
+ * descriptor IDs couldn't be computed from the given data. */
+rend_data_t *
+rend_data_client_create(const char *onion_address, const char *desc_id,
+                        const char *cookie, rend_auth_type_t auth_type)
+{
+  /* Create a rend_data_t object for version 2. */
+  rend_data_t *rend_data = rend_data_alloc(HS_VERSION_TWO);
+  rend_data_v2_t *v2= TO_REND_DATA_V2(rend_data);
+
+  /* We need at least one else the call is wrong. */
+  tor_assert(onion_address != NULL || desc_id != NULL);
+
+  if (cookie) {
+    memcpy(v2->descriptor_cookie, cookie, sizeof(v2->descriptor_cookie));
+  }
+  if (desc_id) {
+    memcpy(v2->desc_id_fetch, desc_id, sizeof(v2->desc_id_fetch));
+  }
+  if (onion_address) {
+    strlcpy(v2->onion_address, onion_address, sizeof(v2->onion_address));
+    if (compute_desc_id(rend_data) < 0) {
+      goto error;
+    }
+  }
+
+  v2->auth_type = auth_type;
+
+  return rend_data;
+
+ error:
+  rend_data_free(rend_data);
+  return NULL;
+}
+
+/* Return the onion address from the rend data. Depending on the version,
+ * the size of the address can vary but it's always NUL terminated. */
+const char *
+rend_data_get_address(const rend_data_t *rend_data)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    return TO_REND_DATA_V2(rend_data)->onion_address;
+  default:
+    /* We should always have a supported version. */
+    tor_assert(0);
+  }
+}
+
+/* Return the descriptor ID for a specific replica number from the rend
+ * data. The returned data is a binary digest and depending on the version its
+ * size can vary. The size of the descriptor ID is put in <b>len_out</b> if
+ * non NULL. */
+const char *
+rend_data_get_desc_id(const rend_data_t *rend_data, uint8_t replica,
+                      size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+    tor_assert(replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS);
+    if (len_out) {
+      *len_out = DIGEST_LEN;
+    }
+    return TO_REND_DATA_V2(rend_data)->descriptor_id[replica];
+  default:
+    /* We should always have a supported version. */
+    tor_assert(0);
+  }
+}
+
+/* Return the public key digest using the given <b>rend_data</b>. The size of
+ * the digest is put in <b>len_out</b> (if set) which can differ depending on
+ * the version. */
+const uint8_t *
+rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out)
+{
+  tor_assert(rend_data);
+
+  switch (rend_data->version) {
+  case HS_VERSION_TWO:
+  {
+    const rend_data_v2_t *v2_data = TO_REND_DATA_V2(rend_data);
+    if (len_out) {
+      *len_out = sizeof(v2_data->rend_pk_digest);
+    }
+    return (const uint8_t *) v2_data->rend_pk_digest;
+  }
+  default:
+    /* We should always have a supported version. */
+    tor_assert(0);
+  }
+}
+
+/* Default, minimum and maximum values for the maximum rendezvous failures
+ * consensus parameter. */
+#define MAX_REND_FAILURES_DEFAULT 2
+#define MAX_REND_FAILURES_MIN 1
+#define MAX_REND_FAILURES_MAX 10
+
+/** How many times will a hidden service operator attempt to connect to
+ * a requested rendezvous point before giving up? */
+int
+hs_get_service_max_rend_failures(void)
+{
+  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
+                                 MAX_REND_FAILURES_DEFAULT,
+                                 MAX_REND_FAILURES_MIN,
+                                 MAX_REND_FAILURES_MAX);
+}
+
diff --git a/src/or/hs_common.h b/src/or/hs_common.h
new file mode 100644
index 0000000000..7eef5fc97e
--- /dev/null
+++ b/src/or/hs_common.h
@@ -0,0 +1,87 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_common.h
+ * \brief Header file containing common data for the whole HS subsytem.
+ **/
+
+#ifndef TOR_HS_COMMON_H
+#define TOR_HS_COMMON_H
+
+#include "or.h"
+
+/* Protocol version 2. Use this instead of hardcoding "2" in the code base,
+ * this adds a clearer semantic to the value when used. */
+#define HS_VERSION_TWO 2
+/* Version 3 of the protocol (prop224). */
+#define HS_VERSION_THREE 3
+
+/** Try to maintain this many intro points per service by default. */
+#define NUM_INTRO_POINTS_DEFAULT 3
+/** Maximum number of intro points per service. */
+#define NUM_INTRO_POINTS_MAX 10
+/** Number of extra intro points we launch if our set of intro nodes is empty.
+ * See proposal 155, section 4. */
+#define NUM_INTRO_POINTS_EXTRA 2
+
+/** If we can't build our intro circuits, don't retry for this long. */
+#define INTRO_CIRC_RETRY_PERIOD (60*5)
+/** Don't try to build more than this many circuits before giving up for a
+ * while.*/
+#define MAX_INTRO_CIRCS_PER_PERIOD 10
+/** How many times will a hidden service operator attempt to connect to a
+ * requested rendezvous point before giving up? */
+#define MAX_REND_FAILURES 1
+/** How many seconds should we spend trying to connect to a requested
+ * rendezvous point before giving up? */
+#define MAX_REND_TIMEOUT 30
+
+/* String prefix for the signature of ESTABLISH_INTRO */
+#define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1"
+
+/* The default HS time period length */
+#define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
+/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
+/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */
+#define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */
+
+int hs_check_service_private_dir(const char *username, const char *path,
+                                 unsigned int dir_group_readable,
+                                 unsigned int create);
+int hs_get_service_max_rend_failures(void);
+
+void rend_data_free(rend_data_t *data);
+rend_data_t *rend_data_dup(const rend_data_t *data);
+rend_data_t *rend_data_client_create(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *cookie,
+                                     rend_auth_type_t auth_type);
+rend_data_t *rend_data_service_create(const char *onion_address,
+                                      const char *pk_digest,
+                                      const uint8_t *cookie,
+                                      rend_auth_type_t auth_type);
+const char *rend_data_get_address(const rend_data_t *rend_data);
+const char *rend_data_get_desc_id(const rend_data_t *rend_data,
+                                  uint8_t replica, size_t *len_out);
+const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data,
+                                       size_t *len_out);
+
+uint64_t hs_get_next_time_period_num(time_t now);
+
+#ifdef HS_COMMON_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC uint64_t get_time_period_length(void);
+STATIC uint64_t get_time_period_num(time_t now);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* HS_COMMON_PRIVATE */
+
+#endif /* TOR_HS_COMMON_H */
+
diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c
new file mode 100644
index 0000000000..8e10c0fff8
--- /dev/null
+++ b/src/or/hs_descriptor.c
@@ -0,0 +1,2363 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.c
+ * \brief Handle hidden service descriptor encoding/decoding.
+ *
+ * \details
+ * Here is a graphical depiction of an HS descriptor and its layers:
+ *
+ *      +------------------------------------------------------+
+ *      |DESCRIPTOR HEADER:                                    |
+ *      |  hs-descriptor 3                                     |
+ *      |  descriptor-lifetime 180                             |
+ *      |  ...                                                 |
+ *      |  superencrypted                                      |
+ *      |+---------------------------------------------------+ |
+ *      ||SUPERENCRYPTED LAYER (aka OUTER ENCRYPTED LAYER):  | |
+ *      ||  desc-auth-type x25519                            | |
+ *      ||  desc-auth-ephemeral-key                          | |
+ *      ||  auth-client                                      | |
+ *      ||  auth-client                                      | |
+ *      ||  ...                                              | |
+ *      ||  encrypted                                        | |
+ *      ||+-------------------------------------------------+| |
+ *      |||ENCRYPTED LAYER (aka INNER ENCRYPTED LAYER):     || |
+ *      |||  create2-formats                                || |
+ *      |||  intro-auth-required                            || |
+ *      |||  introduction-point                             || |
+ *      |||  introduction-point                             || |
+ *      |||  ...                                            || |
+ *      ||+-------------------------------------------------+| |
+ *      |+---------------------------------------------------+ |
+ *      +------------------------------------------------------+
+ *
+ * The DESCRIPTOR HEADER section is completely unencrypted and contains generic
+ * descriptor metadata.
+ *
+ * The SUPERENCRYPTED LAYER section is the first layer of encryption, and it's
+ * encrypted using the blinded public key of the hidden service to protect
+ * against entities who don't know its onion address. The clients of the hidden
+ * service know its onion address and blinded public key, whereas third-parties
+ * (like HSDirs) don't know it (except if it's a public hidden service).
+ *
+ * The ENCRYPTED LAYER section is the second layer of encryption, and it's
+ * encrypted using the client authorization key material (if those exist). When
+ * client authorization is enabled, this second layer of encryption protects
+ * the descriptor content from unauthorized entities. If client authorization
+ * is disabled, this second layer of encryption does not provide any extra
+ * security but is still present. The plaintext of this layer contains all the
+ * information required to connect to the hidden service like its list of
+ * introduction points.
+ **/
+
+/* For unit tests.*/
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "hs_descriptor.h"
+
+#include "or.h"
+#include "ed25519_cert.h" /* Trunnel interface. */
+#include "parsecommon.h"
+#include "rendcache.h"
+#include "hs_cache.h"
+#include "torcert.h" /* tor_cert_encode_ed22519() */
+
+/* Constant string value used for the descriptor format. */
+#define str_hs_desc "hs-descriptor"
+#define str_desc_cert "descriptor-signing-key-cert"
+#define str_rev_counter "revision-counter"
+#define str_superencrypted "superencrypted"
+#define str_encrypted "encrypted"
+#define str_signature "signature"
+#define str_lifetime "descriptor-lifetime"
+/* Constant string value for the encrypted part of the descriptor. */
+#define str_create2_formats "create2-formats"
+#define str_intro_auth_required "intro-auth-required"
+#define str_single_onion "single-onion-service"
+#define str_intro_point "introduction-point"
+#define str_ip_auth_key "auth-key"
+#define str_ip_enc_key "enc-key"
+#define str_ip_enc_key_cert "enc-key-cert"
+#define str_ip_legacy_key "legacy-key"
+#define str_ip_legacy_key_cert "legacy-key-cert"
+#define str_intro_point_start "\n" str_intro_point " "
+/* Constant string value for the construction to encrypt the encrypted data
+ * section. */
+#define str_enc_const_superencryption "hsdir-superencrypted-data"
+#define str_enc_const_encryption "hsdir-encrypted-data"
+/* Prefix required to compute/verify HS desc signatures */
+#define str_desc_sig_prefix "Tor onion service descriptor sig v3"
+#define str_desc_auth_type "desc-auth-type"
+#define str_desc_auth_key "desc-auth-ephemeral-key"
+#define str_desc_auth_client "auth-client"
+#define str_encrypted "encrypted"
+
+/* Authentication supported types. */
+static const struct {
+  hs_desc_auth_type_t type;
+  const char *identifier;
+} intro_auth_types[] = {
+  { HS_DESC_AUTH_ED25519, "ed25519" },
+  /* Indicate end of array. */
+  { 0, NULL }
+};
+
+/* Descriptor ruleset. */
+static token_rule_t hs_desc_v3_token_table[] = {
+  T1_START(str_hs_desc, R_HS_DESCRIPTOR, EQ(1), NO_OBJ),
+  T1(str_lifetime, R3_DESC_LIFETIME, EQ(1), NO_OBJ),
+  T1(str_desc_cert, R3_DESC_SIGNING_CERT, NO_ARGS, NEED_OBJ),
+  T1(str_rev_counter, R3_REVISION_COUNTER, EQ(1), NO_OBJ),
+  T1(str_superencrypted, R3_SUPERENCRYPTED, NO_ARGS, NEED_OBJ),
+  T1_END(str_signature, R3_SIGNATURE, EQ(1), NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the superencrypted section. */
+static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
+  T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ),
+  T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ),
+  T1N(str_desc_auth_client, R3_DESC_AUTH_CLIENT, GE(3), NO_OBJ),
+  T1(str_encrypted, R3_ENCRYPTED, NO_ARGS, NEED_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the encrypted section. */
+static token_rule_t hs_desc_encrypted_v3_token_table[] = {
+  T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ),
+  T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, ARGS, NO_OBJ),
+  T01(str_single_onion, R3_SINGLE_ONION_SERVICE, ARGS, NO_OBJ),
+  END_OF_TABLE
+};
+
+/* Descriptor ruleset for the introduction points section. */
+static token_rule_t hs_desc_intro_point_v3_token_table[] = {
+  T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ),
+  T1(str_ip_auth_key, R3_INTRO_AUTH_KEY, NO_ARGS, NEED_OBJ),
+  T1(str_ip_enc_key, R3_INTRO_ENC_KEY, GE(2), OBJ_OK),
+  T1(str_ip_enc_key_cert, R3_INTRO_ENC_KEY_CERT, ARGS, OBJ_OK),
+  T01(str_ip_legacy_key, R3_INTRO_LEGACY_KEY, ARGS, NEED_KEY_1024),
+  T01(str_ip_legacy_key_cert, R3_INTRO_LEGACY_KEY_CERT, ARGS, OBJ_OK),
+  END_OF_TABLE
+};
+
+/* Free a descriptor intro point object. */
+STATIC void
+desc_intro_point_free(hs_desc_intro_point_t *ip)
+{
+  if (!ip) {
+    return;
+  }
+  if (ip->link_specifiers) {
+    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
+                      ls, tor_free(ls));
+    smartlist_free(ip->link_specifiers);
+  }
+  tor_cert_free(ip->auth_key_cert);
+  tor_cert_free(ip->enc_key_cert);
+  if (ip->legacy.key) {
+    crypto_pk_free(ip->legacy.key);
+  }
+  if (ip->legacy.cert.encoded) {
+    tor_free(ip->legacy.cert.encoded);
+  }
+  tor_free(ip);
+}
+
+/* Free the content of the plaintext section of a descriptor. */
+STATIC void
+desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->superencrypted_blob) {
+    tor_free(desc->superencrypted_blob);
+  }
+  tor_cert_free(desc->signing_key_cert);
+
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Free the content of the encrypted section of a descriptor. */
+static void
+desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  if (desc->intro_auth_types) {
+    SMARTLIST_FOREACH(desc->intro_auth_types, char *, a, tor_free(a));
+    smartlist_free(desc->intro_auth_types);
+  }
+  if (desc->intro_points) {
+    SMARTLIST_FOREACH(desc->intro_points, hs_desc_intro_point_t *, ip,
+                      desc_intro_point_free(ip));
+    smartlist_free(desc->intro_points);
+  }
+  memwipe(desc, 0, sizeof(*desc));
+}
+
+/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
+ * We use SHA3-256 for the MAC computation.
+ * This function can't fail. */
+static void
+build_mac(const uint8_t *mac_key, size_t mac_key_len,
+          const uint8_t *salt, size_t salt_len,
+          const uint8_t *encrypted, size_t encrypted_len,
+          uint8_t *mac_out, size_t mac_len)
+{
+  crypto_digest_t *digest;
+
+  const uint64_t mac_len_netorder = tor_htonll(mac_key_len);
+  const uint64_t salt_len_netorder = tor_htonll(salt_len);
+
+  tor_assert(mac_key);
+  tor_assert(salt);
+  tor_assert(encrypted);
+  tor_assert(mac_out);
+
+  digest = crypto_digest256_new(DIGEST_SHA3_256);
+  /* As specified in section 2.5 of proposal 224, first add the mac key
+   * then add the salt first and then the encrypted section. */
+
+  crypto_digest_add_bytes(digest, (const char *) &mac_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) mac_key, mac_key_len);
+  crypto_digest_add_bytes(digest, (const char *) &salt_len_netorder, 8);
+  crypto_digest_add_bytes(digest, (const char *) salt, salt_len);
+  crypto_digest_add_bytes(digest, (const char *) encrypted, encrypted_len);
+  crypto_digest_get_digest(digest, (char *) mac_out, mac_len);
+  crypto_digest_free(digest);
+}
+
+/* Using a given decriptor object, build the secret input needed for the
+ * KDF and put it in the dst pointer which is an already allocated buffer
+ * of size dstlen. */
+static void
+build_secret_input(const hs_descriptor_t *desc, uint8_t *dst, size_t dstlen)
+{
+  size_t offset = 0;
+
+  tor_assert(desc);
+  tor_assert(dst);
+  tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN <= dstlen);
+
+  /* XXX use the destination length as the memcpy length */
+  /* Copy blinded public key. */
+  memcpy(dst, desc->plaintext_data.blinded_pubkey.pubkey,
+         sizeof(desc->plaintext_data.blinded_pubkey.pubkey));
+  offset += sizeof(desc->plaintext_data.blinded_pubkey.pubkey);
+  /* Copy subcredential. */
+  memcpy(dst + offset, desc->subcredential, sizeof(desc->subcredential));
+  offset += sizeof(desc->subcredential);
+  /* Copy revision counter value. */
+  set_uint64(dst + offset, tor_ntohll(desc->plaintext_data.revision_counter));
+  offset += sizeof(uint64_t);
+  tor_assert(HS_DESC_ENCRYPTED_SECRET_INPUT_LEN == offset);
+}
+
+/* Do the KDF construction and put the resulting data in key_out which is of
+ * key_out_len length. It uses SHAKE-256 as specified in the spec. */
+static void
+build_kdf_key(const hs_descriptor_t *desc,
+              const uint8_t *salt, size_t salt_len,
+              uint8_t *key_out, size_t key_out_len,
+              int is_superencrypted_layer)
+{
+  uint8_t secret_input[HS_DESC_ENCRYPTED_SECRET_INPUT_LEN];
+  crypto_xof_t *xof;
+
+  tor_assert(desc);
+  tor_assert(salt);
+  tor_assert(key_out);
+
+  /* Build the secret input for the KDF computation. */
+  build_secret_input(desc, secret_input, sizeof(secret_input));
+
+  xof = crypto_xof_new();
+  /* Feed our KDF. [SHAKE it like a polaroid picture --Yawning]. */
+  crypto_xof_add_bytes(xof, secret_input, sizeof(secret_input));
+  crypto_xof_add_bytes(xof, salt, salt_len);
+
+  /* Feed in the right string constant based on the desc layer */
+  if (is_superencrypted_layer) {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_superencryption,
+                         strlen(str_enc_const_superencryption));
+  } else {
+    crypto_xof_add_bytes(xof, (const uint8_t *) str_enc_const_encryption,
+                         strlen(str_enc_const_encryption));
+  }
+
+  /* Eat from our KDF. */
+  crypto_xof_squeeze_bytes(xof, key_out, key_out_len);
+  crypto_xof_free(xof);
+  memwipe(secret_input,  0, sizeof(secret_input));
+}
+
+/* Using the given descriptor and salt, run it through our KDF function and
+ * then extract a secret key in key_out, the IV in iv_out and MAC in mac_out.
+ * This function can't fail. */
+static void
+build_secret_key_iv_mac(const hs_descriptor_t *desc,
+                        const uint8_t *salt, size_t salt_len,
+                        uint8_t *key_out, size_t key_len,
+                        uint8_t *iv_out, size_t iv_len,
+                        uint8_t *mac_out, size_t mac_len,
+                        int is_superencrypted_layer)
+{
+  size_t offset = 0;
+  uint8_t kdf_key[HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN];
+
+  tor_assert(desc);
+  tor_assert(salt);
+  tor_assert(key_out);
+  tor_assert(iv_out);
+  tor_assert(mac_out);
+
+  build_kdf_key(desc, salt, salt_len, kdf_key, sizeof(kdf_key),
+                is_superencrypted_layer);
+  /* Copy the bytes we need for both the secret key and IV. */
+  memcpy(key_out, kdf_key, key_len);
+  offset += key_len;
+  memcpy(iv_out, kdf_key + offset, iv_len);
+  offset += iv_len;
+  memcpy(mac_out, kdf_key + offset, mac_len);
+  /* Extra precaution to make sure we are not out of bound. */
+  tor_assert((offset + mac_len) == sizeof(kdf_key));
+  memwipe(kdf_key, 0, sizeof(kdf_key));
+}
+
+/* === ENCODING === */
+
+/* Encode the given link specifier objects into a newly allocated string.
+ * This can't fail so caller can always assume a valid string being
+ * returned. */
+STATIC char *
+encode_link_specifiers(const smartlist_t *specs)
+{
+  char *encoded_b64 = NULL;
+  link_specifier_list_t *lslist = link_specifier_list_new();
+
+  tor_assert(specs);
+  /* No link specifiers is a code flow error, can't happen. */
+  tor_assert(smartlist_len(specs) > 0);
+  tor_assert(smartlist_len(specs) <= UINT8_MAX);
+
+  link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
+
+  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+                          spec) {
+    link_specifier_t *ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, spec->type);
+
+    switch (spec->type) {
+    case LS_IPV4:
+      link_specifier_set_un_ipv4_addr(ls,
+                                      tor_addr_to_ipv4h(&spec->u.ap.addr));
+      link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
+      /* Four bytes IPv4 and two bytes port. */
+      link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
+                                    sizeof(spec->u.ap.port));
+      break;
+    case LS_IPV6:
+    {
+      size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+      const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
+      uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+      memcpy(ipv6_array, in6_addr, addr_len);
+      link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
+      /* Sixteen bytes IPv6 and two bytes port. */
+      link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
+      break;
+    }
+    case LS_LEGACY_ID:
+    {
+      size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
+      uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
+      memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
+      link_specifier_set_ls_len(ls, legacy_id_len);
+      break;
+    }
+    default:
+      tor_assert(0);
+    }
+
+    link_specifier_list_add_spec(lslist, ls);
+  } SMARTLIST_FOREACH_END(spec);
+
+  {
+    uint8_t *encoded;
+    ssize_t encoded_len, encoded_b64_len, ret;
+
+    encoded_len = link_specifier_list_encoded_len(lslist);
+    tor_assert(encoded_len > 0);
+    encoded = tor_malloc_zero(encoded_len);
+    ret = link_specifier_list_encode(encoded, encoded_len, lslist);
+    tor_assert(ret == encoded_len);
+
+    /* Base64 encode our binary format. Add extra NUL byte for the base64
+     * encoded value. */
+    encoded_b64_len = base64_encode_size(encoded_len, 0) + 1;
+    encoded_b64 = tor_malloc_zero(encoded_b64_len);
+    ret = base64_encode(encoded_b64, encoded_b64_len, (const char *) encoded,
+                        encoded_len, 0);
+    tor_assert(ret == (encoded_b64_len - 1));
+    tor_free(encoded);
+  }
+
+  link_specifier_list_free(lslist);
+  return encoded_b64;
+}
+
+/* Encode an introduction point legacy key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_legacy_key(const hs_desc_intro_point_t *ip)
+{
+  char *key_str, b64_cert[256], *encoded = NULL;
+  size_t key_str_len;
+
+  tor_assert(ip);
+
+  /* Encode cross cert. */
+  if (base64_encode(b64_cert, sizeof(b64_cert),
+                    (const char *) ip->legacy.cert.encoded,
+                    ip->legacy.cert.len, BASE64_ENCODE_MULTILINE) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy crosscert.");
+    goto done;
+  }
+  /* Convert the encryption key to PEM format NUL terminated. */
+  if (crypto_pk_write_public_key_to_string(ip->legacy.key, &key_str,
+                                           &key_str_len) < 0) {
+    log_warn(LD_REND, "Unable to encode legacy encryption key.");
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s \n%s"  /* Newline is added by the call above. */
+               "%s\n"
+               "-----BEGIN CROSSCERT-----\n"
+               "%s"
+               "-----END CROSSCERT-----",
+               str_ip_legacy_key, key_str,
+               str_ip_legacy_key_cert, b64_cert);
+  tor_free(key_str);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point encryption key and certificate. Return a newly
+ * allocated string with it. On failure, return NULL. */
+static char *
+encode_enc_key(const hs_desc_intro_point_t *ip)
+{
+  char *encoded = NULL, *encoded_cert;
+  char key_b64[CURVE25519_BASE64_PADDED_LEN + 1];
+
+  tor_assert(ip);
+
+  /* Base64 encode the encryption key for the "enc-key" field. */
+  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
+    goto done;
+  }
+  if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
+    goto done;
+  }
+  tor_asprintf(&encoded,
+               "%s ntor %s\n"
+               "%s\n%s",
+               str_ip_enc_key, key_b64,
+               str_ip_enc_key_cert, encoded_cert);
+  tor_free(encoded_cert);
+
+ done:
+  return encoded;
+}
+
+/* Encode an introduction point object and return a newly allocated string
+ * with it. On failure, return NULL. */
+static char *
+encode_intro_point(const ed25519_public_key_t *sig_key,
+                   const hs_desc_intro_point_t *ip)
+{
+  char *encoded_ip = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(ip);
+  tor_assert(sig_key);
+
+  /* Encode link specifier. */
+  {
+    char *ls_str = encode_link_specifiers(ip->link_specifiers);
+    smartlist_add_asprintf(lines, "%s %s", str_intro_point, ls_str);
+    tor_free(ls_str);
+  }
+
+  /* Authentication key encoding. */
+  {
+    char *encoded_cert;
+    if (tor_cert_encode_ed22519(ip->auth_key_cert, &encoded_cert) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s\n%s", str_ip_auth_key, encoded_cert);
+    tor_free(encoded_cert);
+  }
+
+  /* Encryption key encoding. */
+  {
+    char *encoded_enc_key = encode_enc_key(ip);
+    if (encoded_enc_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_enc_key);
+    tor_free(encoded_enc_key);
+  }
+
+  /* Legacy key if any. */
+  if (ip->legacy.key != NULL) {
+    /* Strong requirement else the IP creation was badly done. */
+    tor_assert(ip->legacy.cert.encoded);
+    char *encoded_legacy_key = encode_legacy_key(ip);
+    if (encoded_legacy_key == NULL) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s", encoded_legacy_key);
+    tor_free(encoded_legacy_key);
+  }
+
+  /* Join them all in one blob of text. */
+  encoded_ip = smartlist_join_strings(lines, "\n", 1, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return encoded_ip;
+}
+
+/* Given a source length, return the new size including padding for the
+ * plaintext encryption. */
+static size_t
+compute_padded_plaintext_length(size_t plaintext_len)
+{
+  size_t plaintext_padded_len;
+  const int padding_block_length = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+
+  /* Make sure we won't overflow. */
+  tor_assert(plaintext_len <= (SIZE_T_CEILING - padding_block_length));
+
+  /* Get the extra length we need to add. For example, if srclen is 10200
+   * bytes, this will expand to (2 * 10k) == 20k thus an extra 9800 bytes. */
+  plaintext_padded_len = CEIL_DIV(plaintext_len, padding_block_length) *
+                         padding_block_length;
+  /* Can never be extra careful. Make sure we are _really_ padded. */
+  tor_assert(!(plaintext_padded_len % padding_block_length));
+  return plaintext_padded_len;
+}
+
+/* Given a buffer, pad it up to the encrypted section padding requirement. Set
+ * the newly allocated string in padded_out and return the length of the
+ * padded buffer. */
+STATIC size_t
+build_plaintext_padding(const char *plaintext, size_t plaintext_len,
+                        uint8_t **padded_out)
+{
+  size_t padded_len;
+  uint8_t *padded;
+
+  tor_assert(plaintext);
+  tor_assert(padded_out);
+
+  /* Allocate the final length including padding. */
+  padded_len = compute_padded_plaintext_length(plaintext_len);
+  tor_assert(padded_len >= plaintext_len);
+  padded = tor_malloc_zero(padded_len);
+
+  memcpy(padded, plaintext, plaintext_len);
+  *padded_out = padded;
+  return padded_len;
+}
+
+/* Using a key, IV and plaintext data of length plaintext_len, create the
+ * encrypted section by encrypting it and setting encrypted_out with the
+ * data. Return size of the encrypted data buffer. */
+static size_t
+build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext,
+                size_t plaintext_len, uint8_t **encrypted_out,
+                int is_superencrypted_layer)
+{
+  size_t encrypted_len;
+  uint8_t *padded_plaintext, *encrypted;
+  crypto_cipher_t *cipher;
+
+  tor_assert(key);
+  tor_assert(iv);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* If we are encrypting the middle layer of the descriptor, we need to first
+     pad the plaintext */
+  if (is_superencrypted_layer) {
+    encrypted_len = build_plaintext_padding(plaintext, plaintext_len,
+                                            &padded_plaintext);
+    /* Extra precautions that we have a valid padding length. */
+    tor_assert(!(encrypted_len % HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE));
+  } else { /* No padding required for inner layers */
+    padded_plaintext = tor_memdup(plaintext, plaintext_len);
+    encrypted_len = plaintext_len;
+  }
+
+  /* This creates a cipher for AES. It can't fail. */
+  cipher = crypto_cipher_new_with_iv_and_bits(key, iv,
+                                              HS_DESC_ENCRYPTED_BIT_SIZE);
+  /* We use a stream cipher so the encrypted length will be the same as the
+   * plaintext padded length. */
+  encrypted = tor_malloc_zero(encrypted_len);
+  /* This can't fail. */
+  crypto_cipher_encrypt(cipher, (char *) encrypted,
+                        (const char *) padded_plaintext, encrypted_len);
+  *encrypted_out = encrypted;
+  /* Cleanup. */
+  crypto_cipher_free(cipher);
+  tor_free(padded_plaintext);
+  return encrypted_len;
+}
+
+/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> to get the
+ * keys. Set encrypted_out with the encrypted data and return the length of
+ * it. <b>is_superencrypted_layer</b> is set if this is the outer encrypted
+ * layer of the descriptor. */
+static size_t
+encrypt_descriptor_data(const hs_descriptor_t *desc, const char *plaintext,
+                        char **encrypted_out, int is_superencrypted_layer)
+{
+  char *final_blob;
+  size_t encrypted_len, final_blob_len, offset = 0;
+  uint8_t *encrypted;
+  uint8_t salt[HS_DESC_ENCRYPTED_SALT_LEN];
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], mac[DIGEST256_LEN];
+
+  tor_assert(desc);
+  tor_assert(plaintext);
+  tor_assert(encrypted_out);
+
+  /* Get our salt. The returned bytes are already hashed. */
+  crypto_strongest_rand(salt, sizeof(salt));
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the encryption. */
+  build_secret_key_iv_mac(desc, salt, sizeof(salt),
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build the encrypted part that is do the actual encryption. */
+  encrypted_len = build_encrypted(secret_key, secret_iv, plaintext,
+                                  strlen(plaintext), &encrypted,
+                                  is_superencrypted_layer);
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  /* This construction is specified in section 2.5 of proposal 224. */
+  final_blob_len = sizeof(salt) + encrypted_len + DIGEST256_LEN;
+  final_blob = tor_malloc_zero(final_blob_len);
+
+  /* Build the MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, sizeof(salt),
+            encrypted, encrypted_len, mac, sizeof(mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+
+  /* The salt is the first value. */
+  memcpy(final_blob, salt, sizeof(salt));
+  offset = sizeof(salt);
+  /* Second value is the encrypted data. */
+  memcpy(final_blob + offset, encrypted, encrypted_len);
+  offset += encrypted_len;
+  /* Third value is the MAC. */
+  memcpy(final_blob + offset, mac, sizeof(mac));
+  offset += sizeof(mac);
+  /* Cleanup the buffers. */
+  memwipe(salt, 0, sizeof(salt));
+  memwipe(encrypted, 0, encrypted_len);
+  tor_free(encrypted);
+  /* Extra precaution. */
+  tor_assert(offset == final_blob_len);
+
+  *encrypted_out = final_blob;
+  return final_blob_len;
+}
+
+/* Create and return a string containing a fake client-auth entry. It's the
+ * responsibility of the caller to free the returned string. This function will
+ * never fail. */
+static char *
+get_fake_auth_client_str(void)
+{
+  char *auth_client_str = NULL;
+  /* We are gonna fill these arrays with fake base64 data. They are all double
+   * the size of their binary representation to fit the base64 overhead. */
+  char client_id_b64[8*2];
+  char iv_b64[16*2];
+  char encrypted_cookie_b64[16*2];
+  int retval;
+
+  /* This is a macro to fill a field with random data and then base64 it. */
+#define FILL_WITH_FAKE_DATA_AND_BASE64(field) STMT_BEGIN         \
+  crypto_rand((char *)field, sizeof(field));                     \
+  retval = base64_encode_nopad(field##_b64, sizeof(field##_b64), \
+                               field, sizeof(field));            \
+  tor_assert(retval > 0);                                        \
+  STMT_END
+
+  { /* Get those fakes! */
+    uint8_t client_id[8]; /* fake client-id */
+    uint8_t iv[16]; /* fake IV (initialization vector) */
+    uint8_t encrypted_cookie[16]; /* fake encrypted cookie */
+
+    FILL_WITH_FAKE_DATA_AND_BASE64(client_id);
+    FILL_WITH_FAKE_DATA_AND_BASE64(iv);
+    FILL_WITH_FAKE_DATA_AND_BASE64(encrypted_cookie);
+  }
+
+  /* Build the final string */
+  tor_asprintf(&auth_client_str, "%s %s %s %s", str_desc_auth_client,
+               client_id_b64, iv_b64, encrypted_cookie_b64);
+
+#undef FILL_WITH_FAKE_DATA_AND_BASE64
+
+  return auth_client_str;
+}
+
+/** How many lines of "client-auth" we want in our descriptors; fake or not. */
+#define CLIENT_AUTH_ENTRIES_BLOCK_SIZE 16
+
+/** Create the "client-auth" part of the descriptor and return a
+ *  newly-allocated string with it. It's the responsibility of the caller to
+ *  free the returned string. */
+static char *
+get_fake_auth_client_lines(void)
+{
+  /* XXX: Client authorization is still not implemented, so all this function
+     does is make fake clients */
+  int i = 0;
+  smartlist_t *auth_client_lines = smartlist_new();
+  char *auth_client_lines_str = NULL;
+
+  /* Make a line for each fake client */
+  const int num_fake_clients = CLIENT_AUTH_ENTRIES_BLOCK_SIZE;
+  for (i = 0; i < num_fake_clients; i++) {
+    char *auth_client_str = get_fake_auth_client_str();
+    tor_assert(auth_client_str);
+    smartlist_add(auth_client_lines, auth_client_str);
+  }
+
+  /* Join all lines together to form final string */
+  auth_client_lines_str = smartlist_join_strings(auth_client_lines,
+                                                 "\n", 1, NULL);
+  /* Cleanup the mess */
+  SMARTLIST_FOREACH(auth_client_lines, char *, a, tor_free(a));
+  smartlist_free(auth_client_lines);
+
+  return auth_client_lines_str;
+}
+
+/* Create the inner layer of the descriptor (which includes the intro points,
+ * etc.). Return a newly-allocated string with the layer plaintext, or NULL if
+ * an error occured. It's the responsibility of the caller to free the returned
+ * string. */
+static char *
+get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
+{
+  char *encoded_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* Build the start of the section prior to the introduction points. */
+  {
+    if (!desc->encrypted_data.create2_ntor) {
+      log_err(LD_BUG, "HS desc doesn't have recognized handshake type.");
+      goto err;
+    }
+    smartlist_add_asprintf(lines, "%s %d\n", str_create2_formats,
+                           ONION_HANDSHAKE_TYPE_NTOR);
+
+    if (desc->encrypted_data.intro_auth_types &&
+        smartlist_len(desc->encrypted_data.intro_auth_types)) {
+      /* Put the authentication-required line. */
+      char *buf = smartlist_join_strings(desc->encrypted_data.intro_auth_types,
+                                         " ", 0, NULL);
+      smartlist_add_asprintf(lines, "%s %s\n", str_intro_auth_required, buf);
+      tor_free(buf);
+    }
+
+    if (desc->encrypted_data.single_onion_service) {
+      smartlist_add_asprintf(lines, "%s\n", str_single_onion);
+    }
+  }
+
+  /* Build the introduction point(s) section. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    char *encoded_ip = encode_intro_point(&desc->plaintext_data.signing_pubkey,
+                                          ip);
+    if (encoded_ip == NULL) {
+      log_err(LD_BUG, "HS desc intro point is malformed.");
+      goto err;
+    }
+    smartlist_add(lines, encoded_ip);
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Build the entire encrypted data section into one encoded plaintext and
+   * then encrypt it. */
+  encoded_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+
+  return encoded_str;
+}
+
+/* Create the middle layer of the descriptor, which includes the client auth
+ * data and the encrypted inner layer (provided as a base64 string at
+ * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
+ * layer plaintext, or NULL if an error occured. It's the responsibility of the
+ * caller to free the returned string. */
+static char *
+get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
+                                    const char *layer2_b64_ciphertext)
+{
+  char *layer1_str = NULL;
+  smartlist_t *lines = smartlist_new();
+
+  /* XXX: Disclaimer: This function generates only _fake_ client auth
+   * data. Real client auth is not yet implemented, but client auth data MUST
+   * always be present in descriptors. In the future this function will be
+   * refactored to use real client auth data if they exist (#20700). */
+  (void) *desc;
+
+  /* Specify auth type */
+  smartlist_add_asprintf(lines, "%s %s\n", str_desc_auth_type, "x25519");
+
+  {  /* Create fake ephemeral x25519 key */
+    char fake_key_base64[CURVE25519_BASE64_PADDED_LEN + 1];
+    curve25519_keypair_t fake_x25519_keypair;
+    if (curve25519_keypair_generate(&fake_x25519_keypair, 0) < 0) {
+      goto done;
+    }
+    if (curve25519_public_to_base64(fake_key_base64,
+                                    &fake_x25519_keypair.pubkey) < 0) {
+      goto done;
+    }
+    smartlist_add_asprintf(lines, "%s %s\n",
+                           str_desc_auth_key, fake_key_base64);
+    /* No need to memwipe any of these fake keys. They will go unused. */
+  }
+
+  {  /* Create fake auth-client lines. */
+    char *auth_client_lines = get_fake_auth_client_lines();
+    tor_assert(auth_client_lines);
+    smartlist_add(lines, auth_client_lines);
+  }
+
+  /* create encrypted section */
+  {
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_encrypted, layer2_b64_ciphertext);
+  }
+
+  layer1_str = smartlist_join_strings(lines, "", 0, NULL);
+
+ done:
+  SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
+  smartlist_free(lines);
+
+  return layer1_str;
+}
+
+/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
+ * returning it. <b>desc</b> is provided to derive the encryption
+ * keys. <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the
+ * middle (superencrypted) layer of the descriptor. It's the responsibility of
+ * the caller to free the returned string. */
+static char *
+encrypt_desc_data_and_base64(const hs_descriptor_t *desc,
+                             const char *encoded_str,
+                             int is_superencrypted_layer)
+{
+  char *enc_b64;
+  ssize_t enc_b64_len, ret_len, enc_len;
+  char *encrypted_blob = NULL;
+
+  enc_len = encrypt_descriptor_data(desc, encoded_str, &encrypted_blob,
+                                    is_superencrypted_layer);
+  /* Get the encoded size plus a NUL terminating byte. */
+  enc_b64_len = base64_encode_size(enc_len, BASE64_ENCODE_MULTILINE) + 1;
+  enc_b64 = tor_malloc_zero(enc_b64_len);
+  /* Base64 the encrypted blob before returning it. */
+  ret_len = base64_encode(enc_b64, enc_b64_len, encrypted_blob, enc_len,
+                          BASE64_ENCODE_MULTILINE);
+  /* Return length doesn't count the NUL byte. */
+  tor_assert(ret_len == (enc_b64_len - 1));
+  tor_free(encrypted_blob);
+
+  return enc_b64;
+}
+
+/* Generate and encode the superencrypted portion of <b>desc</b>. This also
+ * involves generating the encrypted portion of the descriptor, and performing
+ * the superencryption. A newly allocated NUL-terminated string pointer
+ * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0
+ * on success else a negative value. */
+static int
+encode_superencrypted_data(const hs_descriptor_t *desc,
+                           char **encrypted_blob_out)
+{
+  int ret = -1;
+  char *layer2_str = NULL;
+  char *layer2_b64_ciphertext = NULL;
+  char *layer1_str = NULL;
+  char *layer1_b64_ciphertext = NULL;
+
+  tor_assert(desc);
+  tor_assert(encrypted_blob_out);
+
+  /* Func logic: We first create the inner layer of the descriptor (layer2).
+   * We then encrypt it and use it to create the middle layer of the descriptor
+   * (layer1).  Finally we superencrypt the middle layer and return it to our
+   * caller. */
+
+  /* Create inner descriptor layer */
+  layer2_str = get_inner_encrypted_layer_plaintext(desc);
+  if (!layer2_str) {
+    goto err;
+  }
+
+  /* Encrypt and b64 the inner layer */
+  layer2_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer2_str, 0);
+  if (!layer2_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Now create middle descriptor layer given the inner layer */
+  layer1_str = get_outer_encrypted_layer_plaintext(desc,layer2_b64_ciphertext);
+  if (!layer1_str) {
+    goto err;
+  }
+
+  /* Encrypt and base64 the middle layer */
+  layer1_b64_ciphertext = encrypt_desc_data_and_base64(desc, layer1_str, 1);
+  if (!layer1_b64_ciphertext) {
+    goto err;
+  }
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(layer1_str);
+  tor_free(layer2_str);
+  tor_free(layer2_b64_ciphertext);
+
+  *encrypted_blob_out = layer1_b64_ciphertext;
+  return ret;
+}
+
+/* Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
+ * newly allocated string of the encoded descriptor. On error, -1 is returned
+ * and encoded_out is untouched. */
+static int
+desc_encode_v3(const hs_descriptor_t *desc,
+               const ed25519_keypair_t *signing_kp, char **encoded_out)
+{
+  int ret = -1;
+  char *encoded_str = NULL;
+  size_t encoded_len;
+  smartlist_t *lines = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(signing_kp);
+  tor_assert(encoded_out);
+  tor_assert(desc->plaintext_data.version == 3);
+
+  /* Build the non-encrypted values. */
+  {
+    char *encoded_cert;
+    /* Encode certificate then create the first line of the descriptor. */
+    if (desc->plaintext_data.signing_key_cert->cert_type
+        != CERT_TYPE_SIGNING_HS_DESC) {
+      log_err(LD_BUG, "HS descriptor signing key has an unexpected cert type "
+              "(%d)", (int) desc->plaintext_data.signing_key_cert->cert_type);
+      goto err;
+    }
+    if (tor_cert_encode_ed22519(desc->plaintext_data.signing_key_cert,
+                                &encoded_cert) < 0) {
+      /* The function will print error logs. */
+      goto err;
+    }
+    /* Create the hs descriptor line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_hs_desc,
+                           desc->plaintext_data.version);
+    /* Add the descriptor lifetime line (in minutes). */
+    smartlist_add_asprintf(lines, "%s %" PRIu32, str_lifetime,
+                           desc->plaintext_data.lifetime_sec / 60);
+    /* Create the descriptor certificate line. */
+    smartlist_add_asprintf(lines, "%s\n%s", str_desc_cert, encoded_cert);
+    tor_free(encoded_cert);
+    /* Create the revision counter line. */
+    smartlist_add_asprintf(lines, "%s %" PRIu64, str_rev_counter,
+                           desc->plaintext_data.revision_counter);
+  }
+
+  /* Build the superencrypted data section. */
+  {
+    char *enc_b64_blob=NULL;
+    if (encode_superencrypted_data(desc, &enc_b64_blob) < 0) {
+      goto err;
+    }
+    smartlist_add_asprintf(lines,
+                           "%s\n"
+                           "-----BEGIN MESSAGE-----\n"
+                           "%s"
+                           "-----END MESSAGE-----",
+                           str_superencrypted, enc_b64_blob);
+    tor_free(enc_b64_blob);
+  }
+
+  /* Join all lines in one string so we can generate a signature and append
+   * it to the descriptor. */
+  encoded_str = smartlist_join_strings(lines, "\n", 1, &encoded_len);
+
+  /* Sign all fields of the descriptor with our short term signing key. */
+  {
+    ed25519_signature_t sig;
+    char ed_sig_b64[ED25519_SIG_BASE64_LEN + 1];
+    if (ed25519_sign_prefixed(&sig,
+                              (const uint8_t *) encoded_str, encoded_len,
+                              str_desc_sig_prefix, signing_kp) < 0) {
+      log_warn(LD_BUG, "Can't sign encoded HS descriptor!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
+      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
+      tor_free(encoded_str);
+      goto err;
+    }
+    /* Create the signature line. */
+    smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
+  }
+  /* Free previous string that we used so compute the signature. */
+  tor_free(encoded_str);
+  encoded_str = smartlist_join_strings(lines, "\n", 1, NULL);
+  *encoded_out = encoded_str;
+
+  if (strlen(encoded_str) >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_GENERAL, "We just made an HS descriptor that's too big (%d)."
+             "Failing.", (int)strlen(encoded_str));
+    tor_free(encoded_str);
+    goto err;
+  }
+
+  /* XXX: Trigger a control port event. */
+
+  /* Success! */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(lines, char *, l, tor_free(l));
+  smartlist_free(lines);
+  return ret;
+}
+
+/* === DECODING === */
+
+/* Given an encoded string of the link specifiers, return a newly allocated
+ * list of decoded link specifiers. Return NULL on error. */
+STATIC smartlist_t *
+decode_link_specifiers(const char *encoded)
+{
+  int decoded_len;
+  size_t encoded_len, i;
+  uint8_t *decoded;
+  smartlist_t *results = NULL;
+  link_specifier_list_t *specs = NULL;
+
+  tor_assert(encoded);
+
+  encoded_len = strlen(encoded);
+  decoded = tor_malloc(encoded_len);
+  decoded_len = base64_decode((char *) decoded, encoded_len, encoded,
+                              encoded_len);
+  if (decoded_len < 0) {
+    goto err;
+  }
+
+  if (link_specifier_list_parse(&specs, decoded,
+                                (size_t) decoded_len) < decoded_len) {
+    goto err;
+  }
+  tor_assert(specs);
+  results = smartlist_new();
+
+  for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
+    hs_desc_link_specifier_t *hs_spec;
+    link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
+    tor_assert(ls);
+
+    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
+    hs_spec->type = link_specifier_get_ls_type(ls);
+    switch (hs_spec->type) {
+    case LS_IPV4:
+      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
+                          link_specifier_get_un_ipv4_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
+      break;
+    case LS_IPV6:
+      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
+                               link_specifier_getarray_un_ipv6_addr(ls));
+      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
+      break;
+    case LS_LEGACY_ID:
+      /* Both are known at compile time so let's make sure they are the same
+       * else we can copy memory out of bound. */
+      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
+                 sizeof(hs_spec->u.legacy_id));
+      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
+             sizeof(hs_spec->u.legacy_id));
+      break;
+    default:
+      tor_free(hs_spec);
+      goto err;
+    }
+
+    smartlist_add(results, hs_spec);
+  }
+
+  goto done;
+ err:
+  if (results) {
+    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    smartlist_free(results);
+    results = NULL;
+  }
+ done:
+  link_specifier_list_free(specs);
+  tor_free(decoded);
+  return results;
+}
+
+/* Given a list of authentication types, decode it and put it in the encrypted
+ * data section. Return 1 if we at least know one of the type or 0 if we know
+ * none of them. */
+static int
+decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  int match = 0;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  desc->intro_auth_types = smartlist_new();
+  smartlist_split_string(desc->intro_auth_types, list, " ", 0, 0);
+
+  /* Validate the types that we at least know about one. */
+  SMARTLIST_FOREACH_BEGIN(desc->intro_auth_types, const char *, auth) {
+    for (int idx = 0; intro_auth_types[idx].identifier; idx++) {
+      if (!strncmp(auth, intro_auth_types[idx].identifier,
+                   strlen(intro_auth_types[idx].identifier))) {
+        match = 1;
+        break;
+      }
+    }
+  } SMARTLIST_FOREACH_END(auth);
+
+  return match;
+}
+
+/* Parse a space-delimited list of integers representing CREATE2 formats into
+ * the bitfield in hs_desc_encrypted_data_t. Ignore unrecognized values. */
+static void
+decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
+{
+  smartlist_t *tokens;
+
+  tor_assert(desc);
+  tor_assert(list);
+
+  tokens = smartlist_new();
+  smartlist_split_string(tokens, list, " ", 0, 0);
+
+  SMARTLIST_FOREACH_BEGIN(tokens, char *, s) {
+    int ok;
+    unsigned long type = tor_parse_ulong(s, 10, 1, UINT16_MAX, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_REND, "Unparseable value %s in create2 list", escaped(s));
+      continue;
+    }
+    switch (type) {
+    case ONION_HANDSHAKE_TYPE_NTOR:
+      desc->create2_ntor = 1;
+      break;
+    default:
+      /* We deliberately ignore unsupported handshake types */
+      continue;
+    }
+  } SMARTLIST_FOREACH_END(s);
+
+  SMARTLIST_FOREACH(tokens, char *, s, tor_free(s));
+  smartlist_free(tokens);
+}
+
+/* Given a certificate, validate the certificate for certain conditions which
+ * are if the given type matches the cert's one, if the signing key is
+ * included and if the that key was actually used to sign the certificate.
+ *
+ * Return 1 iff if all conditions pass or 0 if one of them fails. */
+STATIC int
+cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
+{
+  tor_assert(log_obj_type);
+
+  if (cert == NULL) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", log_obj_type);
+    goto err;
+  }
+  if (cert->cert_type != type) {
+    log_warn(LD_REND, "Invalid cert type %02x for %s.", cert->cert_type,
+             log_obj_type);
+    goto err;
+  }
+  /* All certificate must have its signing key included. */
+  if (!cert->signing_key_included) {
+    log_warn(LD_REND, "Signing key is NOT included for %s.", log_obj_type);
+    goto err;
+  }
+  /* The following will not only check if the signature matches but also the
+   * expiration date and overall validity. */
+  if (tor_cert_checksig(cert, &cert->signing_key, approx_time()) < 0) {
+    log_warn(LD_REND, "Invalid signature for %s.", log_obj_type);
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/* Given some binary data, try to parse it to get a certificate object. If we
+ * have a valid cert, validate it using the given wanted type. On error, print
+ * a log using the err_msg has the certificate identifier adding semantic to
+ * the log and cert_out is set to NULL. On success, 0 is returned and cert_out
+ * points to a newly allocated certificate object. */
+static int
+cert_parse_and_validate(tor_cert_t **cert_out, const char *data,
+                        size_t data_len, unsigned int cert_type_wanted,
+                        const char *err_msg)
+{
+  tor_cert_t *cert;
+
+  tor_assert(cert_out);
+  tor_assert(data);
+  tor_assert(err_msg);
+
+  /* Parse certificate. */
+  cert = tor_cert_parse((const uint8_t *) data, data_len);
+  if (!cert) {
+    log_warn(LD_REND, "Certificate for %s couldn't be parsed.", err_msg);
+    goto err;
+  }
+
+  /* Validate certificate. */
+  if (!cert_is_valid(cert, cert_type_wanted, err_msg)) {
+    goto err;
+  }
+
+  *cert_out = cert;
+  return 0;
+
+ err:
+  tor_cert_free(cert);
+  *cert_out = NULL;
+  return -1;
+}
+
+/* Return true iff the given length of the encrypted data of a descriptor
+ * passes validation. */
+STATIC int
+encrypted_data_length_is_valid(size_t len)
+{
+  /* Make sure there is enough data for the salt and the mac. The equality is
+     there to ensure that there is at least one byte of encrypted data. */
+  if (len <= HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN) {
+    log_warn(LD_REND, "Length of descriptor's encrypted data is too small. "
+                      "Got %lu but minimum value is %d",
+             (unsigned long)len, HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+    goto err;
+  }
+
+  return 1;
+ err:
+  return 0;
+}
+
+/** Decrypt an encrypted descriptor layer at <b>encrypted_blob</b> of size
+ *  <b>encrypted_blob_size</b>. Use the descriptor object <b>desc</b> to
+ *  generate the right decryption keys; set <b>decrypted_out</b> to the
+ *  plaintext. If <b>is_superencrypted_layer</b> is set, this is the outter
+ *  encrypted layer of the descriptor. */
+static size_t
+decrypt_desc_layer(const hs_descriptor_t *desc,
+                   const uint8_t *encrypted_blob,
+                   size_t encrypted_blob_size,
+                   int is_superencrypted_layer,
+                   char **decrypted_out)
+{
+  uint8_t *decrypted = NULL;
+  uint8_t secret_key[HS_DESC_ENCRYPTED_KEY_LEN], secret_iv[CIPHER_IV_LEN];
+  uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
+  const uint8_t *salt, *encrypted, *desc_mac;
+  size_t encrypted_len, result_len = 0;
+
+  tor_assert(decrypted_out);
+  tor_assert(desc);
+  tor_assert(encrypted_blob);
+
+  /* Construction is as follow: SALT | ENCRYPTED_DATA | MAC .
+   * Make sure we have enough space for all these things. */
+  if (!encrypted_data_length_is_valid(encrypted_blob_size)) {
+    goto err;
+  }
+
+  /* Start of the blob thus the salt. */
+  salt = encrypted_blob;
+
+  /* Next is the encrypted data. */
+  encrypted = encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN;
+  encrypted_len = encrypted_blob_size -
+    (HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN);
+  tor_assert(encrypted_len > 0); /* guaranteed by the check above */
+
+  /* And last comes the MAC. */
+  desc_mac = encrypted_blob + encrypted_blob_size - DIGEST256_LEN;
+
+  /* KDF construction resulting in a key from which the secret key, IV and MAC
+   * key are extracted which is what we need for the decryption. */
+  build_secret_key_iv_mac(desc, salt, HS_DESC_ENCRYPTED_SALT_LEN,
+                          secret_key, sizeof(secret_key),
+                          secret_iv, sizeof(secret_iv),
+                          mac_key, sizeof(mac_key),
+                          is_superencrypted_layer);
+
+  /* Build MAC. */
+  build_mac(mac_key, sizeof(mac_key), salt, HS_DESC_ENCRYPTED_SALT_LEN,
+            encrypted, encrypted_len, our_mac, sizeof(our_mac));
+  memwipe(mac_key, 0, sizeof(mac_key));
+  /* Verify MAC; MAC is H(mac_key || salt || encrypted)
+   *
+   * This is a critical check that is making sure the computed MAC matches the
+   * one in the descriptor. */
+  if (!tor_memeq(our_mac, desc_mac, sizeof(our_mac))) {
+    log_warn(LD_REND, "Encrypted service descriptor MAC check failed");
+    goto err;
+  }
+
+  {
+    /* Decrypt. Here we are assured that the encrypted length is valid for
+     * decryption. */
+    crypto_cipher_t *cipher;
+
+    cipher = crypto_cipher_new_with_iv_and_bits(secret_key, secret_iv,
+                                                HS_DESC_ENCRYPTED_BIT_SIZE);
+    /* Extra byte for the NUL terminated byte. */
+    decrypted = tor_malloc_zero(encrypted_len + 1);
+    crypto_cipher_decrypt(cipher, (char *) decrypted,
+                          (const char *) encrypted, encrypted_len);
+    crypto_cipher_free(cipher);
+  }
+
+  {
+    /* Adjust length to remove NUL padding bytes */
+    uint8_t *end = memchr(decrypted, 0, encrypted_len);
+    result_len = encrypted_len;
+    if (end) {
+      result_len = end - decrypted;
+    }
+  }
+
+  /* Make sure to NUL terminate the string. */
+  decrypted[encrypted_len] = '\0';
+  *decrypted_out = (char *) decrypted;
+  goto done;
+
+ err:
+  if (decrypted) {
+    tor_free(decrypted);
+  }
+  *decrypted_out = NULL;
+  result_len = 0;
+
+ done:
+  memwipe(secret_key, 0, sizeof(secret_key));
+  memwipe(secret_iv, 0, sizeof(secret_iv));
+  return result_len;
+}
+
+/* Basic validation that the superencrypted client auth portion of the
+ * descriptor is well-formed and recognized. Return True if so, otherwise
+ * return False. */
+static int
+superencrypted_auth_data_is_valid(smartlist_t *tokens)
+{
+  /* XXX: This is just basic validation for now. When we implement client auth,
+     we can refactor this function so that it actually parses and saves the
+     data. */
+
+  { /* verify desc auth type */
+    const directory_token_t *tok;
+    tok = find_by_keyword(tokens, R3_DESC_AUTH_TYPE);
+    tor_assert(tok->n_args >= 1);
+    if (strcmp(tok->args[0], "x25519")) {
+      log_warn(LD_DIR, "Unrecognized desc auth type");
+      return 0;
+    }
+  }
+
+  { /* verify desc auth key */
+    const directory_token_t *tok;
+    curve25519_public_key_t k;
+    tok = find_by_keyword(tokens, R3_DESC_AUTH_KEY);
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus desc auth key in HS desc");
+      return 0;
+    }
+  }
+
+  /* verify desc auth client items */
+  SMARTLIST_FOREACH_BEGIN(tokens, const directory_token_t *, tok) {
+    if (tok->tp == R3_DESC_AUTH_CLIENT) {
+      tor_assert(tok->n_args >= 3);
+    }
+  } SMARTLIST_FOREACH_END(tok);
+
+  return 1;
+}
+
+/* Parse <b>message</b>, the plaintext of the superencrypted portion of an HS
+ * descriptor. Set <b>encrypted_out</b> to the encrypted blob, and return its
+ * size */
+STATIC size_t
+decode_superencrypted(const char *message, size_t message_len,
+                     uint8_t **encrypted_out)
+{
+  int retval = 0;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len, tokens,
+                      hs_desc_superencrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Superencrypted portion is not parseable");
+    goto err;
+  }
+
+  /* Do some rudimentary validation of the authentication data */
+  if (!superencrypted_auth_data_is_valid(tokens)) {
+    log_warn(LD_REND, "Invalid auth data");
+    goto err;
+  }
+
+  /* Extract the encrypted data section. */
+  {
+    const directory_token_t *tok;
+    tok = find_by_keyword(tokens, R3_ENCRYPTED);
+    tor_assert(tok->object_body);
+    if (strcmp(tok->object_type, "MESSAGE") != 0) {
+      log_warn(LD_REND, "Desc superencrypted data section is invalid");
+      goto err;
+    }
+    /* Make sure the length of the encrypted blob is valid. */
+    if (!encrypted_data_length_is_valid(tok->object_size)) {
+      goto err;
+    }
+
+    /* Copy the encrypted blob to the descriptor object so we can handle it
+     * latter if needed. */
+    tor_assert(tok->object_size <= INT_MAX);
+    *encrypted_out = tor_memdup(tok->object_body, tok->object_size);
+    retval = (int) tok->object_size;
+  }
+
+ err:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return retval;
+}
+
+/* Decrypt both the superencrypted and the encrypted section of the descriptor
+ * using the given descriptor object <b>desc</b>. A newly allocated NUL
+ * terminated string is put in decrypted_out which contains the inner encrypted
+ * layer of the descriptor. Return the length of decrypted_out on success else
+ * 0 is returned and decrypted_out is set to NULL. */
+static size_t
+desc_decrypt_all(const hs_descriptor_t *desc, char **decrypted_out)
+{
+  size_t  decrypted_len = 0;
+  size_t encrypted_len = 0;
+  size_t superencrypted_len = 0;
+  char *superencrypted_plaintext = NULL;
+  uint8_t *encrypted_blob = NULL;
+
+  /** Function logic: This function takes us from the descriptor header to the
+   *  inner encrypted layer, by decrypting and decoding the middle descriptor
+   *  layer. In the end we return the contents of the inner encrypted layer to
+   *  our caller. */
+
+  /* 1. Decrypt middle layer of descriptor */
+  superencrypted_len = decrypt_desc_layer(desc,
+                                 desc->plaintext_data.superencrypted_blob,
+                                 desc->plaintext_data.superencrypted_blob_size,
+                                 1,
+                                 &superencrypted_plaintext);
+  if (!superencrypted_len) {
+    log_warn(LD_REND, "Decrypting superencrypted desc failed.");
+    goto err;
+  }
+  tor_assert(superencrypted_plaintext);
+
+  /* 2. Parse "superencrypted" */
+  encrypted_len = decode_superencrypted(superencrypted_plaintext,
+                                        superencrypted_len,
+                                        &encrypted_blob);
+  if (!encrypted_len) {
+    log_warn(LD_REND, "Decrypting encrypted desc failed.");
+    goto err;
+  }
+  tor_assert(encrypted_blob);
+
+  /* 3. Decrypt "encrypted" and set decrypted_out */
+  char *decrypted_desc;
+  decrypted_len = decrypt_desc_layer(desc,
+                                     encrypted_blob, encrypted_len,
+                                     0, &decrypted_desc);
+  if (!decrypted_len) {
+    log_warn(LD_REND, "Decrypting encrypted desc failed.");
+    goto err;
+  }
+  tor_assert(decrypted_desc);
+
+  *decrypted_out = decrypted_desc;
+
+ err:
+  tor_free(superencrypted_plaintext);
+  tor_free(encrypted_blob);
+
+  return decrypted_len;
+}
+
+/* Given the token tok for an intro point legacy key, the list of tokens, the
+ * introduction point ip being decoded and the descriptor desc from which it
+ * comes from, decode the legacy key and set the intro point object. Return 0
+ * on success else -1 on failure. */
+static int
+decode_intro_legacy_key(const directory_token_t *tok,
+                        smartlist_t *tokens,
+                        hs_desc_intro_point_t *ip,
+                        const hs_descriptor_t *desc)
+{
+  tor_assert(tok);
+  tor_assert(tokens);
+  tor_assert(ip);
+  tor_assert(desc);
+
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_REND, "Introduction point legacy key is invalid");
+    goto err;
+  }
+  ip->legacy.key = crypto_pk_dup_key(tok->key);
+  /* Extract the legacy cross certification cert which MUST be present if we
+   * have a legacy key. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY_CERT);
+  if (!tok) {
+    log_warn(LD_REND, "Introduction point legacy key cert is missing");
+    goto err;
+  }
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "CROSSCERT")) {
+    /* Info level because this might be an unknown field that we should
+     * ignore. */
+    log_info(LD_REND, "Introduction point legacy encryption key "
+                      "cross-certification has an unknown format.");
+    goto err;
+  }
+  /* Keep a copy of the certificate. */
+  ip->legacy.cert.encoded = tor_memdup(tok->object_body, tok->object_size);
+  ip->legacy.cert.len = tok->object_size;
+  /* The check on the expiration date is for the entire lifetime of a
+   * certificate which is 24 hours. However, a descriptor has a maximum
+   * lifetime of 12 hours meaning we have a 12h difference between the two
+   * which ultimately accomodate the clock skewed client. */
+  if (rsa_ed25519_crosscert_check(ip->legacy.cert.encoded,
+                                  ip->legacy.cert.len, ip->legacy.key,
+                                  &desc->plaintext_data.signing_pubkey,
+                                  approx_time() - HS_DESC_CERT_LIFETIME)) {
+    log_warn(LD_REND, "Unable to check cross-certification on the "
+                      "introduction point legacy encryption key.");
+    ip->cross_certified = 0;
+    goto err;
+  }
+
+  /* Success. */
+  return 0;
+ err:
+  return -1;
+}
+
+/* Given the start of a section and the end of it, decode a single
+ * introduction point from that section. Return a newly allocated introduction
+ * point object containing the decoded data. Return NULL if the section can't
+ * be decoded. */
+STATIC hs_desc_intro_point_t *
+decode_introduction_point(const hs_descriptor_t *desc, const char *start)
+{
+  hs_desc_intro_point_t *ip = NULL;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  const directory_token_t *tok;
+
+  tor_assert(desc);
+  tor_assert(start);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, start, start + strlen(start),
+                      tokens, hs_desc_intro_point_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Introduction point is not parseable");
+    goto err;
+  }
+
+  /* Ok we seem to have a well formed section containing enough tokens to
+   * parse. Allocate our IP object and try to populate it. */
+  ip = tor_malloc_zero(sizeof(hs_desc_intro_point_t));
+
+  /* "introduction-point" SP link-specifiers NL */
+  tok = find_by_keyword(tokens, R3_INTRODUCTION_POINT);
+  tor_assert(tok->n_args == 1);
+  ip->link_specifiers = decode_link_specifiers(tok->args[0]);
+  if (!ip->link_specifiers) {
+    log_warn(LD_REND, "Introduction point has invalid link specifiers");
+    goto err;
+  }
+
+  /* "auth-key" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_AUTH_KEY);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+    log_warn(LD_REND, "Unexpected object type for introduction auth key");
+    goto err;
+  }
+  /* Parse cert and do some validation. */
+  if (cert_parse_and_validate(&ip->auth_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_AUTH_HS_IP_KEY,
+                              "introduction point auth-key") < 0) {
+    goto err;
+  }
+  /* Validate authentication certificate with descriptor signing key. */
+  if (tor_cert_checksig(ip->auth_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid authentication key signature");
+    goto err;
+  }
+
+  /* Exactly one "enc-key" SP "ntor" SP key NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY);
+  if (!strcmp(tok->args[0], "ntor")) {
+    /* This field is using GE(2) so for possible forward compatibility, we
+     * accept more fields but must be at least 2. */
+    tor_assert(tok->n_args >= 2);
+
+    if (curve25519_public_from_base64(&ip->enc_key, tok->args[1]) < 0) {
+      log_warn(LD_REND, "Introduction point ntor enc-key is invalid");
+      goto err;
+    }
+  } else {
+    /* Unknown key type so we can't use that introduction point. */
+    log_warn(LD_REND, "Introduction point encryption key is unrecognized.");
+    goto err;
+  }
+
+  /* Exactly once "enc-key-cert" NL certificate NL */
+  tok = find_by_keyword(tokens, R3_INTRO_ENC_KEY_CERT);
+  tor_assert(tok->object_body);
+  /* Do the cross certification. */
+  if (strcmp(tok->object_type, "ED25519 CERT")) {
+      log_warn(LD_REND, "Introduction point ntor encryption key "
+                        "cross-certification has an unknown format.");
+      goto err;
+  }
+  if (cert_parse_and_validate(&ip->enc_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_CROSS_HS_IP_KEYS,
+                              "introduction point enc-key-cert") < 0) {
+    goto err;
+  }
+  if (tor_cert_checksig(ip->enc_key_cert,
+                        &desc->plaintext_data.signing_pubkey, 0) < 0) {
+    log_warn(LD_REND, "Invalid encryption key signature");
+    goto err;
+  }
+  /* It is successfully cross certified. Flag the object. */
+  ip->cross_certified = 1;
+
+  /* Do we have a "legacy-key" SP key NL ?*/
+  tok = find_opt_by_keyword(tokens, R3_INTRO_LEGACY_KEY);
+  if (tok) {
+    if (decode_intro_legacy_key(tok, tokens, ip, desc) < 0) {
+      goto err;
+    }
+  }
+
+  /* Introduction point has been parsed successfully. */
+  goto done;
+
+ err:
+  desc_intro_point_free(ip);
+  ip = NULL;
+
+ done:
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  smartlist_free(tokens);
+  if (area) {
+    memarea_drop_all(area);
+  }
+
+  return ip;
+}
+
+/* Given a descriptor string at <b>data</b>, decode all possible introduction
+ * points that we can find. Add the introduction point object to desc_enc as we
+ * find them. Return 0 on success.
+ *
+ * On error, a negative value is returned. It is possible that some intro
+ * point object have been added to the desc_enc, they should be considered
+ * invalid. One single bad encoded introduction point will make this function
+ * return an error. */
+STATIC int
+decode_intro_points(const hs_descriptor_t *desc,
+                    hs_desc_encrypted_data_t *desc_enc,
+                    const char *data)
+{
+  int retval = -1;
+  smartlist_t *chunked_desc = smartlist_new();
+  smartlist_t *intro_points = smartlist_new();
+
+  tor_assert(desc);
+  tor_assert(desc_enc);
+  tor_assert(data);
+  tor_assert(desc_enc->intro_points);
+
+  /* Take the desc string, and extract the intro point substrings out of it */
+  {
+    /* Split the descriptor string using the intro point header as delimiter */
+    smartlist_split_string(chunked_desc, data, str_intro_point_start, 0, 0);
+
+    /* Check if there are actually any intro points included. The first chunk
+     * should be other descriptor fields (e.g. create2-formats), so it's not an
+     * intro point. */
+    if (smartlist_len(chunked_desc) < 2) {
+      goto done;
+    }
+  }
+
+  /* Take the intro point substrings, and prepare them for parsing */
+  {
+    int i = 0;
+    /* Prepend the introduction-point header to all the chunks, since
+       smartlist_split_string() devoured it. */
+    SMARTLIST_FOREACH_BEGIN(chunked_desc, char *, chunk) {
+      /* Ignore first chunk. It's other descriptor fields. */
+      if (i++ == 0) {
+        continue;
+      }
+
+      smartlist_add_asprintf(intro_points, "%s %s", str_intro_point, chunk);
+    } SMARTLIST_FOREACH_END(chunk);
+  }
+
+  /* Parse the intro points! */
+  SMARTLIST_FOREACH_BEGIN(intro_points, const char *, intro_point) {
+    hs_desc_intro_point_t *ip = decode_introduction_point(desc, intro_point);
+    if (!ip) {
+      /* Malformed introduction point section. Stop right away, this
+       * descriptor shouldn't be used. */
+      goto err;
+    }
+    smartlist_add(desc_enc->intro_points, ip);
+  } SMARTLIST_FOREACH_END(intro_point);
+
+ done:
+  retval = 0;
+
+ err:
+  SMARTLIST_FOREACH(chunked_desc, char *, a, tor_free(a));
+  smartlist_free(chunked_desc);
+  SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
+  smartlist_free(intro_points);
+  return retval;
+}
+/* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
+ * descriptor in encoded_desc validates the descriptor content. */
+STATIC int
+desc_sig_is_valid(const char *b64_sig,
+                  const ed25519_public_key_t *signing_pubkey,
+                  const char *encoded_desc, size_t encoded_len)
+{
+  int ret = 0;
+  ed25519_signature_t sig;
+  const char *sig_start;
+
+  tor_assert(b64_sig);
+  tor_assert(signing_pubkey);
+  tor_assert(encoded_desc);
+  /* Verifying nothing won't end well :). */
+  tor_assert(encoded_len > 0);
+
+  /* Signature length check. */
+  if (strlen(b64_sig) != ED25519_SIG_BASE64_LEN) {
+    log_warn(LD_REND, "Service descriptor has an invalid signature length."
+                      "Exptected %d but got %lu",
+             ED25519_SIG_BASE64_LEN, (unsigned long) strlen(b64_sig));
+    goto err;
+  }
+
+  /* First, convert base64 blob to an ed25519 signature. */
+  if (ed25519_signature_from_base64(&sig, b64_sig) != 0) {
+    log_warn(LD_REND, "Service descriptor does not contain a valid "
+                      "signature");
+    goto err;
+  }
+
+  /* Find the start of signature. */
+  sig_start = tor_memstr(encoded_desc, encoded_len, "\n" str_signature " ");
+  /* Getting here means the token parsing worked for the signature so if we
+   * can't find the start of the signature, we have a code flow issue. */
+  if (!sig_start) {
+    log_warn(LD_GENERAL, "Malformed signature line. Rejecting.");
+    goto err;
+  }
+  /* Skip newline, it has to go in the signature check. */
+  sig_start++;
+
+  /* Validate signature with the full body of the descriptor. */
+  if (ed25519_checksig_prefixed(&sig,
+                                (const uint8_t *) encoded_desc,
+                                sig_start - encoded_desc,
+                                str_desc_sig_prefix,
+                                signing_pubkey) != 0) {
+    log_warn(LD_REND, "Invalid signature on service descriptor");
+    goto err;
+  }
+  /* Valid signature! All is good. */
+  ret = 1;
+
+ err:
+  return ret;
+}
+
+/* Decode descriptor plaintext data for version 3. Given a list of tokens, an
+ * allocated plaintext object that will be populated and the encoded
+ * descriptor with its length. The last one is needed for signature
+ * verification. Unknown tokens are simply ignored so this won't error on
+ * unknowns but requires that all v3 token be present and valid.
+ *
+ * Return 0 on success else a negative value. */
+static int
+desc_decode_plaintext_v3(smartlist_t *tokens,
+                         hs_desc_plaintext_data_t *desc,
+                         const char *encoded_desc, size_t encoded_len)
+{
+  int ok;
+  directory_token_t *tok;
+
+  tor_assert(tokens);
+  tor_assert(desc);
+  /* Version higher could still use this function to decode most of the
+   * descriptor and then they decode the extra part. */
+  tor_assert(desc->version >= 3);
+
+  /* Descriptor lifetime parsing. */
+  tok = find_by_keyword(tokens, R3_DESC_LIFETIME);
+  tor_assert(tok->n_args == 1);
+  desc->lifetime_sec = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor lifetime value is invalid");
+    goto err;
+  }
+  /* Put it from minute to second. */
+  desc->lifetime_sec *= 60;
+  if (desc->lifetime_sec > HS_DESC_MAX_LIFETIME) {
+    log_warn(LD_REND, "Service descriptor lifetime is too big. "
+                      "Got %" PRIu32 " but max is %d",
+             desc->lifetime_sec, HS_DESC_MAX_LIFETIME);
+    goto err;
+  }
+
+  /* Descriptor signing certificate. */
+  tok = find_by_keyword(tokens, R3_DESC_SIGNING_CERT);
+  tor_assert(tok->object_body);
+  /* Expecting a prop220 cert with the signing key extension, which contains
+   * the blinded public key. */
+  if (strcmp(tok->object_type, "ED25519 CERT") != 0) {
+    log_warn(LD_REND, "Service descriptor signing cert wrong type (%s)",
+             escaped(tok->object_type));
+    goto err;
+  }
+  if (cert_parse_and_validate(&desc->signing_key_cert, tok->object_body,
+                              tok->object_size, CERT_TYPE_SIGNING_HS_DESC,
+                              "service descriptor signing key") < 0) {
+    goto err;
+  }
+
+  /* Copy the public keys into signing_pubkey and blinded_pubkey */
+  memcpy(&desc->signing_pubkey, &desc->signing_key_cert->signed_key,
+         sizeof(ed25519_public_key_t));
+  memcpy(&desc->blinded_pubkey, &desc->signing_key_cert->signing_key,
+         sizeof(ed25519_public_key_t));
+
+  /* Extract revision counter value. */
+  tok = find_by_keyword(tokens, R3_REVISION_COUNTER);
+  tor_assert(tok->n_args == 1);
+  desc->revision_counter = tor_parse_uint64(tok->args[0], 10, 0,
+                                            UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor revision-counter is invalid");
+    goto err;
+  }
+
+  /* Extract the encrypted data section. */
+  tok = find_by_keyword(tokens, R3_SUPERENCRYPTED);
+  tor_assert(tok->object_body);
+  if (strcmp(tok->object_type, "MESSAGE") != 0) {
+    log_warn(LD_REND, "Service descriptor encrypted data section is invalid");
+    goto err;
+  }
+  /* Make sure the length of the encrypted blob is valid. */
+  if (!encrypted_data_length_is_valid(tok->object_size)) {
+    goto err;
+  }
+
+  /* Copy the encrypted blob to the descriptor object so we can handle it
+   * latter if needed. */
+  desc->superencrypted_blob = tor_memdup(tok->object_body, tok->object_size);
+  desc->superencrypted_blob_size = tok->object_size;
+
+  /* Extract signature and verify it. */
+  tok = find_by_keyword(tokens, R3_SIGNATURE);
+  tor_assert(tok->n_args == 1);
+  /* First arg here is the actual encoded signature. */
+  if (!desc_sig_is_valid(tok->args[0], &desc->signing_pubkey,
+                         encoded_desc, encoded_len)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  return -1;
+}
+
+/* Decode the version 3 encrypted section of the given descriptor desc. The
+ * desc_encrypted_out will be populated with the decoded data. Return 0 on
+ * success else -1. */
+static int
+desc_decode_encrypted_v3(const hs_descriptor_t *desc,
+                         hs_desc_encrypted_data_t *desc_encrypted_out)
+{
+  int result = -1;
+  char *message = NULL;
+  size_t message_len;
+  memarea_t *area = NULL;
+  directory_token_t *tok;
+  smartlist_t *tokens = NULL;
+
+  tor_assert(desc);
+  tor_assert(desc_encrypted_out);
+
+  /* Decrypt the superencrypted data that is located in the plaintext section
+   * in the descriptor as a blob of bytes. */
+  message_len = desc_decrypt_all(desc, &message);
+  if (!message_len) {
+    log_warn(LD_REND, "Service descriptor decryption failed.");
+    goto err;
+  }
+  tor_assert(message);
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  if (tokenize_string(area, message, message + message_len,
+                      tokens, hs_desc_encrypted_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Encrypted service descriptor is not parseable.");
+    goto err;
+  }
+
+  /* CREATE2 supported cell format. It's mandatory. */
+  tok = find_by_keyword(tokens, R3_CREATE2_FORMATS);
+  tor_assert(tok);
+  decode_create2_list(desc_encrypted_out, tok->args[0]);
+  /* Must support ntor according to the specification */
+  if (!desc_encrypted_out->create2_ntor) {
+    log_warn(LD_REND, "Service create2-formats does not include ntor.");
+    goto err;
+  }
+
+  /* Authentication type. It's optional but only once. */
+  tok = find_opt_by_keyword(tokens, R3_INTRO_AUTH_REQUIRED);
+  if (tok) {
+    if (!decode_auth_type(desc_encrypted_out, tok->args[0])) {
+      log_warn(LD_REND, "Service descriptor authentication type has "
+                        "invalid entry(ies).");
+      goto err;
+    }
+  }
+
+  /* Is this service a single onion service? */
+  tok = find_opt_by_keyword(tokens, R3_SINGLE_ONION_SERVICE);
+  if (tok) {
+    desc_encrypted_out->single_onion_service = 1;
+  }
+
+  /* Initialize the descriptor's introduction point list before we start
+   * decoding. Having 0 intro point is valid. Then decode them all. */
+  desc_encrypted_out->intro_points = smartlist_new();
+  if (decode_intro_points(desc, desc_encrypted_out, message) < 0) {
+    goto err;
+  }
+  /* Validation of maximum introduction points allowed. */
+  if (smartlist_len(desc_encrypted_out->intro_points) > MAX_INTRO_POINTS) {
+    log_warn(LD_REND, "Service descriptor contains too many introduction "
+                      "points. Maximum allowed is %d but we have %d",
+             MAX_INTRO_POINTS,
+             smartlist_len(desc_encrypted_out->intro_points));
+    goto err;
+  }
+
+  /* NOTE: Unknown fields are allowed because this function could be used to
+   * decode other descriptor version. */
+
+  result = 0;
+  goto done;
+
+ err:
+  tor_assert(result < 0);
+  desc_encrypted_data_free_contents(desc_encrypted_out);
+
+ done:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  if (message) {
+    tor_free(message);
+  }
+  return result;
+}
+
+/* Table of encrypted decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_encrypted_handlers[])(
+      const hs_descriptor_t *desc,
+      hs_desc_encrypted_data_t *desc_encrypted) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_encrypted_v3,
+};
+
+/* Decode the encrypted data section of the given descriptor and store the
+ * data in the given encrypted data object. Return 0 on success else a
+ * negative value on error. */
+int
+hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                         hs_desc_encrypted_data_t *desc_encrypted)
+{
+  int ret;
+  uint32_t version;
+
+  tor_assert(desc);
+  /* Ease our life a bit. */
+  version = desc->plaintext_data.version;
+  tor_assert(desc_encrypted);
+  /* Calling this function without an encrypted blob to parse is a code flow
+   * error. The plaintext parsing should never succeed in the first place
+   * without an encrypted section. */
+  tor_assert(desc->plaintext_data.superencrypted_blob);
+  /* Let's make sure we have a supported version as well. By correctly parsing
+   * the plaintext, this should not fail. */
+  if (BUG(!hs_desc_is_supported_version(version))) {
+    ret = -1;
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_encrypted_handlers) >= version);
+  tor_assert(decode_encrypted_handlers[version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_encrypted_handlers[version](desc, desc_encrypted);
+  if (ret < 0) {
+    goto err;
+  }
+
+ err:
+  return ret;
+}
+
+/* Table of plaintext decode function version specific. The function are
+ * indexed by the version number so v3 callback is at index 3 in the array. */
+static int
+  (*decode_plaintext_handlers[])(
+      smartlist_t *tokens,
+      hs_desc_plaintext_data_t *desc,
+      const char *encoded_desc,
+      size_t encoded_len) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_decode_plaintext_v3,
+};
+
+/* Fully decode the given descriptor plaintext and store the data in the
+ * plaintext data object. Returns 0 on success else a negative value. */
+int
+hs_desc_decode_plaintext(const char *encoded,
+                         hs_desc_plaintext_data_t *plaintext)
+{
+  int ok = 0, ret = -1;
+  memarea_t *area = NULL;
+  smartlist_t *tokens = NULL;
+  size_t encoded_len;
+  directory_token_t *tok;
+
+  tor_assert(encoded);
+  tor_assert(plaintext);
+
+  /* Check that descriptor is within size limits. */
+  encoded_len = strlen(encoded);
+  if (encoded_len >= hs_cache_get_max_descriptor_size()) {
+    log_warn(LD_REND, "Service descriptor is too big (%lu bytes)",
+             (unsigned long) encoded_len);
+    goto err;
+  }
+
+  area = memarea_new();
+  tokens = smartlist_new();
+  /* Tokenize the descriptor so we can start to parse it. */
+  if (tokenize_string(area, encoded, encoded + encoded_len, tokens,
+                      hs_desc_v3_token_table, 0) < 0) {
+    log_warn(LD_REND, "Service descriptor is not parseable");
+    goto err;
+  }
+
+  /* Get the version of the descriptor which is the first mandatory field of
+   * the descriptor. From there, we'll decode the right descriptor version. */
+  tok = find_by_keyword(tokens, R_HS_DESCRIPTOR);
+  tor_assert(tok->n_args == 1);
+  plaintext->version = (uint32_t) tor_parse_ulong(tok->args[0], 10, 0,
+                                                  UINT32_MAX, &ok, NULL);
+  if (!ok) {
+    log_warn(LD_REND, "Service descriptor has unparseable version %s",
+             escaped(tok->args[0]));
+    goto err;
+  }
+  if (!hs_desc_is_supported_version(plaintext->version)) {
+    log_warn(LD_REND, "Service descriptor has unsupported version %" PRIu32,
+             plaintext->version);
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(decode_plaintext_handlers) >= plaintext->version);
+  tor_assert(decode_plaintext_handlers[plaintext->version]);
+
+  /* Run the version specific plaintext decoder. */
+  ret = decode_plaintext_handlers[plaintext->version](tokens, plaintext,
+                                                      encoded, encoded_len);
+  if (ret < 0) {
+    goto err;
+  }
+  /* Success. Descriptor has been populated with the data. */
+  ret = 0;
+
+ err:
+  if (tokens) {
+    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+    smartlist_free(tokens);
+  }
+  if (area) {
+    memarea_drop_all(area);
+  }
+  return ret;
+}
+
+/* Fully decode an encoded descriptor and set a newly allocated descriptor
+ * object in desc_out. Subcredentials are used if not NULL else it's ignored.
+ *
+ * Return 0 on success. A negative value is returned on error and desc_out is
+ * set to NULL. */
+int
+hs_desc_decode_descriptor(const char *encoded,
+                          const uint8_t *subcredential,
+                          hs_descriptor_t **desc_out)
+{
+  int ret;
+  hs_descriptor_t *desc;
+
+  tor_assert(encoded);
+
+  desc = tor_malloc_zero(sizeof(hs_descriptor_t));
+
+  /* Subcredentials are optional. */
+  if (subcredential) {
+    memcpy(desc->subcredential, subcredential, sizeof(desc->subcredential));
+  }
+
+  ret = hs_desc_decode_plaintext(encoded, &desc->plaintext_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  ret = hs_desc_decode_encrypted(desc, &desc->encrypted_data);
+  if (ret < 0) {
+    goto err;
+  }
+
+  if (desc_out) {
+    *desc_out = desc;
+  } else {
+    hs_descriptor_free(desc);
+  }
+  return ret;
+
+ err:
+  hs_descriptor_free(desc);
+  if (desc_out) {
+    *desc_out = NULL;
+  }
+
+  tor_assert(ret < 0);
+  return ret;
+}
+
+/* Table of encode function version specific. The functions are indexed by the
+ * version number so v3 callback is at index 3 in the array. */
+static int
+  (*encode_handlers[])(
+      const hs_descriptor_t *desc,
+      const ed25519_keypair_t *signing_kp,
+      char **encoded_out) =
+{
+  /* v0 */ NULL, /* v1 */ NULL, /* v2 */ NULL,
+  desc_encode_v3,
+};
+
+/* Encode the given descriptor desc including signing with the given key pair
+ * signing_kp. On success, encoded_out points to a newly allocated NUL
+ * terminated string that contains the encoded descriptor as a string.
+ *
+ * Return 0 on success and encoded_out is a valid pointer. On error, -1 is
+ * returned and encoded_out is set to NULL. */
+int
+hs_desc_encode_descriptor(const hs_descriptor_t *desc,
+                          const ed25519_keypair_t *signing_kp,
+                          char **encoded_out)
+{
+  int ret = -1;
+  uint32_t version;
+
+  tor_assert(desc);
+  tor_assert(encoded_out);
+
+  /* Make sure we support the version of the descriptor format. */
+  version = desc->plaintext_data.version;
+  if (!hs_desc_is_supported_version(version)) {
+    goto err;
+  }
+  /* Extra precaution. Having no handler for the supported version should
+   * never happened else we forgot to add it but we bumped the version. */
+  tor_assert(ARRAY_LENGTH(encode_handlers) >= version);
+  tor_assert(encode_handlers[version]);
+
+  ret = encode_handlers[version](desc, signing_kp, encoded_out);
+  if (ret < 0) {
+    goto err;
+  }
+
+  /* Try to decode what we just encoded. Symmetry is nice! */
+  ret = hs_desc_decode_descriptor(*encoded_out, desc->subcredential, NULL);
+  if (BUG(ret < 0)) {
+    goto err;
+  }
+
+  return 0;
+
+ err:
+  *encoded_out = NULL;
+  return ret;
+}
+
+/* Free the descriptor plaintext data object. */
+void
+hs_desc_plaintext_data_free(hs_desc_plaintext_data_t *desc)
+{
+  desc_plaintext_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the descriptor encrypted data object. */
+void
+hs_desc_encrypted_data_free(hs_desc_encrypted_data_t *desc)
+{
+  desc_encrypted_data_free_contents(desc);
+  tor_free(desc);
+}
+
+/* Free the given descriptor object. */
+void
+hs_descriptor_free(hs_descriptor_t *desc)
+{
+  if (!desc) {
+    return;
+  }
+
+  desc_plaintext_data_free_contents(&desc->plaintext_data);
+  desc_encrypted_data_free_contents(&desc->encrypted_data);
+  tor_free(desc);
+}
+
+/* Return the size in bytes of the given plaintext data object. A sizeof() is
+ * not enough because the object contains pointers and the encrypted blob.
+ * This is particularly useful for our OOM subsystem that tracks the HSDir
+ * cache size for instance. */
+size_t
+hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data)
+{
+  tor_assert(data);
+  return (sizeof(*data) + sizeof(*data->signing_key_cert) +
+          data->superencrypted_blob_size);
+}
+
diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h
new file mode 100644
index 0000000000..136477ae3a
--- /dev/null
+++ b/src/or/hs_descriptor.h
@@ -0,0 +1,243 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_descriptor.h
+ * \brief Header file for hs_descriptor.c
+ **/
+
+#ifndef TOR_HS_DESCRIPTOR_H
+#define TOR_HS_DESCRIPTOR_H
+
+#include <stdint.h>
+
+#include "or.h"
+#include "address.h"
+#include "container.h"
+#include "crypto.h"
+#include "crypto_ed25519.h"
+#include "torcert.h"
+
+/* The earliest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MIN 3
+/* The latest descriptor format version we support. */
+#define HS_DESC_SUPPORTED_FORMAT_VERSION_MAX 3
+
+/* Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
+ * which is 720 minutes or 43200 seconds. */
+#define HS_DESC_MAX_LIFETIME (12 * 60 * 60)
+/* Lifetime of certificate in the descriptor. This defines the lifetime of the
+ * descriptor signing key and the cross certification cert of that key. */
+#define HS_DESC_CERT_LIFETIME (24 * 60 * 60)
+/* Length of the salt needed for the encrypted section of a descriptor. */
+#define HS_DESC_ENCRYPTED_SALT_LEN 16
+/* Length of the secret input needed for the KDF construction which derives
+ * the encryption key for the encrypted data section of the descriptor. This
+ * adds up to 68 bytes being the blinded key, hashed subcredential and
+ * revision counter. */
+#define HS_DESC_ENCRYPTED_SECRET_INPUT_LEN \
+  ED25519_PUBKEY_LEN + DIGEST256_LEN + sizeof(uint64_t)
+/* Length of the KDF output value which is the length of the secret key,
+ * the secret IV and MAC key length which is the length of H() output. */
+#define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \
+  CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN
+/* Pad plaintext of superencrypted data section before encryption so that its
+ * length is a multiple of this value. */
+#define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000
+/* Maximum length in bytes of a full hidden service descriptor. */
+#define HS_DESC_MAX_LEN 50000 /* 50kb max size */
+
+/* Key length for the descriptor symmetric encryption. As specified in the
+ * protocol, we use AES-256 for the encrypted section of the descriptor. The
+ * following is the length in bytes and the bit size. */
+#define HS_DESC_ENCRYPTED_KEY_LEN CIPHER256_KEY_LEN
+#define HS_DESC_ENCRYPTED_BIT_SIZE (HS_DESC_ENCRYPTED_KEY_LEN * 8)
+
+/* Type of authentication in the descriptor. */
+typedef enum {
+  HS_DESC_AUTH_ED25519 = 1
+} hs_desc_auth_type_t;
+
+/* Link specifier object that contains information on how to extend to the
+ * relay that is the address, port and handshake type. */
+typedef struct hs_desc_link_specifier_t {
+  /* Indicate the type of link specifier. See trunnel ed25519_cert
+   * specification. */
+  uint8_t type;
+
+  /* It's either an address/port or a legacy identity fingerprint. */
+  union {
+    /* IP address and port of the relay use to extend. */
+    tor_addr_port_t ap;
+    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
+    uint8_t legacy_id[DIGEST_LEN];
+  } u;
+} hs_desc_link_specifier_t;
+
+/* Introduction point information located in a descriptor. */
+typedef struct hs_desc_intro_point_t {
+  /* Link specifier(s) which details how to extend to the relay. This list
+   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+  smartlist_t *link_specifiers;
+
+  /* Authentication key used to establish the introduction point circuit and
+   * cross-certifies the blinded public key for the replica thus signed by
+   * the blinded key and in turn signs it. */
+  tor_cert_t *auth_key_cert;
+
+  /* Encryption key for the "ntor" type. */
+  curve25519_public_key_t enc_key;
+
+  /* Certificate cross certifying the descriptor signing key by the encryption
+   * curve25519 key. This certificate contains the signing key and is of type
+   * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */
+  tor_cert_t *enc_key_cert;
+
+  /* (Optional): If this introduction point is a legacy one that is version <=
+   * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able
+   * to relay the cells to the service correctly. */
+  struct {
+    /* RSA public key. */
+    crypto_pk_t *key;
+
+    /* Cross certified cert with the descriptor signing key (RSA->Ed). Because
+     * of the cross certification API, we need to keep the certificate binary
+     * blob and its length in order to properly encode it after. */
+    struct {
+      uint8_t *encoded;
+      size_t len;
+    } cert;
+  } legacy;
+
+  /* True iff the introduction point has passed the cross certification. Upon
+   * decoding an intro point, this must be true. */
+  unsigned int cross_certified : 1;
+} hs_desc_intro_point_t;
+
+/* The encrypted data section of a descriptor. Obviously the data in this is
+ * in plaintext but encrypted once encoded. */
+typedef struct hs_desc_encrypted_data_t {
+  /* Bitfield of CREATE2 cell supported formats. The only currently supported
+   * format is ntor. */
+  unsigned int create2_ntor : 1;
+
+  /* A list of authentication types that a client must at least support one
+   * in order to contact the service. Contains NULL terminated strings. */
+  smartlist_t *intro_auth_types;
+
+  /* Is this descriptor a single onion service? */
+  unsigned int single_onion_service : 1;
+
+  /* A list of intro points. Contains hs_desc_intro_point_t objects. */
+  smartlist_t *intro_points;
+} hs_desc_encrypted_data_t;
+
+/* Plaintext data that is unencrypted information of the descriptor. */
+typedef struct hs_desc_plaintext_data_t {
+  /* Version of the descriptor format. Spec specifies this field as a
+   * positive integer. */
+  uint32_t version;
+
+  /* The lifetime of the descriptor in seconds. */
+  uint32_t lifetime_sec;
+
+  /* Certificate with the short-term ed22519 descriptor signing key for the
+   * replica which is signed by the blinded public key for that replica. */
+  tor_cert_t *signing_key_cert;
+
+  /* Signing public key which is used to sign the descriptor. Same public key
+   * as in the signing key certificate. */
+  ed25519_public_key_t signing_pubkey;
+
+  /* Blinded public key used for this descriptor derived from the master
+   * identity key and generated for a specific replica number. */
+  ed25519_public_key_t blinded_pubkey;
+
+  /* Revision counter is incremented at each upload, regardless of whether
+   * the descriptor has changed. This avoids leaking whether the descriptor
+   * has changed. Spec specifies this as a 8 bytes positive integer. */
+  uint64_t revision_counter;
+
+  /* Decoding only: The b64-decoded superencrypted blob from the descriptor */
+  uint8_t *superencrypted_blob;
+
+  /* Decoding only: Size of the superencrypted_blob */
+  size_t superencrypted_blob_size;
+} hs_desc_plaintext_data_t;
+
+/* Service descriptor in its decoded form. */
+typedef struct hs_descriptor_t {
+  /* Contains the plaintext part of the descriptor. */
+  hs_desc_plaintext_data_t plaintext_data;
+
+  /* The following contains what's in the encrypted part of the descriptor.
+   * It's only encrypted in the encoded version of the descriptor thus the
+   * data contained in that object is in plaintext. */
+  hs_desc_encrypted_data_t encrypted_data;
+
+  /* Subcredentials of a service, used by the client and service to decrypt
+   * the encrypted data. */
+  uint8_t subcredential[DIGEST256_LEN];
+} hs_descriptor_t;
+
+/* Return true iff the given descriptor format version is supported. */
+static inline int
+hs_desc_is_supported_version(uint32_t version)
+{
+  if (version < HS_DESC_SUPPORTED_FORMAT_VERSION_MIN ||
+      version > HS_DESC_SUPPORTED_FORMAT_VERSION_MAX) {
+    return 0;
+  }
+  return 1;
+}
+
+/* Public API. */
+
+void hs_descriptor_free(hs_descriptor_t *desc);
+void hs_desc_plaintext_data_free(hs_desc_plaintext_data_t *desc);
+void hs_desc_encrypted_data_free(hs_desc_encrypted_data_t *desc);
+
+int hs_desc_encode_descriptor(const hs_descriptor_t *desc,
+                              const ed25519_keypair_t *signing_kp,
+                              char **encoded_out);
+
+int hs_desc_decode_descriptor(const char *encoded,
+                              const uint8_t *subcredential,
+                              hs_descriptor_t **desc_out);
+int hs_desc_decode_plaintext(const char *encoded,
+                             hs_desc_plaintext_data_t *plaintext);
+int hs_desc_decode_encrypted(const hs_descriptor_t *desc,
+                             hs_desc_encrypted_data_t *desc_out);
+
+size_t hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data);
+
+#ifdef HS_DESCRIPTOR_PRIVATE
+
+/* Encoding. */
+STATIC char *encode_link_specifiers(const smartlist_t *specs);
+STATIC size_t build_plaintext_padding(const char *plaintext,
+                                      size_t plaintext_len,
+                                      uint8_t **padded_out);
+/* Decoding. */
+STATIC smartlist_t *decode_link_specifiers(const char *encoded);
+STATIC hs_desc_intro_point_t *decode_introduction_point(
+                                const hs_descriptor_t *desc,
+                                const char *text);
+STATIC int decode_intro_points(const hs_descriptor_t *desc,
+                               hs_desc_encrypted_data_t *desc_enc,
+                               const char *data);
+STATIC int encrypted_data_length_is_valid(size_t len);
+STATIC int cert_is_valid(tor_cert_t *cert, uint8_t type,
+                         const char *log_obj_type);
+STATIC int desc_sig_is_valid(const char *b64_sig,
+                             const ed25519_public_key_t *signing_pubkey,
+                             const char *encoded_desc, size_t encoded_len);
+STATIC void desc_intro_point_free(hs_desc_intro_point_t *ip);
+STATIC size_t decode_superencrypted(const char *message, size_t message_len,
+                                   uint8_t **encrypted_out);
+STATIC void desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc);
+
+#endif /* HS_DESCRIPTOR_PRIVATE */
+
+#endif /* TOR_HS_DESCRIPTOR_H */
+
diff --git a/src/or/hs_intropoint.c b/src/or/hs_intropoint.c
new file mode 100644
index 0000000000..06f8a2c3ad
--- /dev/null
+++ b/src/or/hs_intropoint.c
@@ -0,0 +1,597 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.c
+ * \brief Implement next generation introductions point functionality
+ **/
+
+#define HS_INTROPOINT_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "config.h"
+#include "relay.h"
+#include "rendmid.h"
+#include "rephist.h"
+
+/* Trunnel */
+#include "ed25519_cert.h"
+#include "hs/cell_common.h"
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+
+#include "hs_circuitmap.h"
+#include "hs_intropoint.h"
+#include "hs_common.h"
+
+/** Extract the authentication key from an ESTABLISH_INTRO or INTRODUCE1 using
+ * the given <b>cell_type</b> from <b>cell</b> and place it in
+ * <b>auth_key_out</b>. */
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell)
+{
+  size_t auth_key_len;
+  const uint8_t *key_array;
+
+  tor_assert(auth_key_out);
+  tor_assert(cell);
+
+  switch (cell_type) {
+  case RELAY_COMMAND_ESTABLISH_INTRO:
+  {
+    const trn_cell_establish_intro_t *c_cell = cell;
+    key_array = trn_cell_establish_intro_getconstarray_auth_key(c_cell);
+    auth_key_len = trn_cell_establish_intro_getlen_auth_key(c_cell);
+    break;
+  }
+  case RELAY_COMMAND_INTRODUCE1:
+  {
+    const trn_cell_introduce1_t *c_cell = cell;
+    key_array = trn_cell_introduce1_getconstarray_auth_key(cell);
+    auth_key_len = trn_cell_introduce1_getlen_auth_key(c_cell);
+    break;
+  }
+  default:
+    /* Getting here is really bad as it means we got a unknown cell type from
+     * this file where every call has an hardcoded value. */
+    tor_assert(0); /* LCOV_EXCL_LINE */
+  }
+  tor_assert(key_array);
+  tor_assert(auth_key_len == sizeof(auth_key_out->pubkey));
+  memcpy(auth_key_out->pubkey, key_array, auth_key_len);
+}
+
+/** We received an ESTABLISH_INTRO <b>cell</b>. Verify its signature and MAC,
+ *  given <b>circuit_key_material</b>. Return 0 on success else -1 on error. */
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *cell,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len)
+{
+  /* We only reach this function if the first byte of the cell is 0x02 which
+   * means that auth_key_type is of ed25519 type, hence this check should
+   * always pass. See hs_intro_received_establish_intro().  */
+  if (BUG(cell->auth_key_type != HS_INTRO_AUTH_KEY_TYPE_ED25519)) {
+    return -1;
+  }
+
+  /* Make sure the auth key length is of the right size for this type. For
+   * EXTRA safety, we check both the size of the array and the length which
+   * must be the same. Safety first!*/
+  if (trn_cell_establish_intro_getlen_auth_key(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_establish_intro_get_auth_key_len(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "ESTABLISH_INTRO auth key length is invalid");
+    return -1;
+  }
+
+  const uint8_t *msg = cell->start_cell;
+
+  /* Verify the sig */
+  {
+    ed25519_signature_t sig_struct;
+    const uint8_t *sig_array =
+      trn_cell_establish_intro_getconstarray_sig(cell);
+
+    /* Make sure the signature length is of the right size. For EXTRA safety,
+     * we check both the size of the array and the length which must be the
+     * same. Safety first!*/
+    if (trn_cell_establish_intro_getlen_sig(cell) != sizeof(sig_struct.sig) ||
+        trn_cell_establish_intro_get_sig_len(cell) != sizeof(sig_struct.sig)) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO sig len is invalid");
+      return -1;
+    }
+    /* We are now sure that sig_len is of the right size. */
+    memcpy(sig_struct.sig, sig_array, cell->sig_len);
+
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO, cell);
+
+    const size_t sig_msg_len = cell->end_sig_fields - msg;
+    int sig_mismatch = ed25519_checksig_prefixed(&sig_struct,
+                                                 msg, sig_msg_len,
+                                                 ESTABLISH_INTRO_SIG_PREFIX,
+                                                 &auth_key);
+    if (sig_mismatch) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO signature not as expected");
+      return -1;
+    }
+  }
+
+  /* Verify the MAC */
+  {
+    const size_t auth_msg_len = cell->end_mac_fields - msg;
+    uint8_t mac[DIGEST256_LEN];
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        circuit_key_material, circuit_key_material_len,
+                        msg, auth_msg_len);
+    if (tor_memneq(mac, cell->handshake_mac, sizeof(mac))) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "ESTABLISH_INTRO handshake_auth not as expected");
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/* Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
+MOCK_IMPL(int,
+hs_intro_send_intro_established_cell,(or_circuit_t *circ))
+{
+  int ret;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_intro_established_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Build the cell payload. */
+  cell = trn_cell_intro_established_new();
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_intro_established_set_extensions(cell, ext);
+  /* Encode the cell to binary format. */
+  encoded_len = trn_cell_intro_established_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_intro_established_encode(encoded_cell, encoded_len,
+                                                cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRO_ESTABLISHED,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_intro_established_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
+ *  well-formed and passed our verifications. Perform appropriate actions to
+ *  establish an intro point. */
+static int
+handle_verified_establish_intro_cell(or_circuit_t *circ,
+                               const trn_cell_establish_intro_t *parsed_cell)
+{
+  /* Get the auth key of this intro point */
+  ed25519_public_key_t auth_key;
+  get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                         parsed_cell);
+
+  /* Then notify the hidden service that the intro point is established by
+     sending an INTRO_ESTABLISHED cell */
+  if (hs_intro_send_intro_established_cell(circ)) {
+    log_warn(LD_PROTOCOL, "Couldn't send INTRO_ESTABLISHED cell.");
+    return -1;
+  }
+
+  /* Associate intro point auth key with this circuit. */
+  hs_circuitmap_register_intro_circ_v3_relay_side(circ, &auth_key);
+  /* Repurpose this circuit into an intro circuit. */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  return 0;
+}
+
+/** We just received an ESTABLISH_INTRO cell in <b>circ</b> with payload in
+ *  <b>request</b>. Handle it by making <b>circ</b> an intro circuit. Return 0
+ *  if everything went well, or -1 if there were errors. */
+static int
+handle_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                       size_t request_len)
+{
+  int cell_ok, retval = -1;
+  trn_cell_establish_intro_t *parsed_cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  log_info(LD_REND, "Received an ESTABLISH_INTRO request on circuit %" PRIu32,
+           circ->p_circ_id);
+
+  /* Check that the circuit is in shape to become an intro point */
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
+    goto err;
+  }
+
+  /* Parse the cell */
+  ssize_t parsing_result = trn_cell_establish_intro_parse(&parsed_cell,
+                                                         request, request_len);
+  if (parsing_result < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s ESTABLISH_INTRO cell.",
+           parsing_result == -1 ? "invalid" : "truncated");
+    goto err;
+  }
+
+  cell_ok = verify_establish_intro_cell(parsed_cell,
+                                        (uint8_t *) circ->rend_circ_nonce,
+                                        sizeof(circ->rend_circ_nonce));
+  if (cell_ok < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Failed to verify ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* This cell is legit. Take the appropriate actions. */
+  cell_ok = handle_verified_establish_intro_cell(circ, parsed_cell);
+  if (cell_ok < 0) {
+    goto err;
+  }
+
+  /* We are done! */
+  retval = 0;
+  goto done;
+
+ err:
+  /* When sending the intro establish ack, on error the circuit can be marked
+   * as closed so avoid a double close. */
+  if (!TO_CIRCUIT(circ)->marked_for_close) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  }
+
+ done:
+  trn_cell_establish_intro_free(parsed_cell);
+  return retval;
+}
+
+/* Return True if circuit is suitable for being an intro circuit. */
+static int
+circuit_is_suitable_intro_point(const or_circuit_t *circ,
+                                const char *log_cell_type_str)
+{
+  tor_assert(circ);
+  tor_assert(log_cell_type_str);
+
+  /* Basic circuit state sanity checks. */
+  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-OR circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  if (circ->base_.n_chan) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s on non-edge circuit.", log_cell_type_str);
+    return 0;
+  }
+
+  /* Suitable. */
+  return 1;
+}
+
+/* Return True if circuit is suitable for being service-side intro circuit. */
+int
+hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ)
+{
+  return circuit_is_suitable_intro_point(circ, "ESTABLISH_INTRO");
+}
+
+/* We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
+ * a legacy or a next gen cell, and pass it to the appropriate handler. */
+int
+hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
+                            size_t request_len)
+{
+  tor_assert(circ);
+  tor_assert(request);
+
+  if (request_len == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Empty ESTABLISH_INTRO cell.");
+    goto err;
+  }
+
+  /* Using the first byte of the cell, figure out the version of
+   * ESTABLISH_INTRO and pass it to the appropriate cell handler */
+  const uint8_t first_byte = request[0];
+  switch (first_byte) {
+    case HS_INTRO_AUTH_KEY_TYPE_LEGACY0:
+    case HS_INTRO_AUTH_KEY_TYPE_LEGACY1:
+      return rend_mid_establish_intro_legacy(circ, request, request_len);
+    case HS_INTRO_AUTH_KEY_TYPE_ED25519:
+      return handle_establish_intro(circ, request, request_len);
+    default:
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unrecognized AUTH_KEY_TYPE %u.", first_byte);
+      goto err;
+  }
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
+/* Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
+ * value in <b>status</b>. Depending on the status, it can be ACK or a NACK.
+ * Return 0 on success else a negative value on error which will close the
+ * circuit. */
+static int
+send_introduce_ack_cell(or_circuit_t *circ, hs_intro_ack_status_t status)
+{
+  int ret = -1;
+  uint8_t *encoded_cell = NULL;
+  ssize_t encoded_len, result_len;
+  trn_cell_introduce_ack_t *cell;
+  trn_cell_extension_t *ext;
+
+  tor_assert(circ);
+
+  /* Setup the INTRODUCE_ACK cell. We have no extensions so the N_EXTENSIONS
+   * field is set to 0 by default with a new object. */
+  cell = trn_cell_introduce_ack_new();
+  ret = trn_cell_introduce_ack_set_status(cell, status);
+  /* We have no cell extensions in an INTRODUCE_ACK cell. */
+  ext = trn_cell_extension_new();
+  trn_cell_extension_set_num(ext, 0);
+  trn_cell_introduce_ack_set_extensions(cell, ext);
+  /* A wrong status is a very bad code flow error as this value is controlled
+   * by the code in this file and not an external input. This means we use a
+   * code that is not known by the trunnel ABI. */
+  tor_assert(ret == 0);
+  /* Encode the payload. We should never fail to get the encoded length. */
+  encoded_len = trn_cell_introduce_ack_encoded_len(cell);
+  tor_assert(encoded_len > 0);
+  encoded_cell = tor_malloc_zero(encoded_len);
+  result_len = trn_cell_introduce_ack_encode(encoded_cell, encoded_len, cell);
+  tor_assert(encoded_len == result_len);
+
+  ret = relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(circ),
+                                     RELAY_COMMAND_INTRODUCE_ACK,
+                                     (char *) encoded_cell, encoded_len,
+                                     NULL);
+  /* On failure, the above function will close the circuit. */
+  trn_cell_introduce_ack_free(cell);
+  tor_free(encoded_cell);
+  return ret;
+}
+
+/* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
+ * negative value for an invalid cell that should be NACKed. */
+STATIC int
+validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
+{
+  size_t legacy_key_id_len;
+  const uint8_t *legacy_key_id;
+
+  tor_assert(cell);
+
+  /* This code path SHOULD NEVER be reached if the cell is a legacy type so
+   * safety net here. The legacy ID must be zeroes in this case. */
+  legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
+  legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
+  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+    goto invalid;
+  }
+
+  /* The auth key of an INTRODUCE1 should be of type ed25519 thus leading to a
+   * known fixed length as well. */
+  if (trn_cell_introduce1_get_auth_key_type(cell) !=
+      HS_INTRO_AUTH_KEY_TYPE_ED25519) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key type. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_get_auth_key_len(cell) != ED25519_PUBKEY_LEN ||
+      trn_cell_introduce1_getlen_auth_key(cell) != ED25519_PUBKEY_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell auth key length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+  if (trn_cell_introduce1_getlen_encrypted(cell) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting invalid INTRODUCE1 cell encrypted length. "
+           "Responding with NACK.");
+    goto invalid;
+  }
+
+  return 0;
+ invalid:
+  return -1;
+}
+
+/* We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
+ * the payload in <b>request</b> of size <b>request_len</b>. Return 0 if
+ * everything went well, or -1 if an error occured. This function is in charge
+ * of sending back an INTRODUCE_ACK cell and will close client_circ on error.
+ */
+STATIC int
+handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
+                  size_t request_len)
+{
+  int ret = -1;
+  or_circuit_t *service_circ;
+  trn_cell_introduce1_t *parsed_cell;
+  hs_intro_ack_status_t status = HS_INTRO_ACK_STATUS_SUCCESS;
+
+  tor_assert(client_circ);
+  tor_assert(request);
+
+  /* Parse cell. Note that we can only parse the non encrypted section for
+   * which we'll use the authentication key to find the service introduction
+   * circuit and relay the cell on it. */
+  ssize_t cell_size = trn_cell_introduce1_parse(&parsed_cell, request,
+                                               request_len);
+  if (cell_size < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Rejecting %s INTRODUCE1 cell. Responding with NACK.",
+           cell_size == -1 ? "invalid" : "truncated");
+    /* Inform client that the INTRODUCE1 has a bad format. */
+    status = HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Once parsed validate the cell format. */
+  if (validate_introduce1_parsed_cell(parsed_cell) < 0) {
+    /* Inform client that the INTRODUCE1 has bad format. */
+    status = HS_INTRO_ACK_STATUS_BAD_FORMAT;
+    goto send_ack;
+  }
+
+  /* Find introduction circuit through our circuit map. */
+  {
+    ed25519_public_key_t auth_key;
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_INTRODUCE1, parsed_cell);
+    service_circ = hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    if (service_circ == NULL) {
+      char b64_key[ED25519_BASE64_LEN + 1];
+      ed25519_public_to_base64(b64_key, &auth_key);
+      log_info(LD_REND, "No intro circuit found for INTRODUCE1 cell "
+                        "with auth key %s from circuit %" PRIu32 ". "
+                        "Responding with NACK.",
+               safe_str(b64_key), client_circ->p_circ_id);
+      /* Inform the client that we don't know the requested service ID. */
+      status = HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+      goto send_ack;
+    }
+  }
+
+  /* Relay the cell to the service on its intro circuit with an INTRODUCE2
+   * cell which is the same exact payload. */
+  if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
+                                   RELAY_COMMAND_INTRODUCE2,
+                                   (char *) request, request_len, NULL)) {
+    log_warn(LD_PROTOCOL, "Unable to send INTRODUCE2 cell to the service.");
+    /* Inform the client that we can't relay the cell. */
+    status = HS_INTRO_ACK_STATUS_CANT_RELAY;
+    goto send_ack;
+  }
+
+  /* Success! Send an INTRODUCE_ACK success status onto the client circuit. */
+  status = HS_INTRO_ACK_STATUS_SUCCESS;
+  ret = 0;
+
+ send_ack:
+  /* Send INTRODUCE_ACK or INTRODUCE_NACK to client */
+  if (send_introduce_ack_cell(client_circ, status) < 0) {
+    log_warn(LD_PROTOCOL, "Unable to send an INTRODUCE ACK status %d "
+                          "to client.", status);
+    /* Circuit has been closed on failure of transmission. */
+    goto done;
+  }
+  if (status != HS_INTRO_ACK_STATUS_SUCCESS) {
+    /* We just sent a NACK that is a non success status code so close the
+     * circuit because it's not useful to keep it open. Remember, a client can
+     * only send one INTRODUCE1 cell on a circuit. */
+    circuit_mark_for_close(TO_CIRCUIT(client_circ), END_CIRC_REASON_INTERNAL);
+  }
+ done:
+  trn_cell_introduce1_free(parsed_cell);
+  return ret;
+}
+
+/* Identify if the encoded cell we just received is a legacy one or not. The
+ * <b>request</b> should be at least DIGEST_LEN bytes long. */
+STATIC int
+introduce1_cell_is_legacy(const uint8_t *request)
+{
+  tor_assert(request);
+
+  /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
+   * indicates a legacy cell (v2). */
+  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+    /* Legacy cell. */
+    return 1;
+  }
+  /* Not a legacy cell. */
+  return 0;
+}
+
+/* Return true iff the circuit <b>circ</b> is suitable for receiving an
+ * INTRODUCE1 cell. */
+STATIC int
+circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* Is this circuit an intro point circuit? */
+  if (!circuit_is_suitable_intro_point(circ, "INTRODUCE1")) {
+    return 0;
+  }
+
+  if (circ->already_received_introduce1) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Blocking multiple introductions on the same circuit. "
+           "Someone might be trying to attack a hidden service through "
+           "this relay.");
+    return 0;
+  }
+
+  return 1;
+}
+
+/* We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
+ * it is and pass it to the appropriate handler. Return 0 on success else a
+ * negative value and the circuit is closed. */
+int
+hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                             size_t request_len)
+{
+  int ret;
+
+  tor_assert(circ);
+  tor_assert(request);
+
+  /* A cell that can't hold a DIGEST_LEN is invalid as we need to check if
+   * it's a legacy cell or not using the first DIGEST_LEN bytes. */
+  if (request_len < DIGEST_LEN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Invalid INTRODUCE1 cell length.");
+    goto err;
+  }
+
+  /* Make sure we have a circuit that can have an INTRODUCE1 cell on it. */
+  if (!circuit_is_suitable_for_introduce1(circ)) {
+    /* We do not send a NACK because the circuit is not suitable for any kind
+     * of response or transmission as it's a violation of the protocol. */
+    goto err;
+  }
+  /* Mark the circuit that we got this cell. None are allowed after this as a
+   * DoS mitigation since one circuit with one client can hammer a service. */
+  circ->already_received_introduce1 = 1;
+
+  /* We are sure here to have at least DIGEST_LEN bytes. */
+  if (introduce1_cell_is_legacy(request)) {
+    /* Handle a legacy cell. */
+    ret = rend_mid_introduce_legacy(circ, request, request_len);
+  } else {
+    /* Handle a non legacy cell. */
+    ret = handle_introduce1(circ, request, request_len);
+  }
+  return ret;
+
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
+  return -1;
+}
+
diff --git a/src/or/hs_intropoint.h b/src/or/hs_intropoint.h
new file mode 100644
index 0000000000..163ed810e7
--- /dev/null
+++ b/src/or/hs_intropoint.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_intropoint.h
+ * \brief Header file for hs_intropoint.c.
+ **/
+
+#ifndef TOR_HS_INTRO_H
+#define TOR_HS_INTRO_H
+
+/* Authentication key type in an ESTABLISH_INTRO cell. */
+enum hs_intro_auth_key_type {
+  HS_INTRO_AUTH_KEY_TYPE_LEGACY0 = 0x00,
+  HS_INTRO_AUTH_KEY_TYPE_LEGACY1 = 0x01,
+  HS_INTRO_AUTH_KEY_TYPE_ED25519 = 0x02,
+};
+
+/* INTRODUCE_ACK status code. */
+typedef enum {
+  HS_INTRO_ACK_STATUS_SUCCESS    = 0x0000,
+  HS_INTRO_ACK_STATUS_UNKNOWN_ID = 0x0001,
+  HS_INTRO_ACK_STATUS_BAD_FORMAT = 0x0002,
+  HS_INTRO_ACK_STATUS_CANT_RELAY = 0x0003,
+} hs_intro_ack_status_t;
+
+int hs_intro_received_establish_intro(or_circuit_t *circ,
+                                      const uint8_t *request,
+                                      size_t request_len);
+int hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
+                                 size_t request_len);
+
+MOCK_DECL(int, hs_intro_send_intro_established_cell,(or_circuit_t *circ));
+
+/* also used by rendservice.c */
+int hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ);
+
+#ifdef HS_INTROPOINT_PRIVATE
+
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+
+STATIC int
+verify_establish_intro_cell(const trn_cell_establish_intro_t *out,
+                            const uint8_t *circuit_key_material,
+                            size_t circuit_key_material_len);
+
+STATIC void
+get_auth_key_from_cell(ed25519_public_key_t *auth_key_out,
+                       unsigned int cell_type, const void *cell);
+
+STATIC int introduce1_cell_is_legacy(const uint8_t *request);
+STATIC int handle_introduce1(or_circuit_t *client_circ,
+                             const uint8_t *request, size_t request_len);
+STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
+STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+
+#endif /* HS_INTROPOINT_PRIVATE */
+
+#endif /* TOR_HS_INTRO_H */
+
diff --git a/src/or/hs_ntor.c b/src/or/hs_ntor.c
new file mode 100644
index 0000000000..119899817e
--- /dev/null
+++ b/src/or/hs_ntor.c
@@ -0,0 +1,626 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** \file hs_ntor.c
+ *  \brief Implements the ntor variant used in Tor hidden services.
+ *
+ *  \details
+ *  This module handles the variant of the ntor handshake that is documented in
+ *  section [NTOR-WITH-EXTRA-DATA] of rend-spec-ng.txt .
+ *
+ *  The functions in this file provide an API that should be used when sending
+ *  or receiving INTRODUCE1/RENDEZVOUS1 cells to generate the various key
+ *  material required to create and handle those cells.
+ *
+ *  In the case of INTRODUCE1 it provides encryption and MAC keys to
+ *  encode/decode the encrypted blob (see hs_ntor_intro_cell_keys_t). The
+ *  relevant pub functions are hs_ntor_{client,service}_get_introduce1_keys().
+ *
+ *  In the case of RENDEZVOUS1 it calculates the MAC required to authenticate
+ *  the cell, and also provides the key seed that is used to derive the crypto
+ *  material for rendezvous encryption (see hs_ntor_rend_cell_keys_t). The
+ *  relevant pub functions are hs_ntor_{client,service}_get_rendezvous1_keys().
+ *  It also provides a function (hs_ntor_circuit_key_expansion()) that does the
+ *  rendezvous key expansion to setup end-to-end rend circuit keys.
+ */
+
+#include "or.h"
+#include "hs_ntor.h"
+
+/* String constants used by the ntor HS protocol */
+#define PROTOID "tor-hs-ntor-curve25519-sha3-256-1"
+#define PROTOID_LEN (sizeof(PROTOID) - 1)
+#define SERVER_STR "Server"
+#define SERVER_STR_LEN (sizeof(SERVER_STR) - 1)
+
+/* Protocol-specific tweaks to our crypto inputs */
+#define T_HSENC PROTOID ":hs_key_extract"
+#define T_HSENC_LEN (sizeof(T_HSENC) - 1)
+#define T_HSVERIFY PROTOID ":hs_verify"
+#define T_HSMAC PROTOID ":hs_mac"
+#define M_HSEXPAND PROTOID ":hs_key_expand"
+#define M_HSEXPAND_LEN (sizeof(M_HSEXPAND) - 1)
+
+/************************* Helper functions: *******************************/
+
+/** Helper macro: copy <b>len</b> bytes from <b>inp</b> to <b>ptr</b> and
+ *advance <b>ptr</b> by the number of bytes copied. Stolen from onion_ntor.c */
+#define APPEND(ptr, inp, len)                   \
+  STMT_BEGIN {                                  \
+    memcpy(ptr, (inp), (len));                  \
+    ptr += len;                                 \
+  } STMT_END
+
+/* Length of EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID */
+#define REND_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN * 2 + \
+  ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN)
+/* Length of auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server" */
+#define REND_AUTH_INPUT_LEN (DIGEST256_LEN + ED25519_PUBKEY_LEN + \
+  CURVE25519_PUBKEY_LEN * 3 + PROTOID_LEN + SERVER_STR_LEN)
+
+/** Helper function: Compute the last part of the HS ntor handshake which
+ *  derives key material necessary to create and handle RENDEZVOUS1
+ *  cells. Function used by both client and service. The actual calculations is
+ *  as follows:
+ *
+ *    NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *    verify = MAC(rend_secret_hs_input, t_hsverify)
+ *    auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *    auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ *  where in the above, AUTH_KEY is <b>intro_auth_pubkey</b>, B is
+ *  <b>intro_enc_pubkey</b>, Y is <b>service_ephemeral_rend_pubkey</b>, and X
+ *  is <b>client_ephemeral_enc_pubkey</b>. The provided
+ *  <b>rend_secret_hs_input</b> is of size REND_SECRET_HS_INPUT_LEN.
+ *
+ *  The final results of NTOR_KEY_SEED and auth_input_mac are placed in
+ *  <b>hs_ntor_rend_cell_keys_out</b>. Return 0 if everything went fine. */
+static int
+get_rendezvous1_key_material(const uint8_t *rend_secret_hs_input,
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t ntor_key_seed[DIGEST256_LEN];
+  uint8_t ntor_verify[DIGEST256_LEN];
+  uint8_t rend_auth_input[REND_AUTH_INPUT_LEN];
+  uint8_t rend_cell_auth[DIGEST256_LEN];
+  uint8_t *ptr;
+
+  /* Let's build NTOR_KEY_SEED */
+  crypto_mac_sha3_256(ntor_key_seed, sizeof(ntor_key_seed),
+                      rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN,
+                      (const uint8_t *)T_HSENC, strlen(T_HSENC));
+  bad |= safe_mem_is_zero(ntor_key_seed, DIGEST256_LEN);
+
+  /* Let's build ntor_verify */
+  crypto_mac_sha3_256(ntor_verify, sizeof(ntor_verify),
+                      rend_secret_hs_input, REND_SECRET_HS_INPUT_LEN,
+                      (const uint8_t *)T_HSVERIFY, strlen(T_HSVERIFY));
+  bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN);
+
+  /* Let's build auth_input: */
+  ptr = rend_auth_input;
+  /* Append ntor_verify */
+  APPEND(ptr, ntor_verify, sizeof(ntor_verify));
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append Y */
+  APPEND(ptr,
+         service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr,
+         client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  /* Append "Server" */
+  APPEND(ptr, SERVER_STR, strlen(SERVER_STR));
+  tor_assert(ptr == rend_auth_input + sizeof(rend_auth_input));
+
+  /* Let's build auth_input_mac that goes in RENDEZVOUS1 cell */
+  crypto_mac_sha3_256(rend_cell_auth, sizeof(rend_cell_auth),
+                      rend_auth_input, sizeof(rend_auth_input),
+                      (const uint8_t *)T_HSMAC, strlen(T_HSMAC));
+  bad |= safe_mem_is_zero(ntor_verify, DIGEST256_LEN);
+
+  { /* Get the computed RENDEZVOUS1 material! */
+    memcpy(&hs_ntor_rend_cell_keys_out->rend_cell_auth_mac,
+           rend_cell_auth, DIGEST256_LEN);
+    memcpy(&hs_ntor_rend_cell_keys_out->ntor_key_seed,
+           ntor_key_seed, DIGEST256_LEN);
+  }
+
+  memwipe(rend_cell_auth, 0, sizeof(rend_cell_auth));
+  memwipe(rend_auth_input, 0, sizeof(rend_auth_input));
+  memwipe(ntor_key_seed, 0, sizeof(ntor_key_seed));
+
+  return bad;
+}
+
+/** Length of secret_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID */
+#define INTRO_SECRET_HS_INPUT_LEN (CURVE25519_OUTPUT_LEN +ED25519_PUBKEY_LEN +\
+  CURVE25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN + PROTOID_LEN)
+/* Length of info = m_hsexpand | subcredential */
+#define INFO_BLOB_LEN (M_HSEXPAND_LEN + DIGEST256_LEN)
+/* Length of KDF input = intro_secret_hs_input | t_hsenc | info */
+#define KDF_INPUT_LEN (INTRO_SECRET_HS_INPUT_LEN + T_HSENC_LEN + INFO_BLOB_LEN)
+
+/** Helper function: Compute the part of the HS ntor handshake that generates
+ *  key material for creating and handling INTRODUCE1 cells. Function used
+ *  by both client and service. Specifically, calculate the following:
+ *
+ *     info = m_hsexpand | subcredential
+ *     hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *     ENC_KEY = hs_keys[0:S_KEY_LEN]
+ *     MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ *  where intro_secret_hs_input is <b>secret_input</b> (of size
+ *  INTRO_SECRET_HS_INPUT_LEN), and <b>subcredential</b> is of size
+ *  DIGEST256_LEN.
+ *
+ * If everything went well, fill <b>hs_ntor_intro_cell_keys_out</b> with the
+ * necessary key material, and return 0. */
+static void
+get_introduce1_key_material(const uint8_t *secret_input,
+                        const uint8_t *subcredential,
+                        hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
+  uint8_t info_blob[INFO_BLOB_LEN];
+  uint8_t kdf_input[KDF_INPUT_LEN];
+  crypto_xof_t *xof;
+  uint8_t *ptr;
+
+  /* Let's build info */
+  ptr = info_blob;
+  APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
+  APPEND(ptr, subcredential, DIGEST256_LEN);
+  tor_assert(ptr == info_blob + sizeof(info_blob));
+
+  /* Let's build the input to the KDF */
+  ptr = kdf_input;
+  APPEND(ptr, secret_input, INTRO_SECRET_HS_INPUT_LEN);
+  APPEND(ptr, T_HSENC, strlen(T_HSENC));
+  APPEND(ptr, info_blob, sizeof(info_blob));
+  tor_assert(ptr == kdf_input + sizeof(kdf_input));
+
+  /* Now we need to run kdf_input over SHAKE-256 */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
+  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
+  crypto_xof_free(xof);
+
+  { /* Get the keys */
+    memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
+    memcpy(&hs_ntor_intro_cell_keys_out->mac_key,
+           keystream+CIPHER256_KEY_LEN, DIGEST256_LEN);
+  }
+
+  memwipe(keystream,  0, sizeof(keystream));
+  memwipe(kdf_input,  0, sizeof(kdf_input));
+}
+
+/** Helper function: Calculate the 'intro_secret_hs_input' element used by the
+ * HS ntor handshake and place it in <b>secret_input_out</b>. This function is
+ * used by both client and service code.
+ *
+ * For the client-side it looks like this:
+ *
+ *         intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+ *
+ * whereas for the service-side it looks like this:
+ *
+ *         intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+ *
+ * In this function, <b>dh_result</b> carries the EXP() result (and has size
+ * CURVE25519_OUTPUT_LEN) <b>intro_auth_pubkey</b> is AUTH_KEY,
+ * <b>client_ephemeral_enc_pubkey</b> is X, and <b>intro_enc_pubkey</b> is B.
+ */
+static void
+get_intro_secret_hs_input(const uint8_t *dh_result,
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    const curve25519_public_key_t *intro_enc_pubkey,
+                    uint8_t *secret_input_out)
+{
+  uint8_t *ptr;
+
+  /* Append EXP() */
+  ptr = secret_input_out;
+  APPEND(ptr, dh_result, CURVE25519_OUTPUT_LEN);
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr, client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  tor_assert(ptr == secret_input_out + INTRO_SECRET_HS_INPUT_LEN);
+}
+
+/** Calculate the 'rend_secret_hs_input' element used by the HS ntor handshake
+ *  and place it in <b>rend_secret_hs_input_out</b>. This function is used by
+ *  both client and service code.
+ *
+ * The computation on the client side is:
+ *  rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+ * whereas on the service side it is:
+ *  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+ *
+ * where:
+ * <b>dh_result1</b> and <b>dh_result2</b> carry the two EXP() results (of size
+ * CURVE25519_OUTPUT_LEN)
+ * <b>intro_auth_pubkey</b> is AUTH_KEY,
+ * <b>intro_enc_pubkey</b> is B,
+ * <b>client_ephemeral_enc_pubkey</b> is X, and
+ * <b>service_ephemeral_rend_pubkey</b> is Y.
+ */
+static void
+get_rend_secret_hs_input(const uint8_t *dh_result1, const uint8_t *dh_result2,
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  uint8_t *rend_secret_hs_input_out)
+{
+  uint8_t *ptr;
+
+  ptr = rend_secret_hs_input_out;
+  /* Append the first EXP() */
+  APPEND(ptr, dh_result1, CURVE25519_OUTPUT_LEN);
+  /* Append the other EXP() */
+  APPEND(ptr, dh_result2, CURVE25519_OUTPUT_LEN);
+  /* Append AUTH_KEY */
+  APPEND(ptr, intro_auth_pubkey->pubkey, ED25519_PUBKEY_LEN);
+  /* Append B */
+  APPEND(ptr, intro_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append X */
+  APPEND(ptr,
+         client_ephemeral_enc_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append Y */
+  APPEND(ptr,
+         service_ephemeral_rend_pubkey->public_key, CURVE25519_PUBKEY_LEN);
+  /* Append PROTOID */
+  APPEND(ptr, PROTOID, strlen(PROTOID));
+  tor_assert(ptr == rend_secret_hs_input_out + REND_SECRET_HS_INPUT_LEN);
+}
+
+/************************* Public functions: *******************************/
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to encrypt and authenticate INTRODUCE1 cells. Return 0 and place the
+ * final key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went
+ * well, otherwise return -1;
+ *
+ * The relevant calculations are as follows:
+ *
+ *     intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+ *     info = m_hsexpand | subcredential
+ *     hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *     ENC_KEY = hs_keys[0:S_KEY_LEN]
+ *     MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor),
+ * <b>intro_enc_pubkey</b> is B (also found in HS descriptor),
+ * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X)
+ * <b>subcredential</b> is the hidden service subcredential (of size
+ * DIGEST256_LEN). */
+int
+hs_ntor_client_get_introduce1_keys(
+                      const ed25519_public_key_t *intro_auth_pubkey,
+                      const curve25519_public_key_t *intro_enc_pubkey,
+                      const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                      const uint8_t *subcredential,
+                      hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_pubkey);
+  tor_assert(client_ephemeral_enc_keypair);
+  tor_assert(subcredential);
+  tor_assert(hs_ntor_intro_cell_keys_out);
+
+  /* Calculate EXP(B,x) */
+  curve25519_handshake(dh_result,
+                       &client_ephemeral_enc_keypair->seckey,
+                       intro_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN);
+
+  /* Get intro_secret_hs_input */
+  get_intro_secret_hs_input(dh_result, intro_auth_pubkey,
+                            &client_ephemeral_enc_keypair->pubkey,
+                            intro_enc_pubkey, secret_input);
+  bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
+
+  /* Get ENC_KEY and MAC_KEY! */
+  get_introduce1_key_material(secret_input, subcredential,
+                              hs_ntor_intro_cell_keys_out);
+
+  /* Cleanup */
+  memwipe(secret_input,  0, sizeof(secret_input));
+  if (bad) {
+    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to verify RENDEZVOUS1 cells and encrypt further rendezvous
+ * traffic. Return 0 and place the final key material in
+ * <b>hs_ntor_rend_cell_keys_out</b> if everything went well, else return -1.
+ *
+ * The relevant calculations are as follows:
+ *
+ *  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+ *  NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *  verify = MAC(rend_secret_hs_input, t_hsverify)
+ *  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *  auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (found in HS descriptor),
+ * <b>client_ephemeral_enc_keypair</b> is freshly generated keypair (x,X)
+ * <b>intro_enc_pubkey</b> is B (also found in HS descriptor),
+ * <b>service_ephemeral_rend_pubkey</b> is Y (SERVER_PK in RENDEZVOUS1 cell) */
+int
+hs_ntor_client_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result1[CURVE25519_OUTPUT_LEN];
+  uint8_t dh_result2[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(client_ephemeral_enc_keypair);
+  tor_assert(intro_enc_pubkey);
+  tor_assert(service_ephemeral_rend_pubkey);
+  tor_assert(hs_ntor_rend_cell_keys_out);
+
+  /* Compute EXP(Y, x) */
+  curve25519_handshake(dh_result1,
+                       &client_ephemeral_enc_keypair->seckey,
+                       service_ephemeral_rend_pubkey);
+  bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN);
+
+  /* Compute EXP(B, x) */
+  curve25519_handshake(dh_result2,
+                       &client_ephemeral_enc_keypair->seckey,
+                       intro_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN);
+
+  /* Get rend_secret_hs_input */
+  get_rend_secret_hs_input(dh_result1, dh_result2,
+                           intro_auth_pubkey, intro_enc_pubkey,
+                           &client_ephemeral_enc_keypair->pubkey,
+                           service_ephemeral_rend_pubkey,
+                           rend_secret_hs_input);
+
+  /* Get NTOR_KEY_SEED and the auth_input MAC */
+  bad |= get_rendezvous1_key_material(rend_secret_hs_input,
+                                      intro_auth_pubkey,
+                                      intro_enc_pubkey,
+                                      service_ephemeral_rend_pubkey,
+                                      &client_ephemeral_enc_keypair->pubkey,
+                                      hs_ntor_rend_cell_keys_out);
+
+  memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input));
+  if (bad) {
+    memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to decrypt and verify INTRODUCE1 cells. Return 0 and place the final
+ * key material in <b>hs_ntor_intro_cell_keys_out</b> if everything went well,
+ * otherwise return -1;
+ *
+ * The relevant calculations are as follows:
+ *
+ *    intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+ *    info = m_hsexpand | subcredential
+ *    hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+ *    HS_DEC_KEY = hs_keys[0:S_KEY_LEN]
+ *    HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (introduction point auth key),
+ * <b>intro_enc_keypair</b> is (b,B) (introduction point encryption keypair),
+ * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell),
+ * <b>subcredential</b> is the HS subcredential (of size DIGEST256_LEN) */
+int
+hs_ntor_service_get_introduce1_keys(
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_keypair_t *intro_enc_keypair,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    const uint8_t *subcredential,
+                    hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_keypair);
+  tor_assert(client_ephemeral_enc_pubkey);
+  tor_assert(subcredential);
+  tor_assert(hs_ntor_intro_cell_keys_out);
+
+  /* Compute EXP(X, b) */
+  curve25519_handshake(dh_result,
+                       &intro_enc_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result, CURVE25519_OUTPUT_LEN);
+
+  /* Get intro_secret_hs_input */
+  get_intro_secret_hs_input(dh_result, intro_auth_pubkey,
+                            client_ephemeral_enc_pubkey,
+                            &intro_enc_keypair->pubkey,
+                            secret_input);
+  bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
+
+  /* Get ENC_KEY and MAC_KEY! */
+  get_introduce1_key_material(secret_input, subcredential,
+                              hs_ntor_intro_cell_keys_out);
+
+  memwipe(secret_input,  0, sizeof(secret_input));
+  if (bad) {
+    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/* Public function: Do the appropriate ntor calculations and derive the keys
+ * needed to create and authenticate RENDEZVOUS1 cells. Return 0 and place the
+ * final key material in <b>hs_ntor_rend_cell_keys_out</b> if all went fine,
+ * return -1 if error happened.
+ *
+ * The relevant calculations are as follows:
+ *
+ *  rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+ *  NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+ *  verify = MAC(rend_secret_hs_input, t_hsverify)
+ *  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+ *  auth_input_mac = MAC(auth_input, t_hsmac)
+ *
+ * where:
+ * <b>intro_auth_pubkey</b> is AUTH_KEY (intro point auth key),
+ * <b>intro_enc_keypair</b> is (b,B) (intro point enc keypair)
+ * <b>service_ephemeral_rend_keypair</b> is a fresh (y,Y) keypair
+ * <b>client_ephemeral_enc_pubkey</b> is X (CLIENT_PK in INTRODUCE2 cell) */
+int
+hs_ntor_service_get_rendezvous1_keys(
+                    const ed25519_public_key_t *intro_auth_pubkey,
+                    const curve25519_keypair_t *intro_enc_keypair,
+                    const curve25519_keypair_t *service_ephemeral_rend_keypair,
+                    const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                    hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out)
+{
+  int bad = 0;
+  uint8_t rend_secret_hs_input[REND_SECRET_HS_INPUT_LEN];
+  uint8_t dh_result1[CURVE25519_OUTPUT_LEN];
+  uint8_t dh_result2[CURVE25519_OUTPUT_LEN];
+
+  tor_assert(intro_auth_pubkey);
+  tor_assert(intro_enc_keypair);
+  tor_assert(service_ephemeral_rend_keypair);
+  tor_assert(client_ephemeral_enc_pubkey);
+  tor_assert(hs_ntor_rend_cell_keys_out);
+
+  /* Compute EXP(X, y) */
+  curve25519_handshake(dh_result1,
+                       &service_ephemeral_rend_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result1, CURVE25519_OUTPUT_LEN);
+
+  /* Compute EXP(X, b) */
+  curve25519_handshake(dh_result2,
+                       &intro_enc_keypair->seckey,
+                       client_ephemeral_enc_pubkey);
+  bad |= safe_mem_is_zero(dh_result2, CURVE25519_OUTPUT_LEN);
+
+  /* Get rend_secret_hs_input */
+  get_rend_secret_hs_input(dh_result1, dh_result2,
+                           intro_auth_pubkey,
+                           &intro_enc_keypair->pubkey,
+                           client_ephemeral_enc_pubkey,
+                           &service_ephemeral_rend_keypair->pubkey,
+                           rend_secret_hs_input);
+
+  /* Get NTOR_KEY_SEED and AUTH_INPUT_MAC! */
+  bad |= get_rendezvous1_key_material(rend_secret_hs_input,
+                                      intro_auth_pubkey,
+                                      &intro_enc_keypair->pubkey,
+                                      &service_ephemeral_rend_keypair->pubkey,
+                                      client_ephemeral_enc_pubkey,
+                                      hs_ntor_rend_cell_keys_out);
+
+  memwipe(rend_secret_hs_input, 0, sizeof(rend_secret_hs_input));
+  if (bad) {
+    memwipe(hs_ntor_rend_cell_keys_out, 0, sizeof(hs_ntor_rend_cell_keys_t));
+  }
+
+  return bad ? -1 : 0;
+}
+
+/** Given a received RENDEZVOUS2 MAC in <b>mac</b> (of length DIGEST256_LEN),
+ *  and the RENDEZVOUS1 key material in <b>hs_ntor_rend_cell_keys</b>, return 1
+ *  if the MAC is good, otherwise return 0. */
+int
+hs_ntor_client_rendezvous2_mac_is_good(
+                        const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys,
+                        const uint8_t *rcvd_mac)
+{
+  tor_assert(rcvd_mac);
+  tor_assert(hs_ntor_rend_cell_keys);
+
+  return tor_memeq(hs_ntor_rend_cell_keys->rend_cell_auth_mac,
+                   rcvd_mac, DIGEST256_LEN);
+}
+
+/* Input length to KDF for key expansion */
+#define NTOR_KEY_EXPANSION_KDF_INPUT_LEN (DIGEST256_LEN + M_HSEXPAND_LEN)
+/* Output length of KDF for key expansion */
+#define NTOR_KEY_EXPANSION_KDF_OUTPUT_LEN (DIGEST256_LEN*3+CIPHER256_KEY_LEN*2)
+
+/** Given the rendezvous key material in <b>hs_ntor_rend_cell_keys</b>, do the
+ *  circuit key expansion as specified by section '4.2.1. Key expansion' and
+ *  return a hs_ntor_rend_circuit_keys_t structure with the computed keys. */
+hs_ntor_rend_circuit_keys_t *
+hs_ntor_circuit_key_expansion(
+                       const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys)
+{
+  uint8_t *ptr;
+  uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
+  uint8_t keys[NTOR_KEY_EXPANSION_KDF_OUTPUT_LEN];
+  crypto_xof_t *xof;
+  hs_ntor_rend_circuit_keys_t *rend_circuit_keys = NULL;
+
+  /* Let's build the input to the KDF */
+  ptr = kdf_input;
+  APPEND(ptr, hs_ntor_rend_cell_keys->ntor_key_seed, DIGEST256_LEN);
+  APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
+  tor_assert(ptr == kdf_input + sizeof(kdf_input));
+
+  /* Generate the keys */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
+  crypto_xof_squeeze_bytes(xof, keys, sizeof(keys));
+  crypto_xof_free(xof);
+
+  /* Generate keys structure and assign keys to it */
+  rend_circuit_keys = tor_malloc_zero(sizeof(hs_ntor_rend_circuit_keys_t));
+  ptr = keys;
+  memcpy(rend_circuit_keys->KH, ptr, DIGEST256_LEN);
+  ptr += DIGEST256_LEN;;
+  memcpy(rend_circuit_keys->Df, ptr, DIGEST256_LEN);
+  ptr += DIGEST256_LEN;
+  memcpy(rend_circuit_keys->Db, ptr, DIGEST256_LEN);
+  ptr += DIGEST256_LEN;
+  memcpy(rend_circuit_keys->Kf, ptr, CIPHER256_KEY_LEN);
+  ptr += CIPHER256_KEY_LEN;
+  memcpy(rend_circuit_keys->Kb, ptr, CIPHER256_KEY_LEN);
+  ptr += CIPHER256_KEY_LEN;
+  tor_assert(ptr == keys + sizeof(keys));
+
+  return rend_circuit_keys;
+}
+
diff --git a/src/or/hs_ntor.h b/src/or/hs_ntor.h
new file mode 100644
index 0000000000..cd75f46a4c
--- /dev/null
+++ b/src/or/hs_ntor.h
@@ -0,0 +1,77 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_HS_NTOR_H
+#define TOR_HS_NTOR_H
+
+#include "or.h"
+
+/* Key material needed to encode/decode INTRODUCE1 cells */
+typedef struct {
+  /* Key used for encryption of encrypted INTRODUCE1 blob */
+  uint8_t enc_key[CIPHER256_KEY_LEN];
+  /* MAC key used to protect encrypted INTRODUCE1 blob */
+  uint8_t mac_key[DIGEST256_LEN];
+} hs_ntor_intro_cell_keys_t;
+
+/* Key material needed to encode/decode RENDEZVOUS1 cells */
+typedef struct {
+  /* This is the MAC of the HANDSHAKE_INFO field */
+  uint8_t rend_cell_auth_mac[DIGEST256_LEN];
+  /* This is the key seed used to derive further rendezvous crypto keys as
+   * detailed in section 4.2.1 of rend-spec-ng.txt. */
+  uint8_t ntor_key_seed[DIGEST256_LEN];
+} hs_ntor_rend_cell_keys_t;
+
+/* Key material resulting from key expansion as detailed in section "4.2.1. Key
+ * expansion" of rend-spec-ng.txt. */
+typedef struct {
+  /* Per-circuit key material used in ESTABLISH_INTRO cell */
+  uint8_t KH[DIGEST256_LEN];
+  /* Authentication key for outgoing RELAY cells */
+  uint8_t Df[DIGEST256_LEN];
+  /* Authentication key for incoming RELAY cells */
+  uint8_t Db[DIGEST256_LEN];
+  /* Encryption key for outgoing RELAY cells */
+  uint8_t Kf[CIPHER256_KEY_LEN];
+  /* Decryption key for incoming RELAY cells */
+  uint8_t Kb[CIPHER256_KEY_LEN];
+} hs_ntor_rend_circuit_keys_t;
+
+int hs_ntor_client_get_introduce1_keys(
+                      const ed25519_public_key_t *intro_auth_pubkey,
+                      const curve25519_public_key_t *intro_enc_pubkey,
+                      const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                      const uint8_t *subcredential,
+                      hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
+int hs_ntor_client_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *client_ephemeral_enc_keypair,
+                  const curve25519_public_key_t *intro_enc_pubkey,
+                  const curve25519_public_key_t *service_ephemeral_rend_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
+
+int hs_ntor_service_get_introduce1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *intro_enc_keypair,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  const uint8_t *subcredential,
+                  hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
+int hs_ntor_service_get_rendezvous1_keys(
+                  const ed25519_public_key_t *intro_auth_pubkey,
+                  const curve25519_keypair_t *intro_enc_keypair,
+                  const curve25519_keypair_t *service_ephemeral_rend_keypair,
+                  const curve25519_public_key_t *client_ephemeral_enc_pubkey,
+                  hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
+
+hs_ntor_rend_circuit_keys_t *hs_ntor_circuit_key_expansion(
+                       const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys);
+
+int hs_ntor_client_rendezvous2_mac_is_good(
+                        const hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys,
+                        const uint8_t *rcvd_mac);
+
+#endif
+
diff --git a/src/or/hs_service.c b/src/or/hs_service.c
new file mode 100644
index 0000000000..b3eec13046
--- /dev/null
+++ b/src/or/hs_service.c
@@ -0,0 +1,175 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.c
+ * \brief Implement next generation hidden service functionality
+ **/
+
+#include "or.h"
+#include "relay.h"
+#include "rendservice.h"
+#include "circuitlist.h"
+#include "circpathbias.h"
+#include "networkstatus.h"
+
+#include "hs_intropoint.h"
+#include "hs_service.h"
+#include "hs_common.h"
+
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_common.h"
+
+/* XXX We don't currently use these functions, apart from generating unittest
+   data. When we start implementing the service-side support for prop224 we
+   should revisit these functions and use them. */
+
+/** Given an ESTABLISH_INTRO <b>cell</b>, encode it and place its payload in
+ *  <b>buf_out</b> which has size <b>buf_out_len</b>. Return the number of
+ *  bytes written, or a negative integer if there was an error. */
+ssize_t
+get_establish_intro_payload(uint8_t *buf_out, size_t buf_out_len,
+                            const trn_cell_establish_intro_t *cell)
+{
+  ssize_t bytes_used = 0;
+
+  if (buf_out_len < RELAY_PAYLOAD_SIZE) {
+    return -1;
+  }
+
+  bytes_used = trn_cell_establish_intro_encode(buf_out, buf_out_len,
+                                              cell);
+  return bytes_used;
+}
+
+/* Set the cell extensions of <b>cell</b>. */
+static void
+set_trn_cell_extensions(trn_cell_establish_intro_t *cell)
+{
+  trn_cell_extension_t *trn_cell_extensions = trn_cell_extension_new();
+
+  /* For now, we don't use extensions at all. */
+  trn_cell_extensions->num = 0; /* It's already zeroed, but be explicit. */
+  trn_cell_establish_intro_set_extensions(cell, trn_cell_extensions);
+}
+
+/** Given the circuit handshake info in <b>circuit_key_material</b>, create and
+ *  return an ESTABLISH_INTRO cell. Return NULL if something went wrong.  The
+ *  returned cell is allocated on the heap and it's the responsibility of the
+ *  caller to free it. */
+trn_cell_establish_intro_t *
+generate_establish_intro_cell(const uint8_t *circuit_key_material,
+                              size_t circuit_key_material_len)
+{
+  trn_cell_establish_intro_t *cell = NULL;
+  ssize_t encoded_len;
+
+  log_warn(LD_GENERAL,
+           "Generating ESTABLISH_INTRO cell (key_material_len: %u)",
+           (unsigned) circuit_key_material_len);
+
+  /* Generate short-term keypair for use in ESTABLISH_INTRO */
+  ed25519_keypair_t key_struct;
+  if (ed25519_keypair_generate(&key_struct, 0) < 0) {
+    goto err;
+  }
+
+  cell = trn_cell_establish_intro_new();
+
+  /* Set AUTH_KEY_TYPE: 2 means ed25519 */
+  trn_cell_establish_intro_set_auth_key_type(cell,
+                                             HS_INTRO_AUTH_KEY_TYPE_ED25519);
+
+  /* Set AUTH_KEY_LEN field */
+  /* Must also set byte-length of AUTH_KEY to match */
+  int auth_key_len = ED25519_PUBKEY_LEN;
+  trn_cell_establish_intro_set_auth_key_len(cell, auth_key_len);
+  trn_cell_establish_intro_setlen_auth_key(cell, auth_key_len);
+
+  /* Set AUTH_KEY field */
+  uint8_t *auth_key_ptr = trn_cell_establish_intro_getarray_auth_key(cell);
+  memcpy(auth_key_ptr, key_struct.pubkey.pubkey, auth_key_len);
+
+  /* No cell extensions needed */
+  set_trn_cell_extensions(cell);
+
+  /* Set signature size.
+     We need to do this up here, because _encode() needs it and we need to call
+     _encode() to calculate the MAC and signature.
+  */
+  int sig_len = ED25519_SIG_LEN;
+  trn_cell_establish_intro_set_sig_len(cell, sig_len);
+  trn_cell_establish_intro_setlen_sig(cell, sig_len);
+
+  /* XXX How to make this process easier and nicer? */
+
+  /* Calculate the cell MAC (aka HANDSHAKE_AUTH). */
+  {
+    /* To calculate HANDSHAKE_AUTH, we dump the cell in bytes, and then derive
+       the MAC from it. */
+    uint8_t cell_bytes_tmp[RELAY_PAYLOAD_SIZE] = {0};
+    uint8_t mac[TRUNNEL_SHA3_256_LEN];
+
+    encoded_len = trn_cell_establish_intro_encode(cell_bytes_tmp,
+                                                 sizeof(cell_bytes_tmp),
+                                                 cell);
+    if (encoded_len < 0) {
+      log_warn(LD_OR, "Unable to pre-encode ESTABLISH_INTRO cell.");
+      goto err;
+    }
+
+    /* sanity check */
+    tor_assert(encoded_len > ED25519_SIG_LEN + 2 + TRUNNEL_SHA3_256_LEN);
+
+    /* Calculate MAC of all fields before HANDSHAKE_AUTH */
+    crypto_mac_sha3_256(mac, sizeof(mac),
+                        circuit_key_material, circuit_key_material_len,
+                        cell_bytes_tmp,
+                        encoded_len -
+                          (ED25519_SIG_LEN + 2 + TRUNNEL_SHA3_256_LEN));
+    /* Write the MAC to the cell */
+    uint8_t *handshake_ptr =
+      trn_cell_establish_intro_getarray_handshake_mac(cell);
+    memcpy(handshake_ptr, mac, sizeof(mac));
+  }
+
+  /* Calculate the cell signature */
+  {
+    /* To calculate the sig we follow the same procedure as above. We first
+       dump the cell up to the sig, and then calculate the sig */
+    uint8_t cell_bytes_tmp[RELAY_PAYLOAD_SIZE] = {0};
+    ed25519_signature_t sig;
+
+    encoded_len = trn_cell_establish_intro_encode(cell_bytes_tmp,
+                                                 sizeof(cell_bytes_tmp),
+                                                 cell);
+    if (encoded_len < 0) {
+      log_warn(LD_OR, "Unable to pre-encode ESTABLISH_INTRO cell (2).");
+      goto err;
+    }
+
+    tor_assert(encoded_len > ED25519_SIG_LEN);
+
+    if (ed25519_sign_prefixed(&sig,
+                              cell_bytes_tmp,
+                              encoded_len -
+                                (ED25519_SIG_LEN + sizeof(cell->sig_len)),
+                              ESTABLISH_INTRO_SIG_PREFIX,
+                              &key_struct)) {
+      log_warn(LD_BUG, "Unable to gen signature for ESTABLISH_INTRO cell.");
+      goto err;
+    }
+
+    /* And write the signature to the cell */
+    uint8_t *sig_ptr = trn_cell_establish_intro_getarray_sig(cell);
+    memcpy(sig_ptr, sig.sig, sig_len);
+  }
+
+  /* We are done! Return the cell! */
+  return cell;
+
+ err:
+  trn_cell_establish_intro_free(cell);
+  return NULL;
+}
+
diff --git a/src/or/hs_service.h b/src/or/hs_service.h
new file mode 100644
index 0000000000..3302592762
--- /dev/null
+++ b/src/or/hs_service.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_service.h
+ * \brief Header file for hs_service.c.
+ **/
+
+#ifndef TOR_HS_SERVICE_H
+#define TOR_HS_SERVICE_H
+
+#include "or.h"
+#include "hs/cell_establish_intro.h"
+
+/* These functions are only used by unit tests and we need to expose them else
+ * hs_service.o ends up with no symbols in libor.a which makes clang throw a
+ * warning at compile time. See #21825. */
+
+trn_cell_establish_intro_t *
+generate_establish_intro_cell(const uint8_t *circuit_key_material,
+                              size_t circuit_key_material_len);
+ssize_t
+get_establish_intro_payload(uint8_t *buf, size_t buf_len,
+                            const trn_cell_establish_intro_t *cell);
+
+#endif /* TOR_HS_SERVICE_H */
+
diff --git a/src/or/include.am b/src/or/include.am
index 19cf00264b..7548ed0946 100644
--- a/src/or/include.am
+++ b/src/or/include.am
@@ -19,8 +19,10 @@ EXTRA_DIST+= src/or/ntmain.c src/or/Makefile.nmake
 
 LIBTOR_A_SOURCES = \
 	src/or/addressmap.c				\
+	src/or/bridges.c				\
 	src/or/buffers.c				\
 	src/or/channel.c				\
+	src/or/channelpadding.c				\
 	src/or/channeltls.c				\
 	src/or/circpathbias.c				\
 	src/or/circuitbuild.c				\
@@ -35,6 +37,9 @@ LIBTOR_A_SOURCES = \
 	src/or/connection.c				\
 	src/or/connection_edge.c			\
 	src/or/connection_or.c				\
+	src/or/conscache.c				\
+	src/or/consdiff.c				\
+	src/or/consdiffmgr.c				\
 	src/or/control.c				\
 	src/or/cpuworker.c				\
 	src/or/dircollate.c				\
@@ -46,9 +51,16 @@ LIBTOR_A_SOURCES = \
 	src/or/dos.c					\
 	src/or/fp_pair.c				\
 	src/or/geoip.c					\
+	src/or/hs_intropoint.c          \
+	src/or/hs_circuitmap.c          \
+	src/or/hs_ntor.c                \
+	src/or/hs_service.c             \
 	src/or/entrynodes.c				\
 	src/or/ext_orport.c				\
 	src/or/hibernate.c				\
+	src/or/hs_cache.c				\
+	src/or/hs_common.c				\
+	src/or/hs_descriptor.c				\
 	src/or/keypin.c					\
 	src/or/main.c					\
 	src/or/microdesc.c				\
@@ -60,6 +72,7 @@ LIBTOR_A_SOURCES = \
 	src/or/shared_random.c			\
 	src/or/shared_random_state.c		\
 	src/or/transports.c				\
+	src/or/parsecommon.c			\
 	src/or/periodic.c				\
 	src/or/protover.c				\
 	src/or/policies.c				\
@@ -109,8 +122,11 @@ src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libev
 src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \
 	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event.a src/trunnel/libor-trunnel.a \
+	src/trace/libor-trace.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
 
 if COVERAGE_ENABLED
 src_or_tor_cov_SOURCES = src/or/tor_main.c
@@ -122,14 +138,17 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \
 	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@
 endif
 
 ORHEADERS = \
 	src/or/addressmap.h				\
 	src/or/auth_dirs.inc				\
+	src/or/bridges.h				\
 	src/or/buffers.h				\
 	src/or/channel.h				\
+	src/or/channelpadding.h				\
 	src/or/channeltls.h				\
 	src/or/circpathbias.h				\
 	src/or/circuitbuild.h				\
@@ -144,6 +163,9 @@ ORHEADERS = \
 	src/or/connection.h				\
 	src/or/connection_edge.h			\
 	src/or/connection_or.h				\
+	src/or/conscache.h				\
+	src/or/consdiff.h				\
+	src/or/consdiffmgr.h				\
 	src/or/control.h				\
 	src/or/cpuworker.h				\
 	src/or/dircollate.h				\
@@ -160,6 +182,13 @@ ORHEADERS = \
 	src/or/geoip.h					\
 	src/or/entrynodes.h				\
 	src/or/hibernate.h				\
+	src/or/hs_cache.h				\
+	src/or/hs_common.h				\
+	src/or/hs_descriptor.h				\
+	src/or/hs_intropoint.h          \
+	src/or/hs_circuitmap.h          \
+	src/or/hs_ntor.h                \
+	src/or/hs_service.h             \
 	src/or/keypin.h					\
 	src/or/main.h					\
 	src/or/microdesc.h				\
@@ -174,6 +203,7 @@ ORHEADERS = \
 	src/or/shared_random.h			\
 	src/or/shared_random_state.h		\
 	src/or/transports.h				\
+	src/or/parsecommon.h			\
 	src/or/periodic.h				\
 	src/or/policies.h				\
 	src/or/protover.h				\
diff --git a/src/or/keypin.c b/src/or/keypin.c
index 2d4c4e92d2..1698dc184f 100644
--- a/src/or/keypin.c
+++ b/src/or/keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/keypin.h b/src/or/keypin.h
index 673f24d9e3..2564f5befb 100644
--- a/src/or/keypin.h
+++ b/src/or/keypin.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_KEYPIN_H
diff --git a/src/or/main.c b/src/or/main.c
index fcd8dc9024..e9723cb0b7 100644
--- a/src/or/main.c
+++ b/src/or/main.c
@@ -1,31 +1,72 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file main.c
  * \brief Toplevel module. Handles signals, multiplexes between
  * connections, implements main loop, and drives scheduled events.
+ *
+ * For the main loop itself; see run_main_loop_once().  It invokes the rest of
+ * Tor mostly through Libevent callbacks.  Libevent callbacks can happen when
+ * a timer elapses, a signal is received, a socket is ready to read or write,
+ * or an event is manually activated.
+ *
+ * Most events in Tor are driven from these callbacks:
+ *  <ul>
+ *   <li>conn_read_callback() and conn_write_callback() here, which are
+ *     invoked when a socket is ready to read or write respectively.
+ *   <li>signal_callback(), which handles incoming signals.
+ *  </ul>
+ * Other events are used for specific purposes, or for building more complex
+ * control structures.  If you search for usage of tor_libevent_new(), you
+ * will find all the events that we construct in Tor.
+ *
+ * Tor has numerous housekeeping operations that need to happen
+ * regularly. They are handled in different ways:
+ * <ul>
+ *   <li>The most frequent operations are handled after every read or write
+ *    event, at the end of connection_handle_read() and
+ *    connection_handle_write().
+ *
+ *   <li>The next most frequent operations happen after each invocation of the
+ *     main loop, in run_main_loop_once().
+ *
+ *   <li>Once per second, we run all of the operations listed in
+ *     second_elapsed_callback(), and in its child, run_scheduled_events().
+ *
+ *   <li>Once-a-second operations are handled in second_elapsed_callback().
+ *
+ *   <li>More infrequent operations take place based on the periodic event
+ *     driver in periodic.c .  These are stored in the periodic_events[]
+ *     table.
+ * </ul>
+ *
  **/
 
 #define MAIN_PRIVATE
 #include "or.h"
 #include "addressmap.h"
 #include "backtrace.h"
+#include "bridges.h"
 #include "buffers.h"
 #include "channel.h"
 #include "channeltls.h"
+#include "channelpadding.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
 #include "command.h"
+#include "compat_rust.h"
+#include "compress.h"
 #include "config.h"
 #include "confparse.h"
 #include "connection.h"
 #include "connection_edge.h"
 #include "connection_or.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "cpuworker.h"
 #include "crypto_s2k.h"
@@ -38,6 +79,8 @@
 #include "entrynodes.h"
 #include "geoip.h"
 #include "hibernate.h"
+#include "hs_cache.h"
+#include "hs_circuitmap.h"
 #include "keypin.h"
 #include "main.h"
 #include "microdesc.h"
@@ -66,7 +109,6 @@
 #include "ext_orport.h"
 #ifdef USE_DMALLOC
 #include <dmalloc.h>
-#include <openssl/crypto.h>
 #endif
 #include "memarea.h"
 #include "sandbox.h"
@@ -138,7 +180,7 @@ static int signewnym_is_pending = 0;
 static unsigned newnym_epoch = 0;
 
 /** Smartlist of all open connections. */
-static smartlist_t *connection_array = NULL;
+STATIC smartlist_t *connection_array = NULL;
 /** List of connections that have been marked for close and need to be freed
  * and removed from connection_array. */
 static smartlist_t *closeable_connection_lst = NULL;
@@ -326,7 +368,7 @@ connection_unlink(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_OR) {
     if (!tor_digest_is_zero(TO_OR_CONN(conn)->identity_digest))
-      connection_or_remove_from_identity_map(TO_OR_CONN(conn));
+      connection_or_clear_identity(TO_OR_CONN(conn));
     /* connection_unlink() can only get called if the connection
      * was already on the closeable list, and it got there by
      * connection_mark_for_close(), which was called from
@@ -448,7 +490,7 @@ connection_check_event(connection_t *conn, struct event *ev)
      */
     bad = ev != NULL;
   } else {
-    /* Everytyhing else should have an underlying socket, or a linked
+    /* Everything else should have an underlying socket, or a linked
      * connection (which is also tracked with a read_event/write_event pair).
      */
     bad = ev == NULL;
@@ -927,6 +969,15 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 {
   const or_options_t *options = get_options();
 
+  /* if we have enough dir info, then update our guard status with
+   * whatever we just learned. */
+  int invalidate_circs = guards_update_all();
+
+  if (invalidate_circs) {
+    circuit_mark_all_unused_circs();
+    circuit_mark_all_dirty_circs_as_unusable();
+  }
+
   if (!router_have_minimum_dir_info()) {
     int quiet = suppress_logs || from_cache ||
                 directory_too_idle_to_fetch_descriptors(options, now);
@@ -940,9 +991,6 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
       update_all_descriptor_downloads(now);
     }
 
-    /* if we have enough dir info, then update our guard status with
-     * whatever we just learned. */
-    entry_guards_compute_status(options, now);
     /* Don't even bother trying to get extrainfo until the rest of our
      * directory info is up-to-date */
     if (options->DownloadExtraInfo)
@@ -1051,8 +1099,9 @@ run_connection_housekeeping(int i, time_t now)
   } else if (!have_any_circuits &&
              now - or_conn->idle_timeout >=
                                          chan->timestamp_last_had_circuits) {
-    log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) "
-             "[no circuits for %d; timeout %d; %scanonical].",
+    log_info(LD_OR,"Expiring non-used OR connection "U64_FORMAT" to fd %d "
+             "(%s:%d) [no circuits for %d; timeout %d; %scanonical].",
+             U64_PRINTF_ARG(chan->global_identifier),
              (int)conn->s, conn->address, conn->port,
              (int)(now - chan->timestamp_last_had_circuits),
              or_conn->idle_timeout,
@@ -1075,6 +1124,8 @@ run_connection_housekeeping(int i, time_t now)
     memset(&cell,0,sizeof(cell_t));
     cell.command = CELL_PADDING;
     connection_or_write_cell_to_buf(&cell, or_conn);
+  } else {
+    channelpadding_decide_to_pad_channel(chan);
   }
 }
 
@@ -1117,6 +1168,7 @@ static int periodic_events_initialized = 0;
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
 CALLBACK(rotate_onion_key);
+CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(check_ed_keys);
 CALLBACK(launch_descriptor_fetches);
 CALLBACK(rotate_x509_certificate);
@@ -1140,6 +1192,9 @@ CALLBACK(check_dns_honesty);
 CALLBACK(write_bridge_ns);
 CALLBACK(check_fw_helper_app);
 CALLBACK(heartbeat);
+CALLBACK(clean_consdiffmgr);
+CALLBACK(reset_padding_counts);
+CALLBACK(check_canonical_channels);
 
 #undef CALLBACK
 
@@ -1148,6 +1203,7 @@ CALLBACK(heartbeat);
 
 static periodic_event_item_t periodic_events[] = {
   CALLBACK(rotate_onion_key),
+  CALLBACK(check_onion_keys_expiry_time),
   CALLBACK(check_ed_keys),
   CALLBACK(launch_descriptor_fetches),
   CALLBACK(rotate_x509_certificate),
@@ -1171,6 +1227,9 @@ static periodic_event_item_t periodic_events[] = {
   CALLBACK(write_bridge_ns),
   CALLBACK(check_fw_helper_app),
   CALLBACK(heartbeat),
+  CALLBACK(clean_consdiffmgr),
+  CALLBACK(reset_padding_counts),
+  CALLBACK(check_canonical_channels),
   END_OF_PERIODIC_EVENTS
 };
 #undef CALLBACK
@@ -1339,6 +1398,9 @@ run_scheduled_events(time_t now)
   /* 0c. If we've deferred log messages for the controller, handle them now */
   flush_pending_log_callbacks();
 
+  /* Maybe enough time elapsed for us to reconsider a circuit. */
+  circuit_upgrade_circuits_from_guard_wait();
+
   if (options->UseBridges && !options->DisableNetwork) {
     fetch_bridge_descriptors(options, now);
   }
@@ -1359,6 +1421,7 @@ run_scheduled_events(time_t now)
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
   circuit_expire_building();
+  circuit_expire_waiting_for_better_guard();
 
   /* 3b. Also look at pending streams and prune the ones that 'began'
    *     a long time ago but haven't gotten a 'connected' yet.
@@ -1390,7 +1453,7 @@ run_scheduled_events(time_t now)
   }
 
   /* 5. We do housekeeping for each connection... */
-  connection_or_set_bad_connections(NULL, 0);
+  channel_update_bad_for_new_circs(NULL, 0);
   int i;
   for (i=0;i<smartlist_len(connection_array);i++) {
     run_connection_housekeeping(i, now);
@@ -1422,19 +1485,26 @@ run_scheduled_events(time_t now)
   /* 11b. check pending unconfigured managed proxies */
   if (!net_is_disabled() && pt_proxies_configuration_pending())
     pt_configure_remaining_proxies();
+
+  /* 12. launch diff computations.  (This is free if there are none to
+   * launch.) */
+  if (dir_server_mode(options)) {
+    consdiffmgr_rescan();
+  }
 }
 
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
 static int
 rotate_onion_key_callback(time_t now, const or_options_t *options)
 {
-  /* 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys,
-   *  shut down and restart all cpuworkers, and update the directory if
-   *  necessary.
-   */
   if (server_mode(options)) {
-    time_t rotation_time = get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME;
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
     if (rotation_time > now) {
-      return safe_timer_diff(now, rotation_time);
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
     }
 
     log_info(LD_GENERAL,"Rotating onion key.");
@@ -1445,21 +1515,49 @@ rotate_onion_key_callback(time_t now, const or_options_t *options)
     }
     if (advertised_server_mode() && !options->DisableNetwork)
       router_upload_dir_desc_to_dirservers(0);
-    return MIN_ONION_KEY_LIFETIME;
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
   }
+
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
 static int
 check_ed_keys_callback(time_t now, const or_options_t *options)
 {
   if (server_mode(options)) {
     if (should_make_new_ed_keys(options, now)) {
-      if (load_ed_keys(options, now) < 0 ||
-          generate_ed_link_cert(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
         log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
         tor_cleanup();
-        exit(0);
+        exit(1);
       }
     }
     return 30;
@@ -1467,6 +1565,11 @@ check_ed_keys_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: Every {LAZY,GREEDY}_DESCRIPTOR_RETRY_INTERVAL,
+ * see about fetching descriptors, microdescriptors, and extrainfo
+ * documents.
+ */
 static int
 launch_descriptor_fetches_callback(time_t now, const or_options_t *options)
 {
@@ -1481,6 +1584,10 @@ launch_descriptor_fetches_callback(time_t now, const or_options_t *options)
     return GREEDY_DESCRIPTOR_RETRY_INTERVAL;
 }
 
+/**
+ * Periodic event: Rotate our X.509 certificates and TLS keys once every
+ * MAX_SSL_KEY_LIFETIME_INTERNAL.
+ */
 static int
 rotate_x509_certificate_callback(time_t now, const or_options_t *options)
 {
@@ -1499,6 +1606,11 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options)
     log_err(LD_BUG, "Error reinitializing TLS context");
     tor_assert_unreached();
   }
+  if (generate_ed_link_cert(options, now, 1)) {
+    log_err(LD_OR, "Unable to update Ed25519->TLS link certificate for "
+            "new TLS context.");
+    tor_assert_unreached();
+  }
 
   /* We also make sure to rotate the TLS connections themselves if they've
    * been up for too long -- but that's done via is_bad_for_new_circs in
@@ -1506,6 +1618,10 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options)
   return MAX_SSL_KEY_LIFETIME_INTERNAL;
 }
 
+/**
+ * Periodic callback: once an hour, grab some more entropy from the
+ * kernel and feed it to our CSPRNG.
+ **/
 static int
 add_entropy_callback(time_t now, const or_options_t *options)
 {
@@ -1522,6 +1638,10 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
 static int
 launch_reachability_tests_callback(time_t now, const or_options_t *options)
 {
@@ -1533,6 +1653,10 @@ launch_reachability_tests_callback(time_t now, const or_options_t *options)
   return REACHABILITY_TEST_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
 static int
 downrate_stability_callback(time_t now, const or_options_t *options)
 {
@@ -1544,6 +1668,10 @@ downrate_stability_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next);
 }
 
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
 static int
 save_stability_callback(time_t now, const or_options_t *options)
 {
@@ -1556,6 +1684,10 @@ save_stability_callback(time_t now, const or_options_t *options)
   return SAVE_STABILITY_INTERVAL;
 }
 
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
 static int
 check_authority_cert_callback(time_t now, const or_options_t *options)
 {
@@ -1568,12 +1700,15 @@ check_authority_cert_callback(time_t now, const or_options_t *options)
   return CHECK_V3_CERTIFICATE_INTERVAL;
 }
 
+/**
+ * Periodic callback: If our consensus is too old, recalculate whether
+ * we can actually use it.
+ */
 static int
 check_expired_networkstatus_callback(time_t now, const or_options_t *options)
 {
   (void)options;
-  /* 1f. Check whether our networkstatus has expired.
-   */
+  /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
   /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
    * networkstatus_get_reasonably_live_consensus(), but that value is way
@@ -1587,6 +1722,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   return CHECK_EXPIRED_NS_INTERVAL;
 }
 
+/**
+ * Periodic callback: Write statistics to disk if appropriate.
+ */
 static int
 write_stats_file_callback(time_t now, const or_options_t *options)
 {
@@ -1634,6 +1772,31 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
+#define CHANNEL_CHECK_INTERVAL (60*60)
+static int
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+static int
+reset_padding_counts_callback(time_t now, const or_options_t *options)
+{
+  if (options->PaddingStatistics) {
+    rep_hist_prep_published_padding_counts(now);
+  }
+
+  rep_hist_reset_padding_counts();
+  return REPHIST_CELL_PADDING_COUNTS_INTERVAL;
+}
+
+/**
+ * Periodic callback: Write bridge statistics to disk if appropriate.
+ */
 static int
 record_bridge_stats_callback(time_t now, const or_options_t *options)
 {
@@ -1661,6 +1824,9 @@ record_bridge_stats_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: Clean in-memory caches every once in a while
+ */
 static int
 clean_caches_callback(time_t now, const or_options_t *options)
 {
@@ -1668,12 +1834,16 @@ clean_caches_callback(time_t now, const or_options_t *options)
   rep_history_clean(now - options->RephistTrackTime);
   rend_cache_clean(now, REND_CACHE_TYPE_CLIENT);
   rend_cache_clean(now, REND_CACHE_TYPE_SERVICE);
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  hs_cache_clean_as_dir(now);
   microdesc_cache_rebuild(NULL, 0);
 #define CLEAN_CACHES_INTERVAL (30*60)
   return CLEAN_CACHES_INTERVAL;
 }
 
+/**
+ * Periodic callback: Clean the cache of failed hidden service lookups
+ * frequently.
+ */
 static int
 rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 {
@@ -1685,20 +1855,21 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
   return 30;
 }
 
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
 static int
 retry_dns_callback(time_t now, const or_options_t *options)
 {
   (void)now;
 #define RETRY_DNS_INTERVAL (10*60)
-  /* If we're a server and initializing dns failed, retry periodically. */
   if (server_mode(options) && has_dns_init_failed())
     dns_init();
   return RETRY_DNS_INTERVAL;
 }
 
-  /* 2. Periodically, we consider force-uploading our descriptor
-   * (if we've passed our internal checks). */
-
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
 static int
 check_descriptor_callback(time_t now, const or_options_t *options)
 {
@@ -1725,6 +1896,11 @@ check_descriptor_callback(time_t now, const or_options_t *options)
   return CHECK_DESCRIPTOR_INTERVAL;
 }
 
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
 static int
 check_for_reachability_bw_callback(time_t now, const or_options_t *options)
 {
@@ -1761,13 +1937,13 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options)
   return CHECK_DESCRIPTOR_INTERVAL;
 }
 
+/**
+ * Periodic event: once a minute, (or every second if TestingTorNetwork, or
+ * during client bootstrap), check whether we want to download any
+ * networkstatus documents. */
 static int
 fetch_networkstatus_callback(time_t now, const or_options_t *options)
 {
-  /* 2c. Every minute (or every second if TestingTorNetwork, or during
-   * client bootstrap), check whether we want to download any networkstatus
-   * documents. */
-
   /* How often do we check whether we should download network status
    * documents? */
   const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
@@ -1789,12 +1965,13 @@ fetch_networkstatus_callback(time_t now, const or_options_t *options)
   return networkstatus_dl_check_interval;
 }
 
+/**
+ * Periodic callback: Every 60 seconds, we relaunch listeners if any died. */
 static int
 retry_listeners_callback(time_t now, const or_options_t *options)
 {
   (void)now;
   (void)options;
-  /* 3d. And every 60 seconds, we relaunch listeners if any died. */
   if (!net_is_disabled()) {
     retry_all_listeners(NULL, NULL, 0);
     return 60;
@@ -1802,6 +1979,9 @@ retry_listeners_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
 static int
 expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
 {
@@ -1811,6 +1991,10 @@ expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
   return 11;
 }
 
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
 static int
 check_dns_honesty_callback(time_t now, const or_options_t *options)
 {
@@ -1833,6 +2017,10 @@ check_dns_honesty_callback(time_t now, const or_options_t *options)
   return 12*3600 + crypto_rand_int(12*3600);
 }
 
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
 static int
 write_bridge_ns_callback(time_t now, const or_options_t *options)
 {
@@ -1845,6 +2033,9 @@ write_bridge_ns_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
+/**
+ * Periodic callback: poke the tor-fw-helper app if we're using one.
+ */
 static int
 check_fw_helper_app_callback(time_t now, const or_options_t *options)
 {
@@ -1868,7 +2059,9 @@ check_fw_helper_app_callback(time_t now, const or_options_t *options)
   return PORT_FORWARDING_CHECK_INTERVAL;
 }
 
-/** Callback to write heartbeat message in the logs. */
+/**
+ * Periodic callback: write the heartbeat message in the logs.
+ */
 static int
 heartbeat_callback(time_t now, const or_options_t *options)
 {
@@ -1889,6 +2082,17 @@ heartbeat_callback(time_t now, const or_options_t *options)
   return options->HeartbeatPeriod;
 }
 
+#define CDM_CLEAN_CALLBACK_INTERVAL 600
+static int
+clean_consdiffmgr_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (server_mode(options)) {
+    consdiffmgr_cleanup();
+  }
+  return CDM_CLEAN_CALLBACK_INTERVAL;
+}
+
 /** Timer: used to invoke second_elapsed_callback() once per second. */
 static periodic_timer_t *second_timer = NULL;
 /** Number of libevent errors in the last second: we die if we get too many. */
@@ -2179,8 +2383,9 @@ do_hup(void)
     /* Maybe we've been given a new ed25519 key or certificate?
      */
     time_t now = approx_time();
-    if (load_ed_keys(options, now) < 0 ||
-         generate_ed_link_cert(options, now)) {
+    int new_signing_key = load_ed_keys(options, now);
+    if (new_signing_key < 0 ||
+        generate_ed_link_cert(options, now, new_signing_key > 0)) {
       log_warn(LD_OR, "Problem reloading Ed25519 keys; still using old keys.");
     }
 
@@ -2217,6 +2422,8 @@ do_main_loop(void)
   }
 
   handle_signals(1);
+  monotime_init();
+  timers_initialize();
 
   /* load the private keys, if we're supposed to have them, and set up the
    * TLS context. */
@@ -2280,18 +2487,22 @@ do_main_loop(void)
   now = time(NULL);
   directory_info_has_arrived(now, 1, 0);
 
-  if (server_mode(get_options())) {
+  if (server_mode(get_options()) || dir_server_mode(get_options())) {
     /* launch cpuworkers. Need to do this *after* we've read the onion key. */
     cpu_init();
   }
+  consdiffmgr_enable_background_compression();
 
   /* Setup shared random protocol subsystem. */
-  if (authdir_mode_publishes_statuses(get_options())) {
+  if (authdir_mode_v3(get_options())) {
     if (sr_init(1) < 0) {
       return -1;
     }
   }
 
+  /* Initialize relay-side HS circuitmap */
+  hs_circuitmap_init();
+
   /* set up once-a-second callback. */
   if (! second_timer) {
     struct timeval one_second;
@@ -2374,19 +2585,26 @@ run_main_loop_once(void)
   /* Make it easier to tell whether libevent failure is our fault or not. */
   errno = 0;
 #endif
-  /* All active linked conns should get their read events activated. */
+
+  /* All active linked conns should get their read events activated,
+   * so that libevent knows to run their callbacks. */
   SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn,
                     event_active(conn->read_event, EV_READ, 1));
   called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0;
 
+  /* Make sure we know (about) what time it is. */
   update_approx_time(time(NULL));
 
-  /* poll until we have an event, or the second ends, or until we have
-   * some active linked connections to trigger events for. */
+  /* Here it is: the main loop.  Here we tell Libevent to poll until we have
+   * an event, or the second ends, or until we have some active linked
+   * connections to trigger events for.  Libevent will wait till one
+   * of these happens, then run all the appropriate callbacks. */
   loop_result = event_base_loop(tor_libevent_get_base(),
                                 called_loop_once ? EVLOOP_ONCE : 0);
 
-  /* let catch() handle things like ^c, and otherwise don't worry about it */
+  /* Oh, the loop failed.  That might be an error that we need to
+   * catch, but more likely, it's just an interrupted poll() call or something,
+   * and we should try again. */
   if (loop_result < 0) {
     int e = tor_socket_errno(-1);
     /* let the program survive things like ^z */
@@ -2409,9 +2627,17 @@ run_main_loop_once(void)
     }
   }
 
-  /* This will be pretty fast if nothing new is pending. Note that this gets
-   * called once per libevent loop, which will make it happen once per group
-   * of events that fire, or once per second. */
+  /* And here is where we put callbacks that happen "every time the event loop
+   * runs."  They must be very fast, or else the whole Tor process will get
+   * slowed down.
+   *
+   * Note that this gets called once per libevent loop, which will make it
+   * happen once per group of events that fire, or once per second. */
+
+  /* If there are any pending client connections, try attaching them to
+   * circuits (if we can.)  This will be pretty fast if nothing new is
+   * pending.
+   */
   connection_ap_attach_pending(0);
 
   return 1;
@@ -2789,6 +3015,7 @@ tor_init(int argc, char *argv[])
   rep_hist_init();
   /* Initialize the service cache. */
   rend_cache_init();
+  hs_cache_init();
   addressmap_init(); /* Init the client dns cache. Do it always, since it's
                       * cheap. */
 
@@ -2835,11 +3062,16 @@ tor_init(int argc, char *argv[])
     const char *version = get_version();
 
     log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, "
-               "OpenSSL %s and Zlib %s.", version,
+               "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version,
                get_uname(),
                tor_libevent_get_version_str(),
                crypto_openssl_get_version_str(),
-               tor_zlib_get_version_str());
+               tor_compress_supports_method(ZLIB_METHOD) ?
+                 tor_compress_version_str(ZLIB_METHOD) : "N/A",
+               tor_compress_supports_method(LZMA_METHOD) ?
+                 tor_compress_version_str(LZMA_METHOD) : "N/A",
+               tor_compress_supports_method(ZSTD_METHOD) ?
+                 tor_compress_version_str(ZSTD_METHOD) : "N/A");
 
     log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! "
                "Learn how to be safe at "
@@ -2850,6 +3082,15 @@ tor_init(int argc, char *argv[])
                  "Expect more bugs than usual.");
   }
 
+  {
+    rust_str_t rust_str = rust_welcome_string();
+    const char *s = rust_str_get(rust_str);
+    if (strlen(s) > 0) {
+      log_notice(LD_GENERAL, "%s", s);
+    }
+    rust_str_free(rust_str);
+  }
+
   if (network_init()<0) {
     log_err(LD_BUG,"Error initializing network; exiting.");
     return -1;
@@ -2864,6 +3105,13 @@ tor_init(int argc, char *argv[])
   /* The options are now initialised */
   const or_options_t *options = get_options();
 
+  /* Initialize channelpadding parameters to defaults until we get
+   * a consensus */
+  channelpadding_new_consensus_params(NULL);
+
+  /* Initialize predicted ports list after loading options */
+  predicted_ports_init();
+
 #ifndef _WIN32
   if (geteuid()==0)
     log_warn(LD_GENERAL,"You are running Tor as root. You don't need to, "
@@ -2921,7 +3169,7 @@ try_locking(const or_options_t *options, int err_if_locked)
         r = try_locking(options, 0);
         if (r<0) {
           log_err(LD_GENERAL, "No, it's still there.  Exiting.");
-          exit(0);
+          exit(1);
         }
         return r;
       }
@@ -2972,6 +3220,7 @@ tor_free_all(int postfork)
   rend_service_free_all();
   rend_cache_free_all();
   rend_service_authorization_free_all();
+  hs_cache_free_all();
   rep_hist_free_all();
   dns_free_all();
   clear_pending_onions();
@@ -2984,12 +3233,15 @@ tor_free_all(int postfork)
   connection_edge_free_all();
   scheduler_free_all();
   nodelist_free_all();
+  hs_circuitmap_free_all();
   microdesc_free_all();
   routerparse_free_all();
   ext_orport_free_all();
   control_free_all();
   sandbox_free_getaddrinfo_cache();
   protover_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
   dos_free_all();
   if (!postfork) {
     config_free_all();
@@ -3020,6 +3272,7 @@ tor_free_all(int postfork)
   if (!postfork) {
     escaped(NULL);
     esc_router_info(NULL);
+    clean_up_backtrace_handler();
     logs_free_all(); /* free log strings. do this last so logs keep working. */
   }
 }
@@ -3057,6 +3310,9 @@ tor_cleanup(void)
       rep_hist_record_mtbf_data(now, 0);
     keypin_close_journal();
   }
+
+  timers_shutdown();
+
 #ifdef USE_DMALLOC
   dmalloc_log_stats();
 #endif
@@ -3412,6 +3668,8 @@ sandbox_init_filter(void)
     OPEN_DATADIR("stats");
     STAT_DATADIR("stats");
     STAT_DATADIR2("stats", "dirreq-stats");
+
+    consdiffmgr_register_with_sandbox(&cfg);
   }
 
   init_addrinfo();
@@ -3452,13 +3710,15 @@ tor_main(int argc, char *argv[])
 
   update_approx_time(time(NULL));
   tor_threads_init();
+  tor_compress_init();
   init_logging(0);
+  monotime_init();
 #ifdef USE_DMALLOC
   {
     /* Instruct OpenSSL to use our internal wrappers for malloc,
        realloc and free. */
-    int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
-    tor_assert(r);
+    int r = crypto_use_tor_alloc_functions();
+    tor_assert(r == 0);
   }
 #endif
 #ifdef NT_SERVICE
@@ -3486,8 +3746,6 @@ tor_main(int argc, char *argv[])
 #endif
   }
 
-  monotime_init();
-
   switch (get_options()->command) {
   case CMD_RUN_TOR:
 #ifdef NT_SERVICE
@@ -3496,7 +3754,7 @@ tor_main(int argc, char *argv[])
     result = do_main_loop();
     break;
   case CMD_KEYGEN:
-    result = load_ed_keys(get_options(), time(NULL));
+    result = load_ed_keys(get_options(), time(NULL)) < 0;
     break;
   case CMD_LIST_FINGERPRINT:
     result = do_list_fingerprint();
diff --git a/src/or/main.h b/src/or/main.h
index 07b22598b1..57aa372750 100644
--- a/src/or/main.h
+++ b/src/or/main.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -92,6 +92,9 @@ STATIC void init_connection_lists(void);
 STATIC void close_closeable_connections(void);
 STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *connection_array;
+#endif
 #endif
 
 #endif
diff --git a/src/or/microdesc.c b/src/or/microdesc.c
index a81dc54628..32242d0054 100644
--- a/src/or/microdesc.c
+++ b/src/or/microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2016, The Tor Project, Inc. */
+/* Copyright (c) 2009-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,6 +74,102 @@ HT_GENERATE2(microdesc_map, microdesc_t, node,
              microdesc_hash_, microdesc_eq_, 0.6,
              tor_reallocarray_, tor_free_)
 
+/************************* md fetch fail cache *****************************/
+
+/* If we end up with too many outdated dirservers, something probably went
+ * wrong so clean up the list. */
+#define TOO_MANY_OUTDATED_DIRSERVERS 30
+
+/** List of dirservers with outdated microdesc information. The smartlist is
+ *  filled with the hex digests of outdated dirservers. */
+static smartlist_t *outdated_dirserver_list = NULL;
+
+/** Note that we failed to fetch a microdescriptor from the relay with
+ *  <b>relay_digest</b> (of size DIGEST_LEN). */
+void
+microdesc_note_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  /* Don't register outdated dirservers if we don't have a live consensus,
+   * since we might be trying to fetch microdescriptors that are not even
+   * currently active. */
+  if (!networkstatus_get_live_consensus(approx_time())) {
+    return;
+  }
+
+  if (!outdated_dirserver_list) {
+    outdated_dirserver_list = smartlist_new();
+  }
+
+  tor_assert(outdated_dirserver_list);
+
+  /* If the list grows too big, clean it up */
+  if (BUG(smartlist_len(outdated_dirserver_list) >
+          TOO_MANY_OUTDATED_DIRSERVERS)) {
+    microdesc_reset_outdated_dirservers_list();
+  }
+
+  /* Turn the binary relay digest to a hex since smartlists have better support
+   * for strings than digests. */
+  base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Make sure we don't add a dirauth as an outdated dirserver */
+  if (router_get_trusteddirserver_by_digest(relay_digest)) {
+    log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
+    return;
+  }
+
+  /* Don't double-add outdated dirservers */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return;
+  }
+
+  /* Add it to the list of outdated dirservers */
+  smartlist_add_strdup(outdated_dirserver_list, relay_hexdigest);
+
+  log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
+}
+
+/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
+ *  outdated dirserver */
+int
+microdesc_relay_is_outdated_dirserver(const char *relay_digest)
+{
+  char relay_hexdigest[HEX_DIGEST_LEN+1];
+
+  if (!outdated_dirserver_list) {
+    return 0;
+  }
+
+  /* Convert identity digest to hex digest */
+  base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
+                relay_digest, DIGEST_LEN);
+
+  /* Last time we tried to fetch microdescs, was this directory mirror missing
+   * any mds we asked for? */
+  if (smartlist_contains_string(outdated_dirserver_list, relay_hexdigest)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Reset the list of outdated dirservers. */
+void
+microdesc_reset_outdated_dirservers_list(void)
+{
+  if (!outdated_dirserver_list) {
+    return;
+  }
+
+  SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+  smartlist_clear(outdated_dirserver_list);
+}
+
+/****************************************************************************/
+
 /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
  * On success, return the total number of bytes written, and set
  * *<b>annotation_len_out</b> to the number of bytes written as
@@ -789,6 +885,11 @@ microdesc_free_all(void)
     tor_free(the_microdesc_cache->journal_fname);
     tor_free(the_microdesc_cache);
   }
+
+  if (outdated_dirserver_list) {
+    SMARTLIST_FOREACH(outdated_dirserver_list, char *, cp, tor_free(cp));
+    smartlist_free(outdated_dirserver_list);
+  }
 }
 
 /** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
@@ -804,18 +905,6 @@ microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
   return md;
 }
 
-/** Return the mean size of decriptors added to <b>cache</b> since it was last
- * cleared.  Used to estimate the size of large downloads. */
-size_t
-microdesc_average_size(microdesc_cache_t *cache)
-{
-  if (!cache)
-    cache = get_microdesc_cache();
-  if (!cache->n_seen)
-    return 512;
-  return (size_t)(cache->total_len_seen / cache->n_seen);
-}
-
 /** Return a smartlist of all the sha256 digest of the microdescriptors that
  * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
  * to internals of <b>ns</b>; you should not free the members of the resulting
@@ -917,20 +1006,9 @@ update_microdescs_from_networkstatus(time_t now)
 int
 we_use_microdescriptors_for_circuits(const or_options_t *options)
 {
-  int ret = options->UseMicrodescriptors;
-  if (ret == -1) {
-    /* UseMicrodescriptors is "auto"; we need to decide: */
-    /* If we are configured to use bridges and none of our bridges
-     * know what a microdescriptor is, the answer is no. */
-    if (options->UseBridges && !any_bridge_supports_microdescriptors())
-      return 0;
-    /* Otherwise, we decide that we'll use microdescriptors iff we are
-     * not a server, and we're not autofetching everything. */
-    /* XXXX++ what does not being a server have to do with it? also there's
-     * a partitioning issue here where bridges differ from clients. */
-    ret = !server_mode(options) && !options->FetchUselessDescriptors;
-  }
-  return ret;
+  if (options->UseMicrodescriptors == 0)
+    return 0; /* the user explicitly picked no */
+  return 1; /* yes and auto both mean yes */
 }
 
 /** Return true iff we should try to download microdescriptors at all. */
diff --git a/src/or/microdesc.h b/src/or/microdesc.h
index 40c83139e9..1be12156a4 100644
--- a/src/or/microdesc.h
+++ b/src/or/microdesc.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,8 +32,6 @@ void microdesc_cache_clear(microdesc_cache_t *cache);
 microdesc_t *microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache,
                                                  const char *d);
 
-size_t microdesc_average_size(microdesc_cache_t *cache);
-
 smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns,
                                               microdesc_cache_t *cache,
                                               int downloadable_only,
@@ -52,5 +50,9 @@ int we_fetch_microdescriptors(const or_options_t *options);
 int we_fetch_router_descriptors(const or_options_t *options);
 int we_use_microdescriptors_for_circuits(const or_options_t *options);
 
-#endif
+void microdesc_note_outdated_dirserver(const char *relay_digest);
+int microdesc_relay_is_outdated_dirserver(const char *relay_digest);
+void microdesc_reset_outdated_dirservers_list(void);
+
+#endif /* !defined(TOR_MICRODESC_H) */
 
diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c
index d8e2c00273..fdaa469faf 100644
--- a/src/or/networkstatus.c
+++ b/src/or/networkstatus.c
@@ -1,17 +1,44 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file networkstatus.c
- * \brief Functions and structures for handling network status documents as a
- * client or cache.
+ * \brief Functions and structures for handling networkstatus documents as a
+ * client or as a directory cache.
+ *
+ * A consensus networkstatus object is created by the directory
+ * authorities.  It authenticates a set of network parameters--most
+ * importantly, the list of all the relays in the network.  This list
+ * of relays is represented as an array of routerstatus_t objects.
+ *
+ * There are currently two flavors of consensus.  With the older "NS"
+ * flavor, each relay is associated with a digest of its router
+ * descriptor. Tor instances that use this consensus keep the list of
+ * router descriptors as routerinfo_t objects stored and managed in
+ * routerlist.c.  With the newer "microdesc" flavor, each relay is
+ * associated with a digest of the microdescriptor that the authorities
+ * made for it.  These are stored and managed in microdesc.c. Information
+ * about the router is divided between the the networkstatus and the
+ * microdescriptor according to the general rule that microdescriptors
+ * should hold information that changes much less frequently than the
+ * information in the networkstatus.
+ *
+ * Modern clients use microdescriptor networkstatuses.  Directory caches
+ * need to keep both kinds of networkstatus document, so they can serve them.
+ *
+ * This module manages fetching, holding, storing, updating, and
+ * validating networkstatus objects.  The download-and-validate process
+ * is slightly complicated by the fact that the keys you need to
+ * validate a consensus are stored in the authority certificates, which
+ * you might not have yet when you download the consensus.
  */
 
 #define NETWORKSTATUS_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "channel.h"
 #include "circuitmux.h"
 #include "circuitmux_ewma.h"
@@ -19,6 +46,7 @@
 #include "config.h"
 #include "connection.h"
 #include "connection_or.h"
+#include "consdiffmgr.h"
 #include "control.h"
 #include "directory.h"
 #include "dirserv.h"
@@ -37,6 +65,7 @@
 #include "shared_random.h"
 #include "transports.h"
 #include "torcert.h"
+#include "channelpadding.h"
 
 /** Map from lowercase nickname to identity digest of named server, if any. */
 static strmap_t *named_server_map = NULL;
@@ -46,11 +75,11 @@ static strmap_t *unnamed_server_map = NULL;
 
 /** Most recently received and validated v3 "ns"-flavored consensus network
  * status. */
-static networkstatus_t *current_ns_consensus = NULL;
+STATIC networkstatus_t *current_ns_consensus = NULL;
 
 /** Most recently received and validated v3 "microdec"-flavored consensus
  * network status. */
-static networkstatus_t *current_md_consensus = NULL;
+STATIC networkstatus_t *current_md_consensus = NULL;
 
 /** A v3 consensus networkstatus that we've received, but which we don't
  * have enough certificates to be happy about. */
@@ -152,53 +181,74 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static char *
+networkstatus_read_cached_consensus_impl(int flav,
+                                         const char *flavorname,
+                                         int unverified_consensus)
+{
+  char buf[128];
+  const char *prefix;
+  if (unverified_consensus) {
+    prefix = "unverified";
+  } else {
+    prefix = "cached";
+  }
+  if (flav == FLAV_NS) {
+    tor_snprintf(buf, sizeof(buf), "%s-consensus", prefix);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
+  }
+
+  char *filename = get_datadir_fname(buf);
+  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  tor_free(filename);
+  return result;
+}
+
+/** Return a new string containing the current cached consensus of flavor
+ * <b>flavorname</b>. */
+char *
+networkstatus_read_cached_consensus(const char *flavorname)
+ {
+  int flav = networkstatus_parse_flavor_name(flavorname);
+  if (flav < 0)
+    return NULL;
+  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+}
+
 /** Read every cached v3 consensus networkstatus from the disk. */
 int
 router_reload_consensus_networkstatus(void)
 {
-  char *filename;
-  char *s;
   const unsigned int flags = NSSET_FROM_CACHE | NSSET_DONT_DOWNLOAD_CERTS;
   int flav;
 
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
-    char buf[128];
     const char *flavor = networkstatus_get_flavor_name(flav);
-    if (flav == FLAV_NS) {
-      filename = get_datadir_fname("cached-consensus");
-    } else {
-      tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor);
-      filename = get_datadir_fname(buf);
-    }
-    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
     if (s) {
       if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
-                 flavor, filename);
+        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
+                 flavor);
       }
       tor_free(s);
     }
-    tor_free(filename);
 
-    if (flav == FLAV_NS) {
-      filename = get_datadir_fname("unverified-consensus");
-    } else {
-      tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor);
-      filename = get_datadir_fname(buf);
-    }
-
-    s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
     if (s) {
       if (networkstatus_set_current_consensus(s, flavor,
                                      flags|NSSET_WAS_WAITING_FOR_CERTS,
                                      NULL)) {
-      log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"",
-               flavor, filename);
-    }
+        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
+                 "from cache", flavor);
+      }
       tor_free(s);
     }
-    tor_free(filename);
   }
 
   if (!networkstatus_get_latest_consensus()) {
@@ -217,7 +267,7 @@ router_reload_consensus_networkstatus(void)
 }
 
 /** Free all storage held by the vote_routerstatus object <b>rs</b>. */
-STATIC void
+void
 vote_routerstatus_free(vote_routerstatus_t *rs)
 {
   vote_microdesc_hash_t *h, *next;
@@ -789,8 +839,11 @@ networkstatus_nickname_is_unnamed(const char *nickname)
 #define NONAUTHORITY_NS_CACHE_INTERVAL (60*60)
 
 /** Return true iff, given the options listed in <b>options</b>, <b>flavor</b>
- *  is the flavor of a consensus networkstatus that we would like to fetch. */
-static int
+ *  is the flavor of a consensus networkstatus that we would like to fetch.
+ *
+ * For certificate fetches, use we_want_to_fetch_unknown_auth_certs, and
+ * for serving fetched documents, use directory_caches_dir_info. */
+int
 we_want_to_fetch_flavor(const or_options_t *options, int flavor)
 {
   if (flavor < 0 || flavor > N_CONSENSUS_FLAVORS) {
@@ -812,6 +865,29 @@ we_want_to_fetch_flavor(const or_options_t *options, int flavor)
   return flavor == usable_consensus_flavor();
 }
 
+/** Return true iff, given the options listed in <b>options</b>, we would like
+ * to fetch and store unknown authority certificates.
+ *
+ * For consensus and descriptor fetches, use we_want_to_fetch_flavor, and
+ * for serving fetched certificates, use directory_caches_unknown_auth_certs.
+ */
+int
+we_want_to_fetch_unknown_auth_certs(const or_options_t *options)
+{
+  if (authdir_mode_v3(options) ||
+      directory_caches_unknown_auth_certs((options))) {
+    /* We want to serve all certs to others, regardless if we would use
+     * them ourselves. */
+    return 1;
+  }
+  if (options->FetchUselessDescriptors) {
+    /* Unknown certificates are definitely useless. */
+    return 1;
+  }
+  /* Otherwise, don't fetch unknown certificates. */
+  return 0;
+}
+
 /** How long will we hang onto a possibly live consensus for which we're
  * fetching certs before we check whether there is a better one? */
 #define DELAY_WHILE_FETCHING_CERTS (20*60)
@@ -1326,6 +1402,32 @@ networkstatus_get_live_consensus,(time_t now))
     return NULL;
 }
 
+/** Determine if <b>consensus</b> is valid or expired recently enough that
+ * we can still use it.
+ *
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ */
+int
+networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                        time_t now)
+{
+  if (BUG(!consensus))
+    return 0;
+
+  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+                                                      now);
+}
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_until
+ * time rather than an entire consensus. */
+int
+networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                             time_t now)
+{
+#define REASONABLY_LIVE_TIME (24*60*60)
+  return (now <= valid_until + REASONABLY_LIVE_TIME);
+}
+
 /* XXXX remove this in favor of get_live_consensus. But actually,
  * leave something like it for bridge users, who need to not totally
  * lose if they spend a while fetching a new consensus. */
@@ -1334,12 +1436,11 @@ networkstatus_get_live_consensus,(time_t now))
 networkstatus_t *
 networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
       consensus->valid_after <= now &&
-      now <= consensus->valid_until+REASONABLY_LIVE_TIME)
+      networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
     return NULL;
@@ -1711,9 +1812,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   if (flav != usable_consensus_flavor() &&
-      !directory_caches_dir_info(options)) {
-    /* This consensus is totally boring to us: we won't use it, and we won't
-     * serve it.  Drop it. */
+      !we_want_to_fetch_flavor(options, flav)) {
+    /* This consensus is totally boring to us: we won't use it, we didn't want
+     * it, and we won't serve it.  Drop it. */
     goto done;
   }
 
@@ -1904,6 +2005,7 @@ networkstatus_set_current_consensus(const char *consensus,
 
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
+    channelpadding_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -1914,11 +2016,15 @@ networkstatus_set_current_consensus(const char *consensus,
       download_status_failed(&consensus_dl_status[flav], 0);
   }
 
-  if (directory_caches_dir_info(options)) {
+  if (we_want_to_fetch_flavor(options, flav)) {
     dirserv_set_cached_consensus_networkstatus(consensus,
                                                flavor,
                                                &c->digests,
+                                               c->digest_sha3_as_signed,
                                                c->valid_after);
+    if (dir_server_mode(get_options())) {
+      consdiffmgr_add_consensus(consensus, c);
+    }
   }
 
   if (!from_cache) {
@@ -1943,6 +2049,9 @@ networkstatus_set_current_consensus(const char *consensus,
                     "CLOCK_SKEW MIN_SKEW=%ld SOURCE=CONSENSUS", delta);
   }
 
+  /* We got a new consesus. Reset our md fetch fail cache */
+  microdesc_reset_outdated_dirservers_list();
+
   router_dir_info_changed();
 
   result = 0;
@@ -2210,13 +2319,23 @@ networkstatus_dump_bridge_status_to_file(time_t now)
   char *thresholds = NULL;
   char *published_thresholds_and_status = NULL;
   char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
 
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
   format_iso_time(published, now);
   dirserv_compute_bridge_flag_thresholds();
   thresholds = dirserv_get_flag_thresholds_line();
   tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s",
-               published, thresholds, status);
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
   tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges",
                options->DataDirectory);
   write_str_to_file(fname,published_thresholds_and_status,0);
@@ -2224,6 +2343,7 @@ networkstatus_dump_bridge_status_to_file(time_t now)
   tor_free(published_thresholds_and_status);
   tor_free(fname);
   tor_free(status);
+  tor_free(fingerprint_line);
 }
 
 /* DOCDOC get_net_param_from_list */
@@ -2285,6 +2405,25 @@ networkstatus_get_param(const networkstatus_t *ns, const char *param_name,
 }
 
 /**
+ * As networkstatus_get_param(), but check torrc_value before checking the
+ * consensus. If torrc_value is in-range, then return it instead of the
+ * value from the consensus.
+ */
+int32_t
+networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                    int32_t torrc_value,
+                                    const char *param_name,
+                                    int32_t default_val,
+                                    int32_t min_val, int32_t max_val)
+{
+  if (torrc_value >= min_val && torrc_value <= max_val)
+    return torrc_value;
+  else
+    return networkstatus_get_param(
+                         ns, param_name, default_val, min_val, max_val);
+}
+
+/**
  * Retrieve the consensus parameter that governs the
  * fixed-point precision of our network balancing 'bandwidth-weights'
  * (which are themselves integer consensus values). We divide them
@@ -2360,12 +2499,11 @@ networkstatus_parse_flavor_name(const char *flavname)
  * running, or otherwise not a descriptor that we would make any
  * use of even if we had it. Else return 1. */
 int
-client_would_use_router(const routerstatus_t *rs, time_t now,
-                        const or_options_t *options)
+client_would_use_router(const routerstatus_t *rs, time_t now)
 {
-  if (!rs->is_flagged_running && !options->FetchUselessDescriptors) {
+  if (!rs->is_flagged_running) {
     /* If we had this router descriptor, we wouldn't even bother using it.
-     * But, if we want to have a complete list, fetch it anyway. */
+     * (Fetching and storing depends on by we_want_to_fetch_flavor().) */
     return 0;
   }
   if (rs->published_on + OLD_ROUTER_DESC_MAX_AGE < now) {
diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h
index 71f36b69ed..e774c4d266 100644
--- a/src/or/networkstatus.h
+++ b/src/or/networkstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,7 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
+char *networkstatus_read_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
 void routerstatus_free(routerstatus_t *rs);
 void networkstatus_vote_free(networkstatus_t *ns);
@@ -66,6 +67,8 @@ const routerstatus_t *router_get_consensus_status_by_nickname(
                                    int warn_if_unnamed);
 const char *networkstatus_get_router_digest_by_nickname(const char *nickname);
 int networkstatus_nickname_is_unnamed(const char *nickname);
+int we_want_to_fetch_flavor(const or_options_t *options, int flavor);
+int we_want_to_fetch_unknown_auth_certs(const or_options_t *options);
 void networkstatus_consensus_download_failed(int status_code,
                                              const char *flavname);
 void update_consensus_networkstatus_fetch_time(time_t now);
@@ -73,12 +76,15 @@ int should_delay_dir_fetches(const or_options_t *options,const char **msg_out);
 void update_networkstatus_downloads(time_t now);
 void update_certificate_downloads(time_t now);
 int consensus_is_waiting_for_certs(void);
-int client_would_use_router(const routerstatus_t *rs, time_t now,
-                            const or_options_t *options);
+int client_would_use_router(const routerstatus_t *rs, time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void));
 MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor,
           (consensus_flavor_t f));
 MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
+int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
+                                            time_t now);
+int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
+                                                 time_t now);
 networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now,
                                                              int flavor);
 MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now));
@@ -111,6 +117,11 @@ int32_t networkstatus_get_param(const networkstatus_t *ns,
                                 const char *param_name,
                                 int32_t default_val, int32_t min_val,
                                 int32_t max_val);
+int32_t networkstatus_get_overridable_param(const networkstatus_t *ns,
+                                            int32_t torrc_value,
+                                            const char *param_name,
+                                            int32_t default_val,
+                                            int32_t min_val, int32_t max_val);
 int getinfo_helper_networkstatus(control_connection_t *conn,
                                  const char *question, char **answer,
                                  const char **errmsg);
@@ -123,12 +134,15 @@ document_signature_t *document_signature_dup(const document_signature_t *sig);
 void networkstatus_free_all(void);
 int networkstatus_get_weight_scale_param(networkstatus_t *ns);
 
+void vote_routerstatus_free(vote_routerstatus_t *rs);
+
 #ifdef NETWORKSTATUS_PRIVATE
-STATIC void vote_routerstatus_free(vote_routerstatus_t *rs);
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
                                                 const char *flavor);
-#endif // TOR_UNIT_TESTS
+extern networkstatus_t *current_ns_consensus;
+extern networkstatus_t *current_md_consensus;
+#endif
 #endif
 
 #endif
diff --git a/src/or/nodelist.c b/src/or/nodelist.c
index 26f990b08c..aaec39f7b8 100644
--- a/src/or/nodelist.c
+++ b/src/or/nodelist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,6 +10,32 @@
  * \brief Structures and functions for tracking what we know about the routers
  *   on the Tor network, and correlating information from networkstatus,
  *   routerinfo, and microdescs.
+ *
+ * The key structure here is node_t: that's the canonical way to refer
+ * to a Tor relay that we might want to build a circuit through.  Every
+ * node_t has either a routerinfo_t, or a routerstatus_t from the current
+ * networkstatus consensus.  If it has a routerstatus_t, it will also
+ * need to have a microdesc_t before you can use it for circuits.
+ *
+ * The nodelist_t is a global singleton that maps identities to node_t
+ * objects.  Access them with the node_get_*() functions.  The nodelist_t
+ * is maintained by calls throughout the codebase
+ *
+ * Generally, other code should not have to reach inside a node_t to
+ * see what information it has.  Instead, you should call one of the
+ * many accessor functions that works on a generic node_t.  If there
+ * isn't one that does what you need, it's better to make such a function,
+ * and then use it.
+ *
+ * For historical reasons, some of the functions that select a node_t
+ * from the list of all usable node_t objects are in the routerlist.c
+ * module, since they originally selected a routerinfo_t. (TODO: They
+ * should move!)
+ *
+ * (TODO: Perhaps someday we should abstract the remaining ways of
+ * talking about a relay to also be node_t instances. Those would be
+ * routerstatus_t as used for directory requests, and dir_server_t as
+ * used for authorities and fallback directories.)
  */
 
 #include "or.h"
@@ -18,16 +44,19 @@
 #include "config.h"
 #include "control.h"
 #include "dirserv.h"
+#include "entrynodes.h"
 #include "geoip.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
+#include "protover.h"
 #include "rendservice.h"
 #include "router.h"
 #include "routerlist.h"
 #include "routerset.h"
+#include "torcert.h"
 
 #include <string.h>
 
@@ -46,7 +75,6 @@ static void count_usable_descriptors(int *num_present,
                                      int *num_usable,
                                      smartlist_t *descs_out,
                                      const networkstatus_t *consensus,
-                                     const or_options_t *options,
                                      time_t now,
                                      routerset_t *in_set,
                                      usable_descriptor_t exit_only);
@@ -697,6 +725,73 @@ node_get_by_nickname,(const char *nickname, int warn_if_unnamed))
   }
 }
 
+/** Return the Ed25519 identity key for the provided node, or NULL if it
+ * doesn't have one. */
+const ed25519_public_key_t *
+node_get_ed25519_id(const node_t *node)
+{
+  if (node->ri) {
+    if (node->ri->cache_info.signing_key_cert) {
+      const ed25519_public_key_t *pk =
+        &node->ri->cache_info.signing_key_cert->signing_key;
+      if (BUG(ed25519_public_key_is_zero(pk)))
+        goto try_the_md;
+      return pk;
+    }
+  }
+ try_the_md:
+  if (node->md) {
+    if (node->md->ed25519_identity_pkey) {
+      return node->md->ed25519_identity_pkey;
+    }
+  }
+  return NULL;
+}
+
+/** Return true iff this node's Ed25519 identity matches <b>id</b>.
+ * (An absent Ed25519 identity matches NULL or zero.) */
+int
+node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
+{
+  const ed25519_public_key_t *node_id = node_get_ed25519_id(node);
+  if (node_id == NULL || ed25519_public_key_is_zero(node_id)) {
+    return id == NULL || ed25519_public_key_is_zero(id);
+  } else {
+    return id && ed25519_pubkey_eq(node_id, id);
+  }
+}
+
+/** Return true iff <b>node</b> supports authenticating itself
+ * by ed25519 ID during the link handshake in a way that we can understand
+ * when we probe it. */
+int
+node_supports_ed25519_link_authentication(const node_t *node)
+{
+  /* XXXX Oh hm. What if some day in the future there are link handshake
+   * versions that aren't 3 but which are ed25519 */
+  if (! node_get_ed25519_id(node))
+    return 0;
+  if (node->ri) {
+    const char *protos = node->ri->protocol_list;
+    if (protos == NULL)
+      return 0;
+    return protocol_list_supports_protocol(protos, PRT_LINKAUTH, 3);
+  }
+  if (node->rs) {
+    return node->rs->supports_ed25519_link_handshake;
+  }
+  tor_assert_nonfatal_unreached_once();
+  return 0;
+}
+
+/** Return the RSA ID key's SHA1 digest for the provided node. */
+const uint8_t *
+node_get_rsa_id_digest(const node_t *node)
+{
+  tor_assert(node);
+  return (const uint8_t*)node->identity;
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -992,16 +1087,6 @@ node_get_platform(const node_t *node)
     return NULL;
 }
 
-/** Return <b>node</b>'s time of publication, or 0 if we don't have one. */
-time_t
-node_get_published_on(const node_t *node)
-{
-  if (node->ri)
-    return node->ri->cache_info.published_on;
-  else
-    return 0;
-}
-
 /** Return true iff <b>node</b> is one representing this router. */
 int
 node_is_me(const node_t *node)
@@ -1146,9 +1231,11 @@ node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out)
   node_assert_ok(node);
   tor_assert(ap_out);
 
-  /* Prefer routerstatus over microdesc for consistency with the
-   * fascist_firewall_* functions. Also check if the address or port are valid,
-   * and try another alternative if they are not. */
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
 
   if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
                                          node->ri->ipv6_orport, 0)) {
@@ -1208,6 +1295,9 @@ node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out)
   node_assert_ok(node);
   tor_assert(ap_out);
 
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address. */
+
   RETURN_IPV4_AP(node->ri, dir_port, ap_out);
   RETURN_IPV4_AP(node->rs, dir_port, ap_out);
   /* Microdescriptors only have an IPv6 address */
@@ -1240,8 +1330,11 @@ node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out)
   node_assert_ok(node);
   tor_assert(ap_out);
 
-  /* Check if the address or port are valid, and try another alternative if
-   * they are not. Note that microdescriptors have no dir_port. */
+  /* Check ri first, because rewrite_node_address_for_bridge() updates
+   * node->ri with the configured bridge address.
+   * Prefer rs over md for consistency with the fascist_firewall_* functions.
+   * Check if the address or port are valid, and try another alternative
+   * if they are not. */
 
   /* Assume IPv4 and IPv6 dirports are the same */
   if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr,
@@ -1321,7 +1414,7 @@ nodelist_refresh_countries(void)
 
 /** Return true iff router1 and router2 have similar enough network addresses
  * that we should treat them as being in the same family */
-static inline int
+int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
@@ -1628,8 +1721,8 @@ router_have_minimum_dir_info(void)
  * this can cause router_have_consensus_path() to be set to
  * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies.
  */
-consensus_path_type_t
-router_have_consensus_path(void)
+MOCK_IMPL(consensus_path_type_t,
+router_have_consensus_path, (void))
 {
   return have_consensus_path;
 }
@@ -1668,7 +1761,7 @@ static void
 count_usable_descriptors(int *num_present, int *num_usable,
                          smartlist_t *descs_out,
                          const networkstatus_t *consensus,
-                         const or_options_t *options, time_t now,
+                         time_t now,
                          routerset_t *in_set,
                          usable_descriptor_t exit_only)
 {
@@ -1685,7 +1778,7 @@ count_usable_descriptors(int *num_present, int *num_usable,
          continue;
        if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1))
          continue;
-       if (client_would_use_router(rs, now, options)) {
+       if (client_would_use_router(rs, now)) {
          const char * const digest = rs->descriptor_digest;
          int present;
          ++*num_usable; /* the consensus says we want it. */
@@ -1718,9 +1811,9 @@ count_usable_descriptors(int *num_present, int *num_usable,
  * If **<b>status_out</b> is present, allocate a new string and print the
  * available percentages of guard, middle, and exit nodes to it, noting
  * whether there are exits in the consensus.
- * If there are no guards in the consensus,
- * we treat the exit fraction as 100%.
- */
+ * If there are no exits in the consensus, we treat the exit fraction as 100%,
+ * but set router_have_consensus_path() so that we can only build internal
+ * paths. */
 static double
 compute_frac_paths_available(const networkstatus_t *consensus,
                              const or_options_t *options, time_t now,
@@ -1739,10 +1832,10 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, options, now, NULL,
+                           mid, consensus, now, NULL,
                            USABLE_DESCRIPTOR_ALL);
   if (options->EntryNodes) {
-    count_usable_descriptors(&np, &nu, guards, consensus, options, now,
+    count_usable_descriptors(&np, &nu, guards, consensus, now,
                              options->EntryNodes, USABLE_DESCRIPTOR_ALL);
   } else {
     SMARTLIST_FOREACH(mid, const node_t *, node, {
@@ -1763,7 +1856,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
    * an unavoidable feature of forcing authorities to declare
    * certain nodes as exits.
    */
-  count_usable_descriptors(&np, &nu, exits, consensus, options, now,
+  count_usable_descriptors(&np, &nu, exits, consensus, now,
                            NULL, USABLE_DESCRIPTOR_EXIT_ONLY);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
@@ -1812,7 +1905,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
     smartlist_t *myexits_unflagged = smartlist_new();
 
     /* All nodes with exit flag in ExitNodes option */
-    count_usable_descriptors(&np, &nu, myexits, consensus, options, now,
+    count_usable_descriptors(&np, &nu, myexits, consensus, now,
                              options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY);
     log_debug(LD_NET,
               "%s: %d present, %d usable",
@@ -1823,7 +1916,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
     /* Now compute the nodes in the ExitNodes option where which we don't know
      * what their exit policy is, or we know it permits something. */
     count_usable_descriptors(&np, &nu, myexits_unflagged,
-                             consensus, options, now,
+                             consensus, now,
                              options->ExitNodes, USABLE_DESCRIPTOR_ALL);
     log_debug(LD_NET,
               "%s: %d present, %d usable",
@@ -1970,6 +2063,13 @@ update_router_have_minimum_dir_info(void)
 
   using_md = consensus->flavor == FLAV_MICRODESC;
 
+  if (! entry_guards_have_enough_dir_info_to_build_circuits()) {
+    strlcpy(dir_info_status, "We're missing descriptors for some of our "
+            "primary entry guards", sizeof(dir_info_status));
+    res = 0;
+    goto done;
+  }
+
   /* Check fraction of available paths */
   {
     char *status = NULL;
diff --git a/src/or/nodelist.h b/src/or/nodelist.h
index 098f1d1555..9cd66f60a2 100644
--- a/src/or/nodelist.h
+++ b/src/or/nodelist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -54,8 +54,12 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-time_t node_get_published_on(const node_t *node);
 const smartlist_t *node_get_declared_family(const node_t *node);
+const ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+int node_ed25519_id_matches(const node_t *node,
+                            const ed25519_public_key_t *id);
+int node_supports_ed25519_link_authentication(const node_t *node);
+const uint8_t *node_get_rsa_id_digest(const node_t *node);
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -90,6 +94,8 @@ int node_is_unreliable(const node_t *router, int need_uptime,
 int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port,
                                         int need_uptime);
 void router_set_status(const char *digest, int up);
+int addrs_in_same_network_family(const tor_addr_t *a1,
+                                 const tor_addr_t *a2);
 
 /** router_have_minimum_dir_info tests to see if we have enough
  * descriptor information to create circuits.
@@ -119,7 +125,8 @@ typedef enum {
    * create exit and internal paths, circuits, streams, ... */
   CONSENSUS_PATH_EXIT = 1
 } consensus_path_type_t;
-consensus_path_type_t router_have_consensus_path(void);
+
+MOCK_DECL(consensus_path_type_t, router_have_consensus_path, (void));
 
 void router_dir_info_changed(void);
 const char *get_dir_info_status_string(void);
diff --git a/src/or/ntmain.c b/src/or/ntmain.c
index 0e6f296d24..d0d5276c48 100644
--- a/src/or/ntmain.c
+++ b/src/or/ntmain.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/ntmain.h b/src/or/ntmain.h
index 31bf38c62c..4b771b1828 100644
--- a/src/or/ntmain.h
+++ b/src/or/ntmain.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/onion.c b/src/or/onion.c
index 4b803a785c..a98b97cb1d 100644
--- a/src/or/onion.c
+++ b/src/or/onion.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -76,6 +76,9 @@
 #include "rephist.h"
 #include "router.h"
 
+// trunnel
+#include "ed25519_cert.h"
+
 /** Type for a linked list of circuits that are waiting for a free CPU worker
  * to process a waiting onion handshake. */
 typedef struct onion_queue_t {
@@ -873,13 +876,114 @@ check_extend_cell(const extend_cell_t *cell)
   return check_create_cell(&cell->create_cell, 1);
 }
 
-/** Protocol constants for specifier types in EXTEND2
- * @{
- */
-#define SPECTYPE_IPV4 0
-#define SPECTYPE_IPV6 1
-#define SPECTYPE_LEGACY_ID 2
-/** @} */
+static int
+extend_cell_from_extend1_cell_body(extend_cell_t *cell_out,
+                                   const extend1_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  memset(cell_out, 0, sizeof(*cell_out));
+  tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
+  tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
+
+  cell_out->cell_type = RELAY_COMMAND_EXTEND;
+  tor_addr_from_ipv4h(&cell_out->orport_ipv4.addr, cell->ipv4addr);
+  cell_out->orport_ipv4.port = cell->port;
+  if (tor_memeq(cell->onionskin, NTOR_CREATE_MAGIC, 16)) {
+    cell_out->create_cell.cell_type = CELL_CREATE2;
+    cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_NTOR;
+    cell_out->create_cell.handshake_len = NTOR_ONIONSKIN_LEN;
+    memcpy(cell_out->create_cell.onionskin, cell->onionskin + 16,
+           NTOR_ONIONSKIN_LEN);
+  } else {
+    cell_out->create_cell.cell_type = CELL_CREATE;
+    cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_TAP;
+    cell_out->create_cell.handshake_len = TAP_ONIONSKIN_CHALLENGE_LEN;
+    memcpy(cell_out->create_cell.onionskin, cell->onionskin,
+           TAP_ONIONSKIN_CHALLENGE_LEN);
+  }
+  memcpy(cell_out->node_id, cell->identity, DIGEST_LEN);
+  return 0;
+}
+
+static int
+create_cell_from_create2_cell_body(create_cell_t *cell_out,
+                                   const create2_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  memset(cell_out, 0, sizeof(create_cell_t));
+  if (BUG(cell->handshake_len > sizeof(cell_out->onionskin))) {
+    /* This should be impossible because there just isn't enough room in the
+     * input cell to make the handshake_len this large and provide a
+     * handshake_data to match. */
+    return -1;
+  }
+
+  cell_out->cell_type = CELL_CREATE2;
+  cell_out->handshake_type = cell->handshake_type;
+  cell_out->handshake_len = cell->handshake_len;
+  memcpy(cell_out->onionskin,
+       create2_cell_body_getconstarray_handshake_data(cell),
+       cell->handshake_len);
+  return 0;
+}
+
+static int
+extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
+                                   const extend2_cell_body_t *cell)
+{
+  tor_assert(cell_out);
+  tor_assert(cell);
+  int found_ipv4 = 0, found_ipv6 = 0, found_rsa_id = 0, found_ed_id = 0;
+  memset(cell_out, 0, sizeof(*cell_out));
+  tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
+  tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
+  cell_out->cell_type = RELAY_COMMAND_EXTEND2;
+
+  unsigned i;
+  for (i = 0; i < cell->n_spec; ++i) {
+    const link_specifier_t *ls = extend2_cell_body_getconst_ls(cell, i);
+    switch (ls->ls_type) {
+      case LS_IPV4:
+        if (found_ipv4)
+          continue;
+        found_ipv4 = 1;
+        tor_addr_from_ipv4h(&cell_out->orport_ipv4.addr, ls->un_ipv4_addr);
+        cell_out->orport_ipv4.port = ls->un_ipv4_port;
+        break;
+      case LS_IPV6:
+        if (found_ipv6)
+          continue;
+        found_ipv6 = 1;
+        tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
+                                 (const char *)ls->un_ipv6_addr);
+        cell_out->orport_ipv6.port = ls->un_ipv6_port;
+        break;
+      case LS_LEGACY_ID:
+        if (found_rsa_id)
+          return -1;
+        found_rsa_id = 1;
+        memcpy(cell_out->node_id, ls->un_legacy_id, 20);
+        break;
+      case LS_ED25519_ID:
+        if (found_ed_id)
+          return -1;
+        found_ed_id = 1;
+        memcpy(cell_out->ed_pubkey.pubkey, ls->un_ed25519_id, 32);
+        break;
+      default:
+        /* Ignore this, whatever it is. */
+        break;
+    }
+  }
+
+  if (!found_rsa_id || !found_ipv4) /* These are mandatory */
+    return -1;
+
+  return create_cell_from_create2_cell_body(&cell_out->create_cell,
+                                            cell->create2);
+}
 
 /** Parse an EXTEND or EXTEND2 cell (according to <b>command</b>) from the
  * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
@@ -888,101 +992,44 @@ int
 extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
                   const uint8_t *payload, size_t payload_length)
 {
-  const uint8_t *eop;
 
-  memset(cell_out, 0, sizeof(*cell_out));
+  tor_assert(cell_out);
+  tor_assert(payload);
+
   if (payload_length > RELAY_PAYLOAD_SIZE)
     return -1;
-  eop = payload + payload_length;
 
   switch (command) {
   case RELAY_COMMAND_EXTEND:
     {
-      if (payload_length != 6 + TAP_ONIONSKIN_CHALLENGE_LEN + DIGEST_LEN)
+      extend1_cell_body_t *cell = NULL;
+      if (extend1_cell_body_parse(&cell, payload, payload_length)<0 ||
+          cell == NULL) {
+        if (cell)
+          extend1_cell_body_free(cell);
         return -1;
-
-      cell_out->cell_type = RELAY_COMMAND_EXTEND;
-      tor_addr_from_ipv4n(&cell_out->orport_ipv4.addr, get_uint32(payload));
-      cell_out->orport_ipv4.port = ntohs(get_uint16(payload+4));
-      tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
-      if (tor_memeq(payload + 6, NTOR_CREATE_MAGIC, 16)) {
-        cell_out->create_cell.cell_type = CELL_CREATE2;
-        cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_NTOR;
-        cell_out->create_cell.handshake_len = NTOR_ONIONSKIN_LEN;
-        memcpy(cell_out->create_cell.onionskin, payload + 22,
-               NTOR_ONIONSKIN_LEN);
-      } else {
-        cell_out->create_cell.cell_type = CELL_CREATE;
-        cell_out->create_cell.handshake_type = ONION_HANDSHAKE_TYPE_TAP;
-        cell_out->create_cell.handshake_len = TAP_ONIONSKIN_CHALLENGE_LEN;
-        memcpy(cell_out->create_cell.onionskin, payload + 6,
-               TAP_ONIONSKIN_CHALLENGE_LEN);
       }
-      memcpy(cell_out->node_id, payload + 6 + TAP_ONIONSKIN_CHALLENGE_LEN,
-             DIGEST_LEN);
-      break;
+      int r = extend_cell_from_extend1_cell_body(cell_out, cell);
+      extend1_cell_body_free(cell);
+      if (r < 0)
+        return r;
     }
+    break;
   case RELAY_COMMAND_EXTEND2:
     {
-      uint8_t n_specs, spectype, speclen;
-      int i;
-      int found_ipv4 = 0, found_ipv6 = 0, found_id = 0;
-      tor_addr_make_unspec(&cell_out->orport_ipv4.addr);
-      tor_addr_make_unspec(&cell_out->orport_ipv6.addr);
-
-      if (payload_length == 0)
+      extend2_cell_body_t *cell = NULL;
+      if (extend2_cell_body_parse(&cell, payload, payload_length) < 0 ||
+          cell == NULL) {
+        if (cell)
+          extend2_cell_body_free(cell);
         return -1;
-
-      cell_out->cell_type = RELAY_COMMAND_EXTEND2;
-      n_specs = *payload++;
-      /* Parse the specifiers. We'll only take the first IPv4 and first IPv6
-       * address, and the node ID, and ignore everything else */
-      for (i = 0; i < n_specs; ++i) {
-        if (eop - payload < 2)
-          return -1;
-        spectype = payload[0];
-        speclen = payload[1];
-        payload += 2;
-        if (eop - payload < speclen)
-          return -1;
-        switch (spectype) {
-        case SPECTYPE_IPV4:
-          if (speclen != 6)
-            return -1;
-          if (!found_ipv4) {
-            tor_addr_from_ipv4n(&cell_out->orport_ipv4.addr,
-                                get_uint32(payload));
-            cell_out->orport_ipv4.port = ntohs(get_uint16(payload+4));
-            found_ipv4 = 1;
-          }
-          break;
-        case SPECTYPE_IPV6:
-          if (speclen != 18)
-            return -1;
-          if (!found_ipv6) {
-            tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
-                                     (const char*)payload);
-            cell_out->orport_ipv6.port = ntohs(get_uint16(payload+16));
-            found_ipv6 = 1;
-          }
-          break;
-        case SPECTYPE_LEGACY_ID:
-          if (speclen != 20)
-            return -1;
-          if (found_id)
-            return -1;
-          memcpy(cell_out->node_id, payload, 20);
-          found_id = 1;
-          break;
-        }
-        payload += speclen;
       }
-      if (!found_id || !found_ipv4)
-        return -1;
-      if (parse_create2_payload(&cell_out->create_cell,payload,eop-payload)<0)
-        return -1;
-      break;
+      int r = extend_cell_from_extend2_cell_body(cell_out, cell);
+      extend2_cell_body_free(cell);
+      if (r < 0)
+        return r;
     }
+    break;
   default:
     return -1;
   }
@@ -994,6 +1041,7 @@ extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
 static int
 check_extended_cell(const extended_cell_t *cell)
 {
+  tor_assert(cell);
   if (cell->created_cell.cell_type == CELL_CREATED) {
     if (cell->cell_type != RELAY_COMMAND_EXTENDED)
       return -1;
@@ -1015,6 +1063,9 @@ extended_cell_parse(extended_cell_t *cell_out,
                     const uint8_t command, const uint8_t *payload,
                     size_t payload_len)
 {
+  tor_assert(cell_out);
+  tor_assert(payload);
+
   memset(cell_out, 0, sizeof(*cell_out));
   if (payload_len > RELAY_PAYLOAD_SIZE)
     return -1;
@@ -1131,6 +1182,21 @@ created_cell_format(cell_t *cell_out, const created_cell_t *cell_in)
   return 0;
 }
 
+/** Return true iff we are configured (by torrc or by the networkstatus
+ * parameters) to use Ed25519 identities in our Extend2 cells. */
+static int
+should_include_ed25519_id_extend_cells(const networkstatus_t *ns,
+                                       const or_options_t *options)
+{
+  if (options->ExtendByEd25519ID != -1)
+    return options->ExtendByEd25519ID; /* The user has an opinion. */
+
+  return (int) networkstatus_get_param(ns, "ExtendByEd25519ID",
+                                       0 /* default */,
+                                       0 /* min */,
+                                       1 /*max*/);
+}
+
 /** Format the EXTEND{,2} cell in <b>cell_in</b>, storing its relay payload in
  * <b>payload_out</b>, the number of bytes used in *<b>len_out</b>, and the
  * relay command in *<b>command_out</b>. The <b>payload_out</b> must have
@@ -1139,12 +1205,11 @@ int
 extend_cell_format(uint8_t *command_out, uint16_t *len_out,
                    uint8_t *payload_out, const extend_cell_t *cell_in)
 {
-  uint8_t *p, *eop;
+  uint8_t *p;
   if (check_extend_cell(cell_in) < 0)
     return -1;
 
   p = payload_out;
-  eop = payload_out + RELAY_PAYLOAD_SIZE;
 
   memset(p, 0, RELAY_PAYLOAD_SIZE);
 
@@ -1167,33 +1232,56 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
     break;
   case RELAY_COMMAND_EXTEND2:
     {
-      uint8_t n = 2;
+      uint8_t n_specifiers = 2;
       *command_out = RELAY_COMMAND_EXTEND2;
-
-      *p++ = n; /* 2 identifiers */
-      *p++ = SPECTYPE_IPV4; /* First is IPV4. */
-      *p++ = 6; /* It's 6 bytes long. */
-      set_uint32(p, tor_addr_to_ipv4n(&cell_in->orport_ipv4.addr));
-      set_uint16(p+4, htons(cell_in->orport_ipv4.port));
-      p += 6;
-      *p++ = SPECTYPE_LEGACY_ID; /* Next is an identity digest. */
-      *p++ = 20; /* It's 20 bytes long */
-      memcpy(p, cell_in->node_id, DIGEST_LEN);
-      p += 20;
-
-      /* Now we can send the handshake */
-      set_uint16(p, htons(cell_in->create_cell.handshake_type));
-      set_uint16(p+2, htons(cell_in->create_cell.handshake_len));
-      p += 4;
-
-      if (cell_in->create_cell.handshake_len > eop - p)
-        return -1;
-
-      memcpy(p, cell_in->create_cell.onionskin,
+      extend2_cell_body_t *cell = extend2_cell_body_new();
+      link_specifier_t *ls;
+      {
+        /* IPv4 specifier first. */
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_IPV4;
+        ls->ls_len = 6;
+        ls->un_ipv4_addr = tor_addr_to_ipv4h(&cell_in->orport_ipv4.addr);
+        ls->un_ipv4_port = cell_in->orport_ipv4.port;
+      }
+      {
+        /* Then RSA id */
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_LEGACY_ID;
+        ls->ls_len = DIGEST_LEN;
+        memcpy(ls->un_legacy_id, cell_in->node_id, DIGEST_LEN);
+      }
+      if (should_include_ed25519_id_extend_cells(NULL, get_options()) &&
+          !ed25519_public_key_is_zero(&cell_in->ed_pubkey)) {
+        /* Then, maybe, the ed25519 id! */
+        ++n_specifiers;
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_ED25519_ID;
+        ls->ls_len = 32;
+        memcpy(ls->un_ed25519_id, cell_in->ed_pubkey.pubkey, 32);
+      }
+      cell->n_spec = n_specifiers;
+
+      /* Now, the handshake */
+      cell->create2 = create2_cell_body_new();
+      cell->create2->handshake_type = cell_in->create_cell.handshake_type;
+      cell->create2->handshake_len = cell_in->create_cell.handshake_len;
+      create2_cell_body_setlen_handshake_data(cell->create2,
+                                         cell_in->create_cell.handshake_len);
+      memcpy(create2_cell_body_getarray_handshake_data(cell->create2),
+             cell_in->create_cell.onionskin,
              cell_in->create_cell.handshake_len);
 
-      p += cell_in->create_cell.handshake_len;
-      *len_out = p - payload_out;
+      ssize_t len_encoded = extend2_cell_body_encode(
+                             payload_out, RELAY_PAYLOAD_SIZE,
+                             cell);
+      extend2_cell_body_free(cell);
+      if (len_encoded < 0 || len_encoded > UINT16_MAX)
+        return -1;
+      *len_out = (uint16_t) len_encoded;
     }
     break;
   default:
diff --git a/src/or/onion.h b/src/or/onion.h
index 0275fa00d2..37a7b08cb6 100644
--- a/src/or/onion.h
+++ b/src/or/onion.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -85,6 +85,8 @@ typedef struct extend_cell_t {
   tor_addr_port_t orport_ipv6;
   /** Identity fingerprint of the node we're conecting to.*/
   uint8_t node_id[DIGEST_LEN];
+  /** Ed25519 public identity key. Zero if not set. */
+  ed25519_public_key_t ed_pubkey;
   /** The "create cell" embedded in this extend cell. Note that unlike the
    * create cells we generate ourself, this once can have a handshake type we
    * don't recognize. */
diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c
index 8dcbfe22d8..146943a273 100644
--- a/src/or/onion_fast.c
+++ b/src/or/onion_fast.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/onion_fast.h b/src/or/onion_fast.h
index b9626002c3..b31f8e9492 100644
--- a/src/or/onion_fast.h
+++ b/src/or/onion_fast.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c
index ded97ee73d..902260b54b 100644
--- a/src/or/onion_ntor.c
+++ b/src/or/onion_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/onion_ntor.h b/src/or/onion_ntor.h
index f637b437fd..158c499de4 100644
--- a/src/or/onion_ntor.h
+++ b/src/or/onion_ntor.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ONION_NTOR_H
diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c
index 2769300945..294fc0df6d 100644
--- a/src/or/onion_tap.c
+++ b/src/or/onion_tap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -159,7 +159,7 @@ onion_skin_TAP_server_handshake(
      * big. That should be impossible. */
     log_info(LD_GENERAL, "crypto_dh_get_public failed.");
     goto err;
-    /* LCOV_EXCP_STOP */
+    /* LCOV_EXCL_STOP */
   }
 
   key_material_len = DIGEST_LEN+key_out_len;
diff --git a/src/or/onion_tap.h b/src/or/onion_tap.h
index a2880f6e98..bd625231f4 100644
--- a/src/or/onion_tap.h
+++ b/src/or/onion_tap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/or.h b/src/or/or.h
index 024a9cff0f..9e7833386c 100644
--- a/src/or/or.h
+++ b/src/or/or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,15 +71,17 @@
 #include "tortls.h"
 #include "torlog.h"
 #include "container.h"
-#include "torgzip.h"
+#include "compress.h"
 #include "address.h"
 #include "compat_libevent.h"
 #include "ht.h"
+#include "confline.h"
 #include "replaycache.h"
 #include "crypto_curve25519.h"
 #include "crypto_ed25519.h"
 #include "tor_queue.h"
 #include "util_format.h"
+#include "hs_circuitmap.h"
 
 /* These signals are defined to help handle_control_signal work.
  */
@@ -114,6 +116,9 @@
 #define NON_ANONYMOUS_MODE_ENABLED 1
 #endif
 
+/** Helper macro: Given a pointer to to.base_, of type from*, return &to. */
+#define DOWNCAST(to, ptr) ((to*)SUBTYPE_P(ptr, to, base_))
+
 /** Length of longest allowable configured nickname. */
 #define MAX_NICKNAME_LEN 19
 /** Length of a router identity encoded as a hexadecimal digest, plus
@@ -143,8 +148,27 @@
 /** Maximum size of a single extrainfo document, as above. */
 #define MAX_EXTRAINFO_UPLOAD_SIZE 50000
 
-/** How often do we rotate onion keys? */
-#define MIN_ONION_KEY_LIFETIME (7*24*60*60)
+/** Minimum lifetime for an onion key in days. */
+#define MIN_ONION_KEY_LIFETIME_DAYS (1)
+
+/** Maximum lifetime for an onion key in days. */
+#define MAX_ONION_KEY_LIFETIME_DAYS (90)
+
+/** Default lifetime for an onion key in days. */
+#define DEFAULT_ONION_KEY_LIFETIME_DAYS (28)
+
+/** Minimum grace period for acceptance of an onion key in days.
+ * The maximum value is defined in proposal #274 as being the current network
+ * consensus parameter for "onion-key-rotation-days". */
+#define MIN_ONION_KEY_GRACE_PERIOD_DAYS (1)
+
+/** Default grace period for acceptance of an onion key in days. */
+#define DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS (7)
+
+/** How often we should check the network consensus if it is time to rotate or
+ * expire onion keys. */
+#define ONION_KEY_CONSENSUS_CHECK_INTERVAL (60*60)
+
 /** How often do we rotate TLS contexts? */
 #define MAX_SSL_KEY_LIFETIME_INTERNAL (2*60*60)
 
@@ -399,12 +423,13 @@ typedef enum {
 #define DIR_PURPOSE_FETCH_MICRODESC 19
 #define DIR_PURPOSE_MAX_ 19
 
-/** True iff <b>p</b> is a purpose corresponding to uploading data to a
- * directory server. */
+/** True iff <b>p</b> is a purpose corresponding to uploading
+ * data to a directory server. */
 #define DIR_PURPOSE_IS_UPLOAD(p)                \
   ((p)==DIR_PURPOSE_UPLOAD_DIR ||               \
    (p)==DIR_PURPOSE_UPLOAD_VOTE ||              \
-   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES)
+   (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \
+   (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2)
 
 #define EXIT_PURPOSE_MIN_ 1
 /** This exit stream wants to do an ordinary connect. */
@@ -423,8 +448,12 @@ typedef enum {
 /** Circuit state: I'd like to deliver a create, but my n_chan is still
  * connecting. */
 #define CIRCUIT_STATE_CHAN_WAIT 2
+/** Circuit state: the circuit is open but we don't want to actually use it
+ * until we find out if a better guard will be available.
+ */
+#define CIRCUIT_STATE_GUARD_WAIT 3
 /** Circuit state: onionskin(s) processed, ready to send/receive cells. */
-#define CIRCUIT_STATE_OPEN 3
+#define CIRCUIT_STATE_OPEN 4
 
 #define CIRCUIT_PURPOSE_MIN_ 1
 
@@ -767,6 +796,24 @@ typedef struct rend_service_authorization_t {
  * establishment. Not all fields contain data depending on where this struct
  * is used. */
 typedef struct rend_data_t {
+  /* Hidden service protocol version of this base object. */
+  uint32_t version;
+
+  /** List of HSDir fingerprints on which this request has been sent to. This
+   * contains binary identity digest of the directory of size DIGEST_LEN. */
+  smartlist_t *hsdirs_fp;
+
+  /** Rendezvous cookie used by both, client and service. */
+  char rend_cookie[REND_COOKIE_LEN];
+
+  /** Number of streams associated with this rendezvous circuit. */
+  int nr_streams;
+} rend_data_t;
+
+typedef struct rend_data_v2_t {
+  /* Rendezvous base data. */
+  rend_data_t base_;
+
   /** Onion address (without the .onion part) that a client requests. */
   char onion_address[REND_SERVICE_ID_LEN_BASE32+1];
 
@@ -788,17 +835,16 @@ typedef struct rend_data_t {
 
   /** Hash of the hidden service's PK used by a service. */
   char rend_pk_digest[DIGEST_LEN];
+} rend_data_v2_t;
 
-  /** Rendezvous cookie used by both, client and service. */
-  char rend_cookie[REND_COOKIE_LEN];
-
-  /** List of HSDir fingerprints on which this request has been sent to.
-   * This contains binary identity digest of the directory. */
-  smartlist_t *hsdirs_fp;
-
-  /** Number of streams associated with this rendezvous circuit. */
-  int nr_streams;
-} rend_data_t;
+/* From a base rend_data_t object <b>d</d>, return the v2 object. */
+static inline
+rend_data_v2_t *TO_REND_DATA_V2(const rend_data_t *d)
+{
+  tor_assert(d);
+  tor_assert(d->version == 2);
+  return DOWNCAST(rend_data_v2_t, d);
+}
 
 /** Time interval for tracking replays of DH public keys received in
  * INTRODUCE2 cells.  Used only to avoid launching multiple
@@ -850,6 +896,7 @@ typedef enum {
 #define CELL_RELAY_EARLY 9
 #define CELL_CREATE2 10
 #define CELL_CREATED2 11
+#define CELL_PADDING_NEGOTIATE 12
 
 #define CELL_VPADDING 128
 #define CELL_CERTS 129
@@ -1351,13 +1398,30 @@ typedef struct listener_connection_t {
 #define OR_CERT_TYPE_RSA_ED_CROSSCERT 7
 /**@}*/
 
-/** The one currently supported type of AUTHENTICATE cell.  It contains
+/** The first supported type of AUTHENTICATE cell.  It contains
  * a bunch of structures signed with an RSA1024 key.  The signed
  * structures include a HMAC using negotiated TLS secrets, and a digest
  * of all cells sent or received before the AUTHENTICATE cell (including
  * the random server-generated AUTH_CHALLENGE cell).
  */
 #define AUTHTYPE_RSA_SHA256_TLSSECRET 1
+/** As AUTHTYPE_RSA_SHA256_TLSSECRET, but instead of using the
+ * negotiated TLS secrets, uses exported keying material from the TLS
+ * session as described in RFC 5705.
+ *
+ * Not used by today's tors, since everything that supports this
+ * also supports ED25519_SHA3_5705, which is better.
+ **/
+#define AUTHTYPE_RSA_SHA256_RFC5705 2
+/** As AUTHTYPE_RSA_SHA256_RFC5705, but uses an Ed25519 identity key to
+ * authenticate.  */
+#define AUTHTYPE_ED25519_SHA256_RFC5705 3
+/*
+ * NOTE: authchallenge_type_is_better() relies on these AUTHTYPE codes
+ * being sorted in order of preference.  If we someday add one with
+ * a higher numerical value that we don't like as much, we should revise
+ * authchallenge_type_is_better().
+ */
 
 /** The length of the part of the AUTHENTICATE cell body that the client and
  * server can generate independently (when using RSA_SHA256_TLSSECRET). It
@@ -1368,6 +1432,34 @@ typedef struct listener_connection_t {
  * signs. */
 #define V3_AUTH_BODY_LEN (V3_AUTH_FIXED_PART_LEN + 8 + 16)
 
+/** Structure to hold all the certificates we've received on an OR connection
+ */
+typedef struct or_handshake_certs_t {
+  /** True iff we originated this connection. */
+  int started_here;
+  /** The cert for the 'auth' RSA key that's supposed to sign the AUTHENTICATE
+   * cell. Signed with the RSA identity key. */
+  tor_x509_cert_t *auth_cert;
+  /** The cert for the 'link' RSA key that was used to negotiate the TLS
+   *  connection.  Signed with the RSA identity key. */
+  tor_x509_cert_t *link_cert;
+  /** A self-signed identity certificate: the RSA identity key signed
+   * with itself.  */
+  tor_x509_cert_t *id_cert;
+  /** The Ed25519 signing key, signed with the Ed25519 identity key. */
+  struct tor_cert_st *ed_id_sign;
+  /** A digest of the X509 link certificate for the TLS connection, signed
+   * with the Ed25519 siging key. */
+  struct tor_cert_st *ed_sign_link;
+  /** The Ed25519 authentication key (that's supposed to sign an AUTHENTICATE
+   * cell) , signed with the Ed25519 siging key. */
+  struct tor_cert_st *ed_sign_auth;
+  /** The Ed25519 identity key, crosssigned with the RSA identity key. */
+  uint8_t *ed_rsa_crosscert;
+  /** The length of <b>ed_rsa_crosscert</b> in bytes */
+  size_t ed_rsa_crosscert_len;
+} or_handshake_certs_t;
+
 /** Stores flags and information related to the portion of a v2/v3 Tor OR
  * connection handshake that happens after the TLS handshake is finished.
  */
@@ -1388,10 +1480,18 @@ typedef struct or_handshake_state_t {
 
   /* True iff we've received valid authentication to some identity. */
   unsigned int authenticated : 1;
+  unsigned int authenticated_rsa : 1;
+  unsigned int authenticated_ed25519 : 1;
 
   /* True iff we have sent a netinfo cell */
   unsigned int sent_netinfo : 1;
 
+  /** The signing->ed25519 link certificate corresponding to the x509
+   * certificate we used on the TLS connection (if this is a server-side
+   * connection). We make a copy of this here to prevent a race condition
+   * caused by TLS context rotation. */
+  struct tor_cert_st *own_link_cert;
+
   /** True iff we should feed outgoing cells into digest_sent and
    * digest_received respectively.
    *
@@ -1405,9 +1505,12 @@ typedef struct or_handshake_state_t {
   unsigned int digest_received_data : 1;
   /**@}*/
 
-  /** Identity digest that we have received and authenticated for our peer
+  /** Identity RSA digest that we have received and authenticated for our peer
    * on this connection. */
-  uint8_t authenticated_peer_id[DIGEST_LEN];
+  uint8_t authenticated_rsa_peer_id[DIGEST_LEN];
+  /** Identity Ed25519 public key that we have received and authenticated for
+   * our peer on this connection. */
+  ed25519_public_key_t authenticated_ed25519_peer_id;
 
   /** Digests of the cells that we have sent or received as part of a V3
    * handshake.  Used for making and checking AUTHENTICATE cells.
@@ -1420,14 +1523,8 @@ typedef struct or_handshake_state_t {
 
   /** Certificates that a connection initiator sent us in a CERTS cell; we're
    * holding on to them until we get an AUTHENTICATE cell.
-   *
-   * @{
    */
-  /** The cert for the key that's supposed to sign the AUTHENTICATE cell */
-  tor_x509_cert_t *auth_cert;
-  /** A self-signed identity certificate */
-  tor_x509_cert_t *id_cert;
-  /**@}*/
+  or_handshake_certs_t *certs;
 } or_handshake_state_t;
 
 /** Length of Extended ORPort connection identifier. */
@@ -1489,10 +1586,6 @@ typedef struct or_connection_t {
    * NETINFO cell listed the address we're connected to as recognized. */
   unsigned int is_canonical:1;
 
-  /** True iff we have decided that the other end of this connection
-   * is a client.  Connections with this flag set should never be used
-   * to satisfy an EXTEND request.  */
-  unsigned int is_connection_with_client:1;
   /** True iff this is an outgoing connection. */
   unsigned int is_outgoing:1;
   unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
@@ -1524,8 +1617,6 @@ typedef struct or_connection_t {
                     * bandwidthburst. (OPEN ORs only) */
   int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */
 
-  struct or_connection_t *next_with_same_id; /**< Next connection with same
-                                              * identity digest as this one. */
   /** Last emptied read token bucket in msec since midnight; only used if
    * TB_EMPTY events are enabled. */
   uint32_t read_emptied_time;
@@ -1603,6 +1694,8 @@ typedef struct entry_connection_t {
   edge_connection_t edge_;
 
   /** Nickname of planned exit node -- used with .exit support. */
+  /* XXX prop220: we need to make chosen_exit_name able to encode Ed IDs too.
+   * That's logically part of the UI parts for prop220 though. */
   char *chosen_exit_name;
 
   socks_request_t *socks_request; /**< SOCKS structure describing request (AP
@@ -1682,14 +1775,6 @@ typedef struct entry_connection_t {
   unsigned int is_socks_socket:1;
 } entry_connection_t;
 
-typedef enum {
-    DIR_SPOOL_NONE=0, DIR_SPOOL_SERVER_BY_DIGEST, DIR_SPOOL_SERVER_BY_FP,
-    DIR_SPOOL_EXTRA_BY_DIGEST, DIR_SPOOL_EXTRA_BY_FP,
-    DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS,
-    DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */
-} dir_spool_source_t;
-#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t)
-
 /** Subtype of connection_t for an "directory connection" -- that is, an HTTP
  * connection to retrieve or serve directory material. */
 typedef struct dir_connection_t {
@@ -1698,33 +1783,29 @@ typedef struct dir_connection_t {
  /** Which 'resource' did we ask the directory for? This is typically the part
   * of the URL string that defines, relative to the directory conn purpose,
   * what thing we want.  For example, in router descriptor downloads by
-  * descriptor digest, it contains "d/", then one ore more +-separated
+  * descriptor digest, it contains "d/", then one or more +-separated
   * fingerprints.
   **/
   char *requested_resource;
   unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
 
-  /* Used only for server sides of some dir connections, to implement
-   * "spooling" of directory material to the outbuf.  Otherwise, we'd have
-   * to append everything to the outbuf in one enormous chunk. */
-  /** What exactly are we spooling right now? */
-  dir_spool_source_bitfield_t  dir_spool_src : 3;
-
   /** If we're fetching descriptors, what router purpose shall we assign
    * to them? */
   uint8_t router_purpose;
-  /** List of fingerprints for networkstatuses or descriptors to be spooled. */
-  smartlist_t *fingerprint_stack;
-  /** A cached_dir_t object that we're currently spooling out */
-  struct cached_dir_t *cached_dir;
-  /** The current offset into cached_dir. */
-  off_t cached_dir_offset;
-  /** The zlib object doing on-the-fly compression for spooled data. */
-  tor_zlib_state_t *zlib_state;
+
+  /** List of spooled_resource_t for objects that we're spooling. We use
+   * it from back to front. */
+  smartlist_t *spool;
+  /** The compression object doing on-the-fly compression for spooled data. */
+  tor_compress_state_t *compress_state;
 
   /** What rendezvous service are we querying for? */
   rend_data_t *rend_data;
 
+  /** If this is a one-hop connection, tracks the state of the directory guard
+   * for this connection (if any). */
+  struct circuit_guard_state_t *guard_state;
+
   char identity_digest[DIGEST_LEN]; /**< Hash of the public RSA key for
                                      * the directory server's signing key. */
 
@@ -1732,6 +1813,14 @@ typedef struct dir_connection_t {
    * that's going away and being used on channels instead.  The dirserver still
    * needs this for the incoming side, so it's moved here. */
   uint64_t dirreq_id;
+
+#ifdef MEASUREMENTS_21206
+  /** Number of RELAY_DATA cells received. */
+  uint32_t data_cells_received;
+
+  /** Number of RELAY_DATA cells sent. */
+  uint32_t data_cells_sent;
+#endif
 } dir_connection_t;
 
 /** Subtype of connection_t for an connection to a controller. */
@@ -1768,8 +1857,6 @@ typedef struct control_connection_t {
 
 /** Cast a connection_t subtype pointer to a connection_t **/
 #define TO_CONN(c) (&(((c)->base_)))
-/** Helper macro: Given a pointer to to.base_, of type from*, return &to. */
-#define DOWNCAST(to, ptr) ((to*)SUBTYPE_P(ptr, to, base_))
 
 /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
 #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
@@ -1872,11 +1959,13 @@ typedef struct addr_policy_t {
  * compressed form. */
 typedef struct cached_dir_t {
   char *dir; /**< Contents of this object, NUL-terminated. */
-  char *dir_z; /**< Compressed contents of this object. */
+  char *dir_compressed; /**< Compressed contents of this object. */
   size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */
-  size_t dir_z_len; /**< Length of <b>dir_z</b>. */
+  size_t dir_compressed_len; /**< Length of <b>dir_compressed</b>. */
   time_t published; /**< When was this object published. */
   common_digests_t digests; /**< Digests of this object (networkstatus only) */
+  /** Sha3 digest (also ns only) */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
   int refcnt; /**< Reference count for this cached_dir_t. */
 } cached_dir_t;
 
@@ -2210,6 +2299,20 @@ typedef struct routerstatus_t {
    * accept EXTEND2 cells */
   unsigned int supports_extend2_cells:1;
 
+  /** True iff this router has a protocol list that allows it to negotiate
+   * ed25519 identity keys on a link handshake. */
+  unsigned int supports_ed25519_link_handshake:1;
+
+  /** True iff this router has a protocol list that allows it to be an
+   * introduction point supporting ed25519 authentication key which is part of
+   * the v3 protocol detailed in proposal 224. This requires HSIntro=4. */
+  unsigned int supports_ed25519_hs_intro : 1;
+
+  /** True iff this router has a protocol list that allows it to be an hidden
+   * service directory supporting version 3 as seen in proposal 224. This
+   * requires HSDir=2. */
+  unsigned int supports_v3_hsdir : 1;
+
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
   unsigned int bw_is_unmeasured:1; /**< This is a consensus entry, with
@@ -2372,9 +2475,6 @@ typedef struct node_t {
   /** Local info: we treat this node as if it rejects everything */
   unsigned int rejects_all:1;
 
-  /** Local info: this node is in our list of guards */
-  unsigned int using_as_guard:1;
-
   /* Local info: derived. */
 
   /** True if the IPv6 OR port is preferred over the IPv4 OR port.
@@ -2566,6 +2666,9 @@ typedef struct networkstatus_t {
 
   /** Digests of this document, as signed. */
   common_digests_t digests;
+  /** A SHA3-256 digest of the document, not including signatures: used for
+   * consensus diffs */
+  uint8_t digest_sha3_as_signed[DIGEST256_LEN];
 
   /** List of router statuses, sorted by identity digest.  For a vote,
    * the elements are vote_routerstatus_t; for a consensus, the elements
@@ -2654,7 +2757,10 @@ typedef struct {
 typedef struct extend_info_t {
   char nickname[MAX_HEX_NICKNAME_LEN+1]; /**< This router's nickname for
                                           * display. */
-  char identity_digest[DIGEST_LEN]; /**< Hash of this router's identity key. */
+  /** Hash of this router's RSA identity key. */
+  char identity_digest[DIGEST_LEN];
+  /** Ed25519 identity for this router, if any. */
+  ed25519_public_key_t ed_identity;
   uint16_t port; /**< OR port. */
   tor_addr_t addr; /**< IP address. */
   crypto_pk_t *onion_key; /**< Current onionskin key. */
@@ -2996,6 +3102,13 @@ typedef struct circuit_t {
    * circuit's queues; used only if CELL_STATS events are enabled and
    * cleared after being sent to control port. */
   smartlist_t *testing_cell_stats;
+
+  /** If set, points to an HS token that this circuit might be carrying.
+   *  Used by the HS circuitmap.  */
+  hs_token_t *hs_token;
+  /** Hashtable node: used to look up the circuit by its HS token using the HS
+      circuitmap. */
+  HT_ENTRY(circuit_t) hs_circuitmap_node;
 } circuit_t;
 
 /** Largest number of relay_early cells that we can send on a given
@@ -3092,6 +3205,15 @@ typedef struct origin_circuit_t {
   /** Holds all rendezvous data on either client or service side. */
   rend_data_t *rend_data;
 
+  /** Holds the data that the entry guard system uses to track the
+   * status of the guard this circuit is using, and thereby to determine
+   * whether this circuit can be used. */
+  struct circuit_guard_state_t *guard_state;
+
+  /** Index into global_origin_circuit_list for this circuit. -1 if not
+   * present. */
+  int global_origin_circuit_list_idx;
+
   /** How many more relay_early cells can we send on this circuit, according
    * to the specification? */
   unsigned int remaining_relay_early_cells : 4;
@@ -3235,6 +3357,13 @@ typedef struct origin_circuit_t {
    * adjust_exit_policy_from_exitpolicy_failure.
    */
   smartlist_t *prepend_policy;
+
+  /** How long do we wait before closing this circuit if it remains
+   * completely idle after it was built, in seconds? This value
+   * is randomized on a per-circuit basis from CircuitsAvailableTimoeut
+   * to 2*CircuitsAvailableTimoeut. */
+  int circuit_idle_timeout;
+
 } origin_circuit_t;
 
 struct onion_queue_t;
@@ -3296,8 +3425,6 @@ typedef struct or_circuit_t {
    * is not marked for close. */
   struct or_circuit_t *rend_splice;
 
-  struct or_circuit_rendinfo_s *rendinfo;
-
   /** Stores KH for the handshake. */
   char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */
 
@@ -3331,25 +3458,11 @@ typedef struct or_circuit_t {
   uint32_t max_middle_cells;
 } or_circuit_t;
 
-typedef struct or_circuit_rendinfo_s {
-
 #if REND_COOKIE_LEN != DIGEST_LEN
 #error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN"
 #endif
 #define REND_TOKEN_LEN DIGEST_LEN
 
-  /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a
-   * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes
-   * otherwise.
-   */
-  char rend_token[REND_TOKEN_LEN];
-
-  /** True if this is a rendezvous point circuit; false if this is an
-   * introduction point. */
-  unsigned is_rend_circ;
-
-} or_circuit_rendinfo_t;
-
 /** Convert a circuit subtype to a circuit_t. */
 #define TO_CIRCUIT(x)  (&((x)->base_))
 
@@ -3392,15 +3505,6 @@ static inline const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(
   return DOWNCAST(origin_circuit_t, x);
 }
 
-/** Bitfield type: things that we're willing to use invalid routers for. */
-typedef enum invalid_router_usage_t {
-  ALLOW_INVALID_ENTRY       =1,
-  ALLOW_INVALID_EXIT        =2,
-  ALLOW_INVALID_MIDDLE      =4,
-  ALLOW_INVALID_RENDEZVOUS  =8,
-  ALLOW_INVALID_INTRODUCTION=16,
-} invalid_router_usage_t;
-
 /* limits for TCP send and recv buffer size used for constrained sockets */
 #define MIN_CONSTRAINED_TCP_BUFFER 2048
 #define MAX_CONSTRAINED_TCP_BUFFER 262144  /* 256k */
@@ -3462,33 +3566,18 @@ typedef struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 } port_cfg_t;
 
-/** Ordinary configuration line. */
-#define CONFIG_LINE_NORMAL 0
-/** Appends to previous configuration for the same option, even if we
- * would ordinary replace it. */
-#define CONFIG_LINE_APPEND 1
-/* Removes all previous configuration for an option. */
-#define CONFIG_LINE_CLEAR 2
-
-/** A linked list of lines in a config file. */
-typedef struct config_line_t {
-  char *key;
-  char *value;
-  struct config_line_t *next;
-  /** What special treatment (if any) does this line require? */
-  unsigned int command:2;
-  /** If true, subsequent assignments to this linelist should replace
-   * it, not extend it.  Set only on the first item in a linelist in an
-   * or_options_t. */
-  unsigned int fragile:1;
-} config_line_t;
-
 typedef struct routerset_t routerset_t;
 
 /** A magic value for the (Socks|OR|...)Port options below, telling Tor
  * to pick its own port. */
 #define CFG_AUTO_PORT 0xc4005e
 
+/** Enumeration of outbound address configuration types:
+ * Exit-only, OR-only, or both */
+typedef enum {OUTBOUND_ADDR_EXIT, OUTBOUND_ADDR_OR,
+              OUTBOUND_ADDR_EXIT_AND_OR,
+              OUTBOUND_ADDR_MAX} outbound_addr_t;
+
 /** Configuration options for a Tor process. */
 typedef struct {
   uint32_t magic_;
@@ -3541,10 +3630,6 @@ typedef struct {
   int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our
                        * process for all current and future memory. */
 
-  /** List of "entry", "middle", "exit", "introduction", "rendezvous". */
-  smartlist_t *AllowInvalidNodes;
-  /** Bitmask; derived from AllowInvalidNodes. */
-  invalid_router_usage_t AllowInvalid_;
   config_line_t *ExitPolicy; /**< Lists of exit policy components. */
   int ExitPolicyRejectPrivate; /**< Should we not exit to reserved private
                                 * addresses, and our own published addresses?
@@ -3555,27 +3640,16 @@ typedef struct {
                                         * configured ports. */
   config_line_t *SocksPolicy; /**< Lists of socks policy components */
   config_line_t *DirPolicy; /**< Lists of dir policy components */
-  /** Addresses to bind for listening for SOCKS connections. */
-  config_line_t *SocksListenAddress;
-  /** Addresses to bind for listening for transparent pf/netfilter
-   * connections. */
-  config_line_t *TransListenAddress;
-  /** Addresses to bind for listening for transparent natd connections */
-  config_line_t *NATDListenAddress;
-  /** Addresses to bind for listening for SOCKS connections. */
-  config_line_t *DNSListenAddress;
-  /** Addresses to bind for listening for OR connections. */
-  config_line_t *ORListenAddress;
-  /** Addresses to bind for listening for directory connections. */
-  config_line_t *DirListenAddress;
-  /** Addresses to bind for listening for control connections. */
-  config_line_t *ControlListenAddress;
   /** Local address to bind outbound sockets */
   config_line_t *OutboundBindAddress;
-  /** IPv4 address derived from OutboundBindAddress. */
-  tor_addr_t OutboundBindAddressIPv4_;
-  /** IPv6 address derived from OutboundBindAddress. */
-  tor_addr_t OutboundBindAddressIPv6_;
+  /** Local address to bind outbound relay sockets */
+  config_line_t *OutboundBindAddressOR;
+  /** Local address to bind outbound exit sockets */
+  config_line_t *OutboundBindAddressExit;
+  /** Addresses derived from the various OutboundBindAddress lines.
+   * [][0] is IPv4, [][1] is IPv6
+   */
+  tor_addr_t OutboundBindAddresses[OUTBOUND_ADDR_MAX][2];
   /** Directory server only: which versions of
    * Tor should we tell users to run? */
   config_line_t *RecommendedVersions;
@@ -3587,7 +3661,6 @@ typedef struct {
   /** Whether routers accept EXTEND cells to routers with private IPs. */
   int ExtendAllowPrivateAddresses;
   char *User; /**< Name of user to run Tor as. */
-  char *Group; /**< Name of group to run Tor as. */
   config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */
   /** Ports to listen on for extended OR connections. */
   config_line_t *ExtORPort_lines;
@@ -3690,6 +3763,15 @@ typedef struct {
   int AvoidDiskWrites; /**< Boolean: should we never cache things to disk?
                         * Not used yet. */
   int ClientOnly; /**< Boolean: should we never evolve into a server role? */
+
+  int ReducedConnectionPadding; /**< Boolean: Should we try to keep connections
+                                  open shorter and pad them less against
+                                  connection-level traffic analysis? */
+  /** Autobool: if auto, then connection padding will be negotiated by client
+   * and server. If 0, it will be fully disabled. If 1, the client will still
+   * pad to the server regardless of server support. */
+  int ConnectionPadding;
+
   /** To what authority types do we publish our descriptor? Choices are
    * "v1", "v2", "v3", "bridge", or "". */
   smartlist_t *PublishServerDescriptor;
@@ -3715,15 +3797,6 @@ typedef struct {
   /** A routerset that should be used when picking RPs for HS circuits. */
   routerset_t *Tor2webRendezvousPoints;
 
-  /** Close hidden service client circuits immediately when they reach
-   * the normal circuit-build timeout, even if they have already sent
-   * an INTRODUCE1 cell on its way to the service. */
-  int CloseHSClientCircuitsImmediatelyOnTimeout;
-
-  /** Close hidden-service-side rendezvous circuits immediately when
-   * they reach the normal circuit-build timeout. */
-  int CloseHSServiceRendCircuitsImmediatelyOnTimeout;
-
   /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct)
    * circuits between the onion service server, and the introduction and
    * rendezvous points. (Onion service descriptors are still posted using
@@ -3804,8 +3877,8 @@ typedef struct {
   int CircuitBuildTimeout; /**< Cull non-open circuits that were born at
                             * least this many seconds ago. Used until
                             * adaptive algorithm learns a new value. */
-  int CircuitIdleTimeout; /**< Cull open clean circuits that were born
-                           * at least this many seconds ago. */
+  int CircuitsAvailableTimeout; /**< Try to have an open circuit for at
+                                     least this long after last activity */
   int CircuitStreamTimeout; /**< If non-zero, detach streams from circuits
                              * and try a new circuit if the stream has been
                              * waiting for this many seconds. If zero, use
@@ -3815,16 +3888,12 @@ typedef struct {
                          * a new one? */
   int MaxCircuitDirtiness; /**< Never use circs that were first used more than
                                 this interval ago. */
-  int PredictedPortsRelevanceTime; /** How long after we've requested a
-                                    * connection for a given port, do we want
-                                    * to continue to pick exits that support
-                                    * that port?  */
   uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing
                            * to use in a second? */
   uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing
                             * to use in a second? */
   uint64_t MaxAdvertisedBandwidth; /**< How much bandwidth are we willing to
-                                    * tell people we have? */
+                                    * tell other nodes we have? */
   uint64_t RelayBandwidthRate; /**< How much bandwidth, on average, are we
                                  * willing to use for all relayed conns? */
   uint64_t RelayBandwidthBurst; /**< How much bandwidth, at maximum, will we
@@ -3832,8 +3901,6 @@ typedef struct {
   uint64_t PerConnBWRate; /**< Long-term bw on a single TLS conn, if set. */
   uint64_t PerConnBWBurst; /**< Allowed burst on a single TLS conn, if set. */
   int NumCPUs; /**< How many CPUs should we try to use? */
-//int RunTesting; /**< If true, create testing circuits to measure how well the
-//                 * other ORs are running. */
   config_line_t *RendConfigLines; /**< List of configuration lines
                                           * for rendezvous services. */
   config_line_t *HidServAuth; /**< List of configuration lines for client-side
@@ -3884,7 +3951,8 @@ typedef struct {
   /** If set, use these bridge authorities and not the default one. */
   config_line_t *AlternateBridgeAuthority;
 
-  char *MyFamily; /**< Declared family for this OR. */
+  config_line_t *MyFamily_lines; /**< Declared family for this OR. */
+  config_line_t *MyFamily; /**< Declared family for this OR, normalized */
   config_line_t *NodeFamilies; /**< List of config lines for
                                 * node families */
   smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */
@@ -3910,9 +3978,6 @@ typedef struct {
                             * and vote for all other exits as good. */
   int AuthDirMaxServersPerAddr; /**< Do not permit more than this
                                  * number of servers per IP address. */
-  int AuthDirMaxServersPerAuthAddr; /**< Do not permit more than this
-                                     * number of servers per IP address shared
-                                     * with an authority. */
   int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6?  */
   int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */
 
@@ -3997,8 +4062,6 @@ typedef struct {
   int UseEntryGuards;
 
   int NumEntryGuards; /**< How many entry guards do we try to establish? */
-  int UseEntryGuardsAsDirGuards; /** Boolean: Do we try to get directory info
-                                  * from a smallish number of fixed nodes? */
 
   /** If 1, we use any guardfraction information we see in the
    * consensus.  If 0, we don't.  If -1, let the consensus parameter
@@ -4008,8 +4071,6 @@ typedef struct {
   int NumDirectoryGuards; /**< How many dir guards do we try to establish?
                            * If 0, use value from NumEntryGuards. */
   int RephistTrackTime; /**< How many seconds do we keep rephist info? */
-  int FastFirstHopPK; /**< If Tor believes it is safe, should we save a third
-                       * of our PK time by sending CREATE_FAST cells? */
   /** Should we always fetch our dir info on the mirror schedule (which
    * means directly from the authorities) no matter our other config? */
   int FetchDirInfoEarly;
@@ -4065,16 +4126,6 @@ typedef struct {
    * if we are a cache).  For authorities, this is always true. */
   int DownloadExtraInfo;
 
-  /** If true, and we are acting as a relay, allow exit circuits even when
-   * we are the first hop of a circuit. */
-  int AllowSingleHopExits;
-  /** If true, don't allow relays with AllowSingleHopExits=1 to be used in
-   * circuits that we build. */
-  int ExcludeSingleHopRelays;
-  /** If true, and the controller tells us to use a one-hop circuit, and the
-   * exit allows it, we use it. */
-  int AllowSingleHopCircuits;
-
   /** If true, we convert "www.google.com.foo.exit" addresses on the
    * socks/trans/natd ports into "www.google.com" addresses that
    * exit from the node "foo". Disabled by default since attacking
@@ -4082,10 +4133,6 @@ typedef struct {
    * selection. */
   int AllowDotExit;
 
-  /** If true, we will warn if a user gives us only an IP address
-   * instead of a hostname. */
-  int WarnUnsafeSocks;
-
   /** If true, we're configured to collect statistics on clients
    * requesting network statuses from us as directory. */
   int DirReqStatistics_option;
@@ -4102,11 +4149,18 @@ typedef struct {
   /** If true, the user wants us to collect cell statistics. */
   int CellStatistics;
 
+  /** If true, the user wants us to collect padding statistics. */
+  int PaddingStatistics;
+
   /** If true, the user wants us to collect statistics as entry node. */
   int EntryStatistics;
 
   /** If true, the user wants us to collect statistics as hidden service
    * directory, introduction point, or rendezvous point. */
+  int HiddenServiceStatistics_option;
+  /** Internal variable to remember whether we're actually acting on
+   * HiddenServiceStatistics_option -- yes if it's set and we're a server,
+   * else no. */
   int HiddenServiceStatistics;
 
   /** If true, include statistics file contents in extra-info documents. */
@@ -4305,8 +4359,7 @@ typedef struct {
   int TestingDirAuthVoteGuardIsStrict;
 
   /** Relays in a testing network which should be voted HSDir
-   * regardless of uptime and DirPort.
-   * Respects VoteOnHidServDirectoriesV2. */
+   * regardless of uptime and DirPort. */
   routerset_t *TestingDirAuthVoteHSDir;
   int TestingDirAuthVoteHSDirIsStrict;
 
@@ -4438,8 +4491,6 @@ typedef struct {
 
   int IPv6Exit; /**< Do we support exiting to IPv6 addresses? */
 
-  char *TLSECGroup; /**< One of "P256", "P224", or nil for auto */
-
   /** Fraction: */
   double PathsNeededToBuildCircuits;
 
@@ -4470,7 +4521,7 @@ typedef struct {
    * XXXX Eventually, the default will be 0. */
   int ExitRelay;
 
-  /** For how long (seconds) do we declare our singning keys to be valid? */
+  /** For how long (seconds) do we declare our signing keys to be valid? */
   int SigningKeyLifetime;
   /** For how long (seconds) do we declare our link keys to be valid? */
   int TestingLinkCertLifetime;
@@ -4515,6 +4566,23 @@ typedef struct {
   /** If 1, we skip all OOS checks. */
   int DisableOOSCheck;
 
+  /** Autobool: Should we include Ed25519 identities in extend2 cells?
+   * If -1, we should do whatever the consensus parameter says. */
+  int ExtendByEd25519ID;
+
+  /** Bool (default: 1): When testing routerinfos as a directory authority,
+   * do we enforce Ed25519 identity match? */
+  /* NOTE: remove this option someday. */
+  int AuthDirTestEd25519LinkKeys;
+
+  /** Bool (default: 0): Tells if a %include was used on torrc */
+  int IncludeUsed;
+
+  /** The seconds after expiration which we as a relay should keep old
+   * consensuses around so that we can generate diffs from them.  If 0,
+   * use the default. */
+  int MaxConsensusAgeForDiffs;
+
   /** Autobool: Is the circuit creation DoS mitigation subsystem enabled? */
   int DoSCircuitCreationEnabled;
   /** Minimum concurrent connection needed from one single address before any
@@ -4566,9 +4634,12 @@ typedef struct {
   uint64_t AccountingBytesAtSoftLimit;
   uint64_t AccountingExpectedUsage;
 
-  /** A list of Entry Guard-related configuration lines. */
+  /** A list of Entry Guard-related configuration lines. (pre-prop271) */
   config_line_t *EntryGuards;
 
+  /** A list of guard-related configuration lines. (post-prop271) */
+  config_line_t *Guard;
+
   config_line_t *TransportProxies;
 
   /** These fields hold information on the history of bandwidth usage for
@@ -4765,7 +4836,7 @@ typedef uint32_t build_time_t;
 double circuit_build_times_quantile_cutoff(void);
 
 /** How often in seconds should we build a test circuit */
-#define CBT_DEFAULT_TEST_FREQUENCY 60
+#define CBT_DEFAULT_TEST_FREQUENCY 10
 #define CBT_MIN_TEST_FREQUENCY 1
 #define CBT_MAX_TEST_FREQUENCY INT32_MAX
 
@@ -5254,7 +5325,8 @@ typedef struct dir_server_t {
                            * address information from published? */
 
   routerstatus_t fake_status; /**< Used when we need to pass this trusted
-                               * dir_server_t to directory_initiate_command_*
+                               * dir_server_t to
+                               * directory_request_set_routerstatus.
                                * as a routerstatus_t.  Not updated by the
                                * router-status management code!
                                **/
@@ -5294,10 +5366,6 @@ typedef struct dir_server_t {
  */
 #define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4)
 
-/** This node is to be chosen as a directory guard, so don't choose any
- * node that's currently a guard. */
-#define PDS_FOR_GUARD (1<<5)
-
 /** Possible ways to weight routers when choosing one randomly.  See
  * routerlist_sl_choose_by_bandwidth() for more information.*/
 typedef enum bandwidth_weight_rule_t {
@@ -5311,7 +5379,6 @@ typedef enum {
   CRN_NEED_UPTIME = 1<<0,
   CRN_NEED_CAPACITY = 1<<1,
   CRN_NEED_GUARD = 1<<2,
-  CRN_ALLOW_INVALID = 1<<3,
   /* XXXX not used, apparently. */
   CRN_WEIGHT_AS_EXIT = 1<<5,
   CRN_NEED_DESC = 1<<6,
@@ -5326,8 +5393,6 @@ typedef enum {
 typedef enum was_router_added_t {
   /* Router was added successfully. */
   ROUTER_ADDED_SUCCESSFULLY = 1,
-  /* Router descriptor was added with warnings to submitter. */
-  ROUTER_ADDED_NOTIFY_GENERATOR = 0,
   /* Extrainfo document was rejected because no corresponding router
    * descriptor was found OR router descriptor was rejected because
    * it was incompatible with its extrainfo document. */
diff --git a/src/or/parsecommon.c b/src/or/parsecommon.c
new file mode 100644
index 0000000000..7959867875
--- /dev/null
+++ b/src/or/parsecommon.c
@@ -0,0 +1,450 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.c
+ * \brief Common code to parse and validate various type of descriptors.
+ **/
+
+#include "parsecommon.h"
+#include "torlog.h"
+#include "util_format.h"
+
+#define MIN_ANNOTATION A_PURPOSE
+#define MAX_ANNOTATION A_UNKNOWN_
+
+#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
+#define ALLOC(sz) memarea_alloc(area,sz)
+#define STRDUP(str) memarea_strdup(area,str)
+#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
+
+#define RET_ERR(msg)                                               \
+  STMT_BEGIN                                                       \
+    if (tok) token_clear(tok);                                      \
+    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
+    tok->tp = ERR_;                                                \
+    tok->error = STRDUP(msg);                                      \
+    goto done_tokenizing;                                          \
+  STMT_END
+
+/** Free all resources allocated for <b>tok</b> */
+void
+token_clear(directory_token_t *tok)
+{
+  if (tok->key)
+    crypto_pk_free(tok->key);
+}
+
+/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
+ * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
+ * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
+ * entire string.
+ */
+int
+tokenize_string(memarea_t *area,
+                const char *start, const char *end, smartlist_t *out,
+                token_rule_t *table, int flags)
+{
+  const char **s;
+  directory_token_t *tok = NULL;
+  int counts[NIL_];
+  int i;
+  int first_nonannotation;
+  int prev_len = smartlist_len(out);
+  tor_assert(area);
+
+  s = &start;
+  if (!end) {
+    end = start+strlen(start);
+  } else {
+    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
+    if (memchr(start, '\0', end-start)) {
+      log_warn(LD_DIR, "parse error: internal NUL character.");
+      return -1;
+    }
+  }
+  for (i = 0; i < NIL_; ++i)
+    counts[i] = 0;
+
+  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
+
+  while (*s < end && (!tok || tok->tp != EOF_)) {
+    tok = get_next_token(area, s, end, table);
+    if (tok->tp == ERR_) {
+      log_warn(LD_DIR, "parse error: %s", tok->error);
+      token_clear(tok);
+      return -1;
+    }
+    ++counts[tok->tp];
+    smartlist_add(out, tok);
+    *s = eat_whitespace_eos(*s, end);
+  }
+
+  if (flags & TS_NOCHECK)
+    return 0;
+
+  if ((flags & TS_ANNOTATIONS_OK)) {
+    first_nonannotation = -1;
+    for (i = 0; i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
+        first_nonannotation = i;
+        break;
+      }
+    }
+    if (first_nonannotation < 0) {
+      log_warn(LD_DIR, "parse error: item contains only annotations");
+      return -1;
+    }
+    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
+        return -1;
+      }
+    }
+    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
+      if (first_nonannotation != prev_len) {
+        log_warn(LD_DIR, "parse error: Unexpected annotations.");
+        return -1;
+      }
+    }
+  } else {
+    for (i=0;  i < smartlist_len(out); ++i) {
+      tok = smartlist_get(out, i);
+      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
+        log_warn(LD_DIR, "parse error: no annotations allowed.");
+        return -1;
+      }
+    }
+    first_nonannotation = 0;
+  }
+  for (i = 0; table[i].t; ++i) {
+    if (counts[table[i].v] < table[i].min_cnt) {
+      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
+      return -1;
+    }
+    if (counts[table[i].v] > table[i].max_cnt) {
+      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
+      return -1;
+    }
+    if (table[i].pos & AT_START) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+    if (table[i].pos & AT_END) {
+      if (smartlist_len(out) < 1 ||
+          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
+        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/** Helper: parse space-separated arguments from the string <b>s</b> ending at
+ * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
+ * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
+ * all storage in <b>area</b>.  Return the number of arguments parsed, or
+ * return -1 if there was an insanely high number of arguments. */
+static inline int
+get_token_arguments(memarea_t *area, directory_token_t *tok,
+                    const char *s, const char *eol)
+{
+/** Largest number of arguments we'll accept to any token, ever. */
+#define MAX_ARGS 512
+  char *mem = memarea_strndup(area, s, eol-s);
+  char *cp = mem;
+  int j = 0;
+  char *args[MAX_ARGS];
+  while (*cp) {
+    if (j == MAX_ARGS)
+      return -1;
+    args[j++] = cp;
+    cp = (char*)find_whitespace(cp);
+    if (!cp || !*cp)
+      break; /* End of the line. */
+    *cp++ = '\0';
+    cp = (char*)eat_whitespace(cp);
+  }
+  tok->n_args = j;
+  tok->args = memarea_memdup(area, args, j*sizeof(char*));
+  return j;
+#undef MAX_ARGS
+}
+
+/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
+ * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
+ * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
+ * conform to the syntax we wanted.
+ **/
+static inline directory_token_t *
+token_check_object(memarea_t *area, const char *kwd,
+                   directory_token_t *tok, obj_syntax o_syn)
+{
+  char ebuf[128];
+  switch (o_syn) {
+    case NO_OBJ:
+      /* No object is allowed for this token. */
+      if (tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_OBJ:
+      /* There must be a (non-key) object. */
+      if (!tok->object_body) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
+    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
+      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
+                     kwd, crypto_pk_num_bits(tok->key));
+        RET_ERR(ebuf);
+      }
+      /* fall through */
+    case NEED_KEY: /* There must be some kind of key. */
+      if (!tok->key) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
+        RET_ERR(ebuf);
+      }
+      if (o_syn != NEED_SKEY_1024) {
+        if (crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Private key given for %s, which wants a public key", kwd);
+          RET_ERR(ebuf);
+        }
+      } else { /* o_syn == NEED_SKEY_1024 */
+        if (!crypto_pk_key_is_private(tok->key)) {
+          tor_snprintf(ebuf, sizeof(ebuf),
+               "Public key given for %s, which wants a private key", kwd);
+          RET_ERR(ebuf);
+        }
+      }
+      break;
+    case OBJ_OK:
+      /* Anything goes with this token. */
+      break;
+  }
+
+ done_tokenizing:
+  return tok;
+}
+
+/** Helper function: read the next token from *s, advance *s to the end of the
+ * token, and return the parsed token.  Parse *<b>s</b> according to the list
+ * of tokens in <b>table</b>.
+ */
+directory_token_t *
+get_next_token(memarea_t *area,
+               const char **s, const char *eos, token_rule_t *table)
+{
+  /** Reject any object at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
+  /** Reject any line at least this big; it is probably an overflow, an
+   * attack, a bug, or some other nonsense. */
+#define MAX_LINE_LENGTH (128*1024)
+
+  const char *next, *eol, *obstart;
+  size_t obname_len;
+  int i;
+  directory_token_t *tok;
+  obj_syntax o_syn = NO_OBJ;
+  char ebuf[128];
+  const char *kwd = "";
+
+  tor_assert(area);
+  tok = ALLOC_ZERO(sizeof(directory_token_t));
+  tok->tp = ERR_;
+
+  /* Set *s to first token, eol to end-of-line, next to after first token */
+  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol)
+    eol = eos;
+  if (eol - *s > MAX_LINE_LENGTH) {
+    RET_ERR("Line far too long");
+  }
+
+  next = find_whitespace_eos(*s, eol);
+
+  if (!strcmp_len(*s, "opt", next-*s)) {
+    /* Skip past an "opt" at the start of the line. */
+    *s = eat_whitespace_eos_no_nl(next, eol);
+    next = find_whitespace_eos(*s, eol);
+  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
+    RET_ERR("Unexpected EOF");
+  }
+
+  /* Search the table for the appropriate entry.  (I tried a binary search
+   * instead, but it wasn't any faster.) */
+  for (i = 0; table[i].t ; ++i) {
+    if (!strcmp_len(*s, table[i].t, next-*s)) {
+      /* We've found the keyword. */
+      kwd = table[i].t;
+      tok->tp = table[i].v;
+      o_syn = table[i].os;
+      *s = eat_whitespace_eos_no_nl(next, eol);
+      /* We go ahead whether there are arguments or not, so that tok->args is
+       * always set if we want arguments. */
+      if (table[i].concat_args) {
+        /* The keyword takes the line as a single argument */
+        tok->args = ALLOC(sizeof(char*));
+        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
+        tok->n_args = 1;
+      } else {
+        /* This keyword takes multiple arguments. */
+        if (get_token_arguments(area, tok, *s, eol)<0) {
+          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
+          RET_ERR(ebuf);
+        }
+        *s = eol;
+      }
+      if (tok->n_args < table[i].min_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
+        RET_ERR(ebuf);
+      } else if (tok->n_args > table[i].max_args) {
+        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
+        RET_ERR(ebuf);
+      }
+      break;
+    }
+  }
+
+  if (tok->tp == ERR_) {
+    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
+    if (*s < eol && **s == '@')
+      tok->tp = A_UNKNOWN_;
+    else
+      tok->tp = K_OPT;
+    tok->args = ALLOC(sizeof(char*));
+    tok->args[0] = STRNDUP(*s, eol-*s);
+    tok->n_args = 1;
+    o_syn = OBJ_OK;
+  }
+
+  /* Check whether there's an object present */
+  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
+  tor_assert(eos >= *s);
+  eol = memchr(*s, '\n', eos-*s);
+  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
+    goto check_object;
+
+  obstart = *s; /* Set obstart to start of object spec */
+  if (*s+16 >= eol || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
+      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
+    RET_ERR("Malformed object: bad begin line");
+  }
+  tok->object_type = STRNDUP(*s+11, eol-*s-16);
+  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
+  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
+
+  /* Go to the end of the object */
+  next = tor_memstr(*s, eos-*s, "-----END ");
+  if (!next) {
+    RET_ERR("Malformed object: missing object end line");
+  }
+  tor_assert(eos >= next);
+  eol = memchr(next, '\n', eos-next);
+  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
+    eol = eos;
+  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
+  if ((size_t)(eol-next) != 9+obname_len+5 ||
+      strcmp_len(next+9, tok->object_type, obname_len) ||
+      strcmp_len(eol-5, "-----", 5)) {
+    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
+             tok->object_type);
+    ebuf[sizeof(ebuf)-1] = '\0';
+    RET_ERR(ebuf);
+  }
+  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
+    RET_ERR("Couldn't parse object: missing footer or object much too big.");
+
+  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse public key.");
+  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
+    tok->key = crypto_pk_new();
+    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
+      RET_ERR("Couldn't parse private key.");
+  } else { /* If it's something else, try to base64-decode it */
+    int r;
+    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
+    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+  *s = eol;
+
+ check_object:
+  tok = token_check_object(area, kwd, tok, o_syn);
+
+ done_tokenizing:
+  return tok;
+
+#undef RET_ERR
+#undef ALLOC
+#undef ALLOC_ZERO
+#undef STRDUP
+#undef STRNDUP
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
+ * with an assert if no such keyword is found.
+ */
+directory_token_t *
+find_by_keyword_(smartlist_t *s, directory_keyword keyword,
+                 const char *keyword_as_string)
+{
+  directory_token_t *tok = find_opt_by_keyword(s, keyword);
+  if (PREDICT_UNLIKELY(!tok)) {
+    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
+         "been validated. Internal error.", keyword_as_string, (int)keyword);
+    tor_assert(tok);
+  }
+  return tok;
+}
+
+/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
+ * NULL if no such keyword is found.
+ */
+directory_token_t *
+find_opt_by_keyword(smartlist_t *s, directory_keyword keyword)
+{
+  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
+  return NULL;
+}
+
+/** If there are any directory_token_t entries in <b>s</b> whose keyword is
+ * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
+ * in the same order in which they occur in <b>s</b>.  Otherwise return
+ * NULL. */
+smartlist_t *
+find_all_by_keyword(smartlist_t *s, directory_keyword k)
+{
+  smartlist_t *out = NULL;
+  SMARTLIST_FOREACH(s, directory_token_t *, t,
+                    if (t->tp == k) {
+                    if (!out)
+                    out = smartlist_new();
+                    smartlist_add(out, t);
+                    });
+  return out;
+}
+
diff --git a/src/or/parsecommon.h b/src/or/parsecommon.h
new file mode 100644
index 0000000000..b9f1613457
--- /dev/null
+++ b/src/or/parsecommon.h
@@ -0,0 +1,321 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file parsecommon.h
+ * \brief Header file for parsecommon.c
+ **/
+
+#ifndef TOR_PARSECOMMON_H
+#define TOR_PARSECOMMON_H
+
+#include "container.h"
+#include "crypto.h"
+#include "memarea.h"
+
+/** Enumeration of possible token types.  The ones starting with K_ correspond
+* to directory 'keywords'. A_ is for an annotation, R or C is related to
+* hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
+* end-of-file marker, and NIL_ is used to encode not-a-token.
+*/
+typedef enum {
+  K_ACCEPT = 0,
+  K_ACCEPT6,
+  K_DIRECTORY_SIGNATURE,
+  K_RECOMMENDED_SOFTWARE,
+  K_REJECT,
+  K_REJECT6,
+  K_ROUTER,
+  K_SIGNED_DIRECTORY,
+  K_SIGNING_KEY,
+  K_ONION_KEY,
+  K_ONION_KEY_NTOR,
+  K_ROUTER_SIGNATURE,
+  K_PUBLISHED,
+  K_RUNNING_ROUTERS,
+  K_ROUTER_STATUS,
+  K_PLATFORM,
+  K_PROTO,
+  K_OPT,
+  K_BANDWIDTH,
+  K_CONTACT,
+  K_NETWORK_STATUS,
+  K_UPTIME,
+  K_DIR_SIGNING_KEY,
+  K_FAMILY,
+  K_FINGERPRINT,
+  K_HIBERNATING,
+  K_READ_HISTORY,
+  K_WRITE_HISTORY,
+  K_NETWORK_STATUS_VERSION,
+  K_DIR_SOURCE,
+  K_DIR_OPTIONS,
+  K_CLIENT_VERSIONS,
+  K_SERVER_VERSIONS,
+  K_RECOMMENDED_CLIENT_PROTOCOLS,
+  K_RECOMMENDED_RELAY_PROTOCOLS,
+  K_REQUIRED_CLIENT_PROTOCOLS,
+  K_REQUIRED_RELAY_PROTOCOLS,
+  K_OR_ADDRESS,
+  K_ID,
+  K_P,
+  K_P6,
+  K_R,
+  K_A,
+  K_S,
+  K_V,
+  K_W,
+  K_M,
+  K_EXTRA_INFO,
+  K_EXTRA_INFO_DIGEST,
+  K_CACHES_EXTRA_INFO,
+  K_HIDDEN_SERVICE_DIR,
+  K_ALLOW_SINGLE_HOP_EXITS,
+  K_IPV6_POLICY,
+  K_ROUTER_SIG_ED25519,
+  K_IDENTITY_ED25519,
+  K_MASTER_KEY_ED25519,
+  K_ONION_KEY_CROSSCERT,
+  K_NTOR_ONION_KEY_CROSSCERT,
+
+  K_DIRREQ_END,
+  K_DIRREQ_V2_IPS,
+  K_DIRREQ_V3_IPS,
+  K_DIRREQ_V2_REQS,
+  K_DIRREQ_V3_REQS,
+  K_DIRREQ_V2_SHARE,
+  K_DIRREQ_V3_SHARE,
+  K_DIRREQ_V2_RESP,
+  K_DIRREQ_V3_RESP,
+  K_DIRREQ_V2_DIR,
+  K_DIRREQ_V3_DIR,
+  K_DIRREQ_V2_TUN,
+  K_DIRREQ_V3_TUN,
+  K_ENTRY_END,
+  K_ENTRY_IPS,
+  K_CELL_END,
+  K_CELL_PROCESSED,
+  K_CELL_QUEUED,
+  K_CELL_TIME,
+  K_CELL_CIRCS,
+  K_EXIT_END,
+  K_EXIT_WRITTEN,
+  K_EXIT_READ,
+  K_EXIT_OPENED,
+
+  K_DIR_KEY_CERTIFICATE_VERSION,
+  K_DIR_IDENTITY_KEY,
+  K_DIR_KEY_PUBLISHED,
+  K_DIR_KEY_EXPIRES,
+  K_DIR_KEY_CERTIFICATION,
+  K_DIR_KEY_CROSSCERT,
+  K_DIR_ADDRESS,
+  K_DIR_TUNNELLED,
+
+  K_VOTE_STATUS,
+  K_VALID_AFTER,
+  K_FRESH_UNTIL,
+  K_VALID_UNTIL,
+  K_VOTING_DELAY,
+
+  K_KNOWN_FLAGS,
+  K_PARAMS,
+  K_BW_WEIGHTS,
+  K_VOTE_DIGEST,
+  K_CONSENSUS_DIGEST,
+  K_ADDITIONAL_DIGEST,
+  K_ADDITIONAL_SIGNATURE,
+  K_CONSENSUS_METHODS,
+  K_CONSENSUS_METHOD,
+  K_LEGACY_DIR_KEY,
+  K_DIRECTORY_FOOTER,
+  K_SIGNING_CERT_ED,
+  K_SR_FLAG,
+  K_COMMIT,
+  K_PREVIOUS_SRV,
+  K_CURRENT_SRV,
+  K_PACKAGE,
+
+  A_PURPOSE,
+  A_LAST_LISTED,
+  A_UNKNOWN_,
+
+  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
+  R_VERSION,
+  R_PERMANENT_KEY,
+  R_SECRET_ID_PART,
+  R_PUBLICATION_TIME,
+  R_PROTOCOL_VERSIONS,
+  R_INTRODUCTION_POINTS,
+  R_SIGNATURE,
+
+  R_HS_DESCRIPTOR, /* From version 3, this MUST be generic to all future
+                      descriptor versions thus making it R_. */
+  R3_DESC_LIFETIME,
+  R3_DESC_SIGNING_CERT,
+  R3_REVISION_COUNTER,
+  R3_SUPERENCRYPTED,
+  R3_SIGNATURE,
+  R3_CREATE2_FORMATS,
+  R3_INTRO_AUTH_REQUIRED,
+  R3_SINGLE_ONION_SERVICE,
+  R3_INTRODUCTION_POINT,
+  R3_INTRO_AUTH_KEY,
+  R3_INTRO_ENC_KEY,
+  R3_INTRO_ENC_KEY_CERT,
+  R3_INTRO_LEGACY_KEY,
+  R3_INTRO_LEGACY_KEY_CERT,
+  R3_DESC_AUTH_TYPE,
+  R3_DESC_AUTH_KEY,
+  R3_DESC_AUTH_CLIENT,
+  R3_ENCRYPTED,
+
+  R_IPO_IDENTIFIER,
+  R_IPO_IP_ADDRESS,
+  R_IPO_ONION_PORT,
+  R_IPO_ONION_KEY,
+  R_IPO_SERVICE_KEY,
+
+  C_CLIENT_NAME,
+  C_DESCRIPTOR_COOKIE,
+  C_CLIENT_KEY,
+
+  ERR_,
+  EOF_,
+  NIL_
+} directory_keyword;
+
+/** Structure to hold a single directory token.
+ *
+ * We parse a directory by breaking it into "tokens", each consisting
+ * of a keyword, a line full of arguments, and a binary object.  The
+ * arguments and object are both optional, depending on the keyword
+ * type.
+ *
+ * This structure is only allocated in memareas; do not allocate it on
+ * the heap, or token_clear() won't work.
+ */
+typedef struct directory_token_t {
+  directory_keyword tp;        /**< Type of the token. */
+  int n_args:30;               /**< Number of elements in args */
+  char **args;                 /**< Array of arguments from keyword line. */
+
+  char *object_type;           /**< -----BEGIN [object_type]-----*/
+  size_t object_size;          /**< Bytes in object_body */
+  char *object_body;           /**< Contents of object, base64-decoded. */
+
+  crypto_pk_t *key;        /**< For public keys only.  Heap-allocated. */
+
+  char *error;                 /**< For ERR_ tokens only. */
+} directory_token_t;
+
+/** We use a table of rules to decide how to parse each token type. */
+
+/** Rules for whether the keyword needs an object. */
+typedef enum {
+  NO_OBJ,        /**< No object, ever. */
+  NEED_OBJ,      /**< Object is required. */
+  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
+  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
+  NEED_KEY,      /**< Object is required, and must be a public key. */
+  OBJ_OK,        /**< Object is optional. */
+} obj_syntax;
+
+#define AT_START 1
+#define AT_END 2
+
+#define TS_ANNOTATIONS_OK 1
+#define TS_NOCHECK 2
+#define TS_NO_NEW_ANNOTATIONS 4
+
+/**
+ * @name macros for defining token rules
+ *
+ * Helper macros to define token tables.  's' is a string, 't' is a
+ * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
+ * object syntax.
+ */
+/**@{*/
+
+/** Appears to indicate the end of a table. */
+#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions: used for obsolete document types */
+#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item with no restrictions on multiplicity or location. */
+#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
+/** An item that must appear exactly once */
+#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
+/** An item that must appear exactly once, at the start of the document */
+#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
+/** An item that must appear exactly once, at the end of the document */
+#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
+/** An item that must appear one or more times */
+#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
+/** An item that must appear no more than once */
+#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
+/** An annotation that must appear no more than once */
+#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
+
+/** Argument multiplicity: any number of arguments. */
+#define ARGS        0,INT_MAX,0
+/** Argument multiplicity: no arguments. */
+#define NO_ARGS     0,0,0
+/** Argument multiplicity: concatenate all arguments. */
+#define CONCAT_ARGS 1,1,1
+/** Argument multiplicity: at least <b>n</b> arguments. */
+#define GE(n)       n,INT_MAX,0
+/** Argument multiplicity: exactly <b>n</b> arguments. */
+#define EQ(n)       n,n,0
+/**@}*/
+
+/** Determines the parsing rules for a single token type. */
+typedef struct token_rule_t {
+  /** The string value of the keyword identifying the type of item. */
+  const char *t;
+  /** The corresponding directory_keyword enum. */
+  directory_keyword v;
+  /** Minimum number of arguments for this item */
+  int min_args;
+  /** Maximum number of arguments for this item */
+  int max_args;
+  /** If true, we concatenate all arguments for this item into a single
+   * string. */
+  int concat_args;
+  /** Requirements on object syntax for this item. */
+  obj_syntax os;
+  /** Lowest number of times this item may appear in a document. */
+  int min_cnt;
+  /** Highest number of times this item may appear in a document. */
+  int max_cnt;
+  /** One or more of AT_START/AT_END to limit where the item may appear in a
+   * document. */
+  int pos;
+  /** True iff this token is an annotation. */
+  int is_annotation;
+} token_rule_t;
+
+void token_clear(directory_token_t *tok);
+
+int tokenize_string(memarea_t *area,
+                    const char *start, const char *end,
+                    smartlist_t *out,
+                    token_rule_t *table,
+                    int flags);
+directory_token_t *get_next_token(memarea_t *area,
+                                  const char **s,
+                                  const char *eos,
+                                  token_rule_t *table);
+
+directory_token_t *find_by_keyword_(smartlist_t *s,
+                                    directory_keyword keyword,
+                                    const char *keyword_str);
+
+#define find_by_keyword(s, keyword) \
+  find_by_keyword_((s), (keyword), #keyword)
+
+directory_token_t *find_opt_by_keyword(smartlist_t *s,
+                                       directory_keyword keyword);
+smartlist_t * find_all_by_keyword(smartlist_t *s, directory_keyword k);
+
+#endif /* TOR_PARSECOMMON_H */
+
diff --git a/src/or/periodic.c b/src/or/periodic.c
index d02d4a7bbb..6896b41c86 100644
--- a/src/or/periodic.c
+++ b/src/or/periodic.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/periodic.h b/src/or/periodic.h
index 021bb4ef5c..88d00cc7e9 100644
--- a/src/or/periodic.h
+++ b/src/or/periodic.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_PERIODIC_H
diff --git a/src/or/policies.c b/src/or/policies.c
index f58bf329ad..9e43fd78bb 100644
--- a/src/or/policies.c
+++ b/src/or/policies.c
@@ -1,11 +1,18 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file policies.c
  * \brief Code to parse and use address policies and exit policies.
+ *
+ * We have two key kinds of address policy: full and compressed.  A full
+ * policy is an array of accept/reject patterns, to be applied in order.
+ * A short policy is simply a list of ports.  This module handles both
+ * kinds, including generic functions to apply them to addresses, and
+ * also including code to manage the global policies that we apply to
+ * incoming and outgoing connections.
  **/
 
 #define POLICIES_PRIVATE
@@ -13,6 +20,7 @@
 #include "or.h"
 #include "config.h"
 #include "dirserv.h"
+#include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "policies.h"
@@ -290,8 +298,8 @@ parse_reachable_addresses(void)
     } else if (fascist_firewall_use_ipv6(options)
        && (policy_is_reject_star(reachable_or_addr_policy, AF_INET6, 0)
          || policy_is_reject_star(reachable_dir_addr_policy, AF_INET6, 0))) {
-          log_warn(LD_CONFIG, "You have configured tor to use IPv6 "
-                   "(ClientUseIPv6 1 or UseBridges 1), but "
+          log_warn(LD_CONFIG, "You have configured tor to use or prefer IPv6 "
+                   "(or UseBridges 1), but "
                    "ReachableAddresses, ReachableORAddresses, or "
                    "ReachableDirAddresses reject all IPv6 addresses. "
                    "Tor will not connect using IPv6.");
@@ -309,10 +317,8 @@ firewall_is_fascist_impl(void)
   const or_options_t *options = get_options();
   /* Assume every non-bridge relay has an IPv4 address.
    * Clients which use bridges may only know the IPv6 address of their
-   * bridge. */
-  return (options->ClientUseIPv4 == 0
-          || (!fascist_firewall_use_ipv6(options)
-              && options->UseBridges == 1));
+   * bridge, but they will connect regardless of the ClientUseIPv6 setting. */
+  return options->ClientUseIPv4 == 0;
 }
 
 /** Return true iff the firewall options, including ClientUseIPv4 0 and
@@ -419,6 +425,9 @@ fascist_firewall_allows_address(const tor_addr_t *addr,
 }
 
 /** Is this client configured to use IPv6?
+ * Returns true if the client might use IPv6 for some of its connections
+ * (including dual-stack and IPv6-only clients), and false if it will never
+ * use IPv6 for any connections.
  * Use node_ipv6_or/dir_preferred() when checking a specific node and OR/Dir
  * port: it supports bridge client per-node IPv6 preferences.
  */
@@ -426,9 +435,11 @@ int
 fascist_firewall_use_ipv6(const or_options_t *options)
 {
   /* Clients use IPv6 if it's set, or they use bridges, or they don't use
-   * IPv4 */
-  return (options->ClientUseIPv6 == 1 || options->UseBridges == 1
-          || options->ClientUseIPv4 == 0);
+   * IPv4, or they prefer it.
+   * ClientPreferIPv6DirPort is deprecated, but check it anyway. */
+  return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 ||
+          options->ClientPreferIPv6ORPort == 1 ||
+          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1);
 }
 
 /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and
@@ -883,6 +894,33 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
                                               pref_ipv6, ap);
 }
 
+/* The microdescriptor consensus has no IPv6 addresses in rs: they are in
+ * the microdescriptors. This means we can't rely on the node's IPv6 address
+ * until its microdescriptor is available (when using microdescs).
+ * But for bridges, rewrite_node_address_for_bridge() updates node->ri with
+ * the configured address, so we can trust bridge addresses.
+ * (Bridges could gain an IPv6 address if their microdescriptor arrives, but
+ * this will never be their preferred address: that is in the config.)
+ * Returns true if the node needs a microdescriptor for its IPv6 address, and
+ * false if the addresses in the node are already up-to-date.
+ */
+static int
+node_awaiting_ipv6(const or_options_t* options, const node_t *node)
+{
+  tor_assert(node);
+
+  /* There's no point waiting for an IPv6 address if we'd never use it */
+  if (!fascist_firewall_use_ipv6(options)) {
+    return 0;
+  }
+
+  /* We are waiting if we_use_microdescriptors_for_circuits() and we have no
+   * md. Bridges have a ri based on their config. They would never use the
+   * address from their md, so there's no need to wait for it. */
+  return (!node->md && we_use_microdescriptors_for_circuits(options) &&
+          !node->ri);
+}
+
 /** Like fascist_firewall_choose_address_base(), but takes <b>rs</b>.
  * Consults the corresponding node, then falls back to rs if node is NULL.
  * This should only happen when there's no valid consensus, and rs doesn't
@@ -899,15 +937,15 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
 
   tor_assert(ap);
 
+  const or_options_t *options = get_options();
   const node_t *node = node_get_by_id(rs->identity_digest);
 
-  if (node) {
+  if (node && !node_awaiting_ipv6(options, node)) {
     return fascist_firewall_choose_address_node(node, fw_connection, pref_only,
                                                 ap);
   } else {
     /* There's no node-specific IPv6 preference, so use the generic IPv6
      * preference instead. */
-    const or_options_t *options = get_options();
     int pref_ipv6 = (fw_connection == FIREWALL_OR_CONNECTION
                      ? fascist_firewall_prefer_ipv6_orport(options)
                      : fascist_firewall_prefer_ipv6_dirport(options));
@@ -941,6 +979,18 @@ fascist_firewall_choose_address_node(const node_t *node,
 
   node_assert_ok(node);
 
+  /* Calling fascist_firewall_choose_address_node() when the node is missing
+   * IPv6 information breaks IPv6-only clients.
+   * If the node is a hard-coded fallback directory or authority, call
+   * fascist_firewall_choose_address_rs() on the fake (hard-coded) routerstatus
+   * for the node.
+   * If it is not hard-coded, check that the node has a microdescriptor, full
+   * descriptor (routerinfo), or is one of our configured bridges before
+   * calling this function. */
+  if (BUG(node_awaiting_ipv6(get_options(), node))) {
+    return 0;
+  }
+
   const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION
                               ? node_ipv6_or_preferred(node)
                               : node_ipv6_dir_preferred(node));
@@ -1971,10 +2021,10 @@ policies_copy_ipv4h_to_smartlist(smartlist_t *addr_list, uint32_t ipv4h_addr)
   }
 }
 
-/** Helper function that adds copies of
- * or_options->OutboundBindAddressIPv[4|6]_ to a smartlist as tor_addr_t *, as
- * long as or_options is non-NULL, and the addresses are not
- * tor_addr_is_null(), by passing them to policies_add_addr_to_smartlist.
+/** Helper function that adds copies of or_options->OutboundBindAddresses
+ * to a smartlist as tor_addr_t *, as long as or_options is non-NULL, and
+ * the addresses are not tor_addr_is_null(), by passing them to
+ * policies_add_addr_to_smartlist.
  *
  * The caller is responsible for freeing all the tor_addr_t* in the smartlist.
  */
@@ -1983,10 +2033,14 @@ policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list,
                                               const or_options_t *or_options)
 {
   if (or_options) {
-    policies_copy_addr_to_smartlist(addr_list,
-                                    &or_options->OutboundBindAddressIPv4_);
-    policies_copy_addr_to_smartlist(addr_list,
-                                    &or_options->OutboundBindAddressIPv6_);
+    for (int i=0;i<OUTBOUND_ADDR_MAX;i++) {
+      for (int j=0;j<2;j++) {
+        if (!tor_addr_is_null(&or_options->OutboundBindAddresses[i][j])) {
+          policies_copy_addr_to_smartlist(addr_list,
+                          &or_options->OutboundBindAddresses[i][j]);
+        }
+      }
+    }
   }
 }
 
@@ -2003,10 +2057,10 @@ policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list,
  *  - if ipv6_local_address is non-NULL, and not the null tor_addr_t, add it
  *    to the list of configured addresses.
  * If <b>or_options->ExitPolicyRejectLocalInterfaces</b> is true:
- *  - if or_options->OutboundBindAddressIPv4_ is not the null tor_addr_t, add
- *    it to the list of configured addresses.
- *  - if or_options->OutboundBindAddressIPv6_ is not the null tor_addr_t, add
- *    it to the list of configured addresses.
+ *  - if or_options->OutboundBindAddresses[][0] (=IPv4) is not the null
+ *    tor_addr_t, add it to the list of configured addresses.
+ *  - if or_options->OutboundBindAddresses[][1] (=IPv6) is not the null
+ *    tor_addr_t, add it to the list of configured addresses.
  *
  * If <b>or_options->BridgeRelay</b> is false, append entries of default
  * Tor exit policy into <b>result</b> smartlist.
@@ -2528,9 +2582,9 @@ policy_summarize(smartlist_t *policy, sa_family_t family)
         tor_snprintf(buf, sizeof(buf), "%d-%d", start_prt, AT(i)->prt_max);
 
       if (AT(i)->accepted)
-        smartlist_add(accepts, tor_strdup(buf));
+        smartlist_add_strdup(accepts, buf);
       else
-        smartlist_add(rejects, tor_strdup(buf));
+        smartlist_add_strdup(rejects, buf);
 
       if (last)
         break;
@@ -2710,7 +2764,7 @@ write_short_policy(const short_policy_t *policy)
       smartlist_add_asprintf(sl, "%d-%d", e->min_port, e->max_port);
     }
     if (i < policy->n_entries-1)
-      smartlist_add(sl, tor_strdup(","));
+      smartlist_add_strdup(sl, ",");
   }
   answer = smartlist_join_strings(sl, "", 0, NULL);
   SMARTLIST_FOREACH(sl, char *, a, tor_free(a));
diff --git a/src/or/policies.h b/src/or/policies.h
index f73f850c21..ce08d497e9 100644
--- a/src/or/policies.h
+++ b/src/or/policies.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/protover.c b/src/or/protover.c
index 31ca13fe61..e8524a25b5 100644
--- a/src/or/protover.c
+++ b/src/or/protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -300,12 +300,12 @@ protover_get_supported_protocols(void)
   return
     "Cons=1-2 "
     "Desc=1-2 "
-    "DirCache=1 "
-    "HSDir=1 "
-    "HSIntro=3 "
+    "DirCache=1-2 "
+    "HSDir=1-2 "
+    "HSIntro=3-4 "
     "HSRend=1-2 "
-    "Link=1-4 "
-    "LinkAuth=1 "
+    "Link=1-5 "
+    "LinkAuth=1,3 "
     "Microdesc=1-2 "
     "Relay=1-2";
 }
@@ -360,7 +360,7 @@ encode_protocol_list(const smartlist_t *sl)
   const char *separator = "";
   smartlist_t *chunks = smartlist_new();
   SMARTLIST_FOREACH_BEGIN(sl, const proto_entry_t *, ent) {
-    smartlist_add(chunks, tor_strdup(separator));
+    smartlist_add_strdup(chunks, separator);
 
     proto_entry_encode_into(chunks, ent);
 
@@ -493,7 +493,7 @@ contract_protocol_list(const smartlist_t *proto_strings)
     smartlist_sort(lst, cmp_single_ent_by_version);
 
     if (! first_entry)
-      smartlist_add(chunks, tor_strdup(" "));
+      smartlist_add_strdup(chunks, " ");
 
     /* We're going to construct this entry from the ranges. */
     proto_entry_t *entry = tor_malloc_zero(sizeof(proto_entry_t));
diff --git a/src/or/protover.h b/src/or/protover.h
index 5c658931ea..22667bed79 100644
--- a/src/or/protover.h
+++ b/src/or/protover.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/reasons.c b/src/or/reasons.c
index a1566e2299..e6c325f1b3 100644
--- a/src/or/reasons.c
+++ b/src/or/reasons.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/reasons.h b/src/or/reasons.h
index 2e12c93728..1cadf4e89e 100644
--- a/src/or/reasons.h
+++ b/src/or/reasons.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/relay.c b/src/or/relay.c
index 1c791e02cc..2451f45c79 100644
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -1,30 +1,68 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file relay.c
  * \brief Handle relay cell encryption/decryption, plus packaging and
  *    receiving from circuits, plus queuing on circuits.
+ *
+ * This is a core modules that makes Tor work. It's responsible for
+ * dealing with RELAY cells (the ones that travel more than one hop along a
+ * circuit), by:
+ *  <ul>
+ *   <li>constructing relays cells,
+ *   <li>encrypting relay cells,
+ *   <li>decrypting relay cells,
+ *   <li>demultiplexing relay cells as they arrive on a connection,
+ *   <li>queueing relay cells for retransmission,
+ *   <li>or handling relay cells that are for us to receive (as an exit or a
+ *   client).
+ *  </ul>
+ *
+ * RELAY cells are generated throughout the code at the client or relay side,
+ * using relay_send_command_from_edge() or one of the functions like
+ * connection_edge_send_command() that calls it.  Of particular interest is
+ * connection_edge_package_raw_inbuf(), which takes information that has
+ * arrived on an edge connection socket, and packages it as a RELAY_DATA cell
+ * -- this is how information is actually sent across the Tor network.  The
+ * cryptography for these functions is handled deep in
+ * circuit_package_relay_cell(), which either adds a single layer of
+ * encryption (if we're an exit), or multiple layers (if we're the origin of
+ * the circuit).  After construction and encryption, the RELAY cells are
+ * passed to append_cell_to_circuit_queue(), which queues them for
+ * transmission and tells the circuitmux (see circuitmux.c) that the circuit
+ * is waiting to send something.
+ *
+ * Incoming RELAY cells arrive at circuit_receive_relay_cell(), called from
+ * command.c.  There they are decrypted and, if they are for us, are passed to
+ * connection_edge_process_relay_cell(). If they're not for us, they're
+ * re-queued for retransmission again with append_cell_to_circuit_queue().
+ *
+ * The connection_edge_process_relay_cell() function handles all the different
+ * types of relay cells, launching requests or transmitting data as needed.
  **/
 
 #define RELAY_PRIVATE
 #include "or.h"
 #include "addressmap.h"
+#include "backtrace.h"
 #include "buffers.h"
 #include "channel.h"
 #include "circpathbias.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
 #include "circuituse.h"
+#include "compress.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
 #include "geoip.h"
+#include "hs_cache.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -38,6 +76,7 @@
 #include "routerlist.h"
 #include "routerparse.h"
 #include "scheduler.h"
+#include "rephist.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
@@ -160,6 +199,82 @@ relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in,
   return 0;
 }
 
+/**
+ * Update channel usage state based on the type of relay cell and
+ * circuit properties.
+ *
+ * This is needed to determine if a client channel is being
+ * used for application traffic, and if a relay channel is being
+ * used for multihop circuits and application traffic. The decision
+ * to pad in channelpadding.c depends upon this info (as well as
+ * consensus parameters) to decide what channels to pad.
+ */
+static void
+circuit_update_channel_usage(circuit_t *circ, cell_t *cell)
+{
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /*
+     * The client state was first set much earlier in
+     * circuit_send_next_onion_skin(), so we can start padding as early as
+     * possible.
+     *
+     * However, if padding turns out to be expensive, we may want to not do
+     * it until actual application traffic starts flowing (which is controlled
+     * via consensus param nf_pad_before_usage).
+     *
+     * So: If we're an origin circuit and we've created a full length circuit,
+     * then any CELL_RELAY cell means application data. Increase the usage
+     * state of the channel to indicate this.
+     *
+     * We want to wait for CELL_RELAY specifically here, so we know that
+     * the channel was definitely being used for data and not for extends.
+     * By default, we pad as soon as a channel has been used for *any*
+     * circuits, so this state is irrelevant to the padding decision in
+     * the default case. However, if padding turns out to be expensive,
+     * we would like the ability to avoid padding until we're absolutely
+     * sure that a channel is used for enough application data to be worth
+     * padding.
+     *
+     * (So it does not matter that CELL_RELAY_EARLY can actually contain
+     * application data. This is only a load reducing option and that edge
+     * case does not matter if we're desperately trying to reduce overhead
+     * anyway. See also consensus parameter nf_pad_before_usage).
+     */
+    if (BUG(!circ->n_chan))
+      return;
+
+    if (circ->n_chan->channel_usage == CHANNEL_USED_FOR_FULL_CIRCS &&
+        cell->command == CELL_RELAY) {
+      circ->n_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC;
+    }
+  } else {
+    /* If we're a relay circuit, the question is more complicated. Basically:
+     * we only want to pad connections that carry multihop (anonymous)
+     * circuits.
+     *
+     * We assume we're more than one hop if either the previous hop
+     * is not a client, or if the previous hop is a client and there's
+     * a next hop. Then, circuit traffic starts at RELAY_EARLY, and
+     * user application traffic starts when we see RELAY cells.
+     */
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+
+    if (BUG(!or_circ->p_chan))
+      return;
+
+    if (!channel_is_client(or_circ->p_chan) ||
+        (channel_is_client(or_circ->p_chan) && circ->n_chan)) {
+      if (cell->command == CELL_RELAY_EARLY) {
+        if (or_circ->p_chan->channel_usage < CHANNEL_USED_FOR_FULL_CIRCS) {
+          or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+        }
+      } else if (cell->command == CELL_RELAY) {
+        or_circ->p_chan->channel_usage = CHANNEL_USED_FOR_USER_TRAFFIC;
+      }
+    }
+  }
+}
+
 /** Receive a relay cell:
  *  - Crypt it (encrypt if headed toward the origin or if we <b>are</b> the
  *    origin; decrypt if we're headed toward the exit).
@@ -189,10 +304,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     return 0;
 
   if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) {
-    log_warn(LD_BUG,"relay crypt failed. Dropping connection.");
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "relay crypt failed. Dropping connection.");
     return -END_CIRC_REASON_INTERNAL;
   }
 
+  circuit_update_channel_usage(circ, cell);
+
   if (recognized) {
     edge_connection_t *conn = NULL;
 
@@ -221,8 +339,13 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(LD_OR,"Sending to origin.");
       if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
                                                        layer_hint)) < 0) {
-        log_warn(LD_OR,
-                 "connection_edge_process_relay_cell (at origin) failed.");
+        /* If a client is trying to connect to unknown hidden service port,
+         * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
+         * Do not log warn as this is an expected behavior for a service. */
+        if (reason != END_CIRC_AT_ORIGIN) {
+          log_warn(LD_OR,
+                   "connection_edge_process_relay_cell (at origin) failed.");
+        }
         return reason;
       }
     }
@@ -394,11 +517,13 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
     if (!chan) {
       log_warn(LD_BUG,"outgoing relay cell sent from %s:%d has n_chan==NULL."
                " Dropping.", filename, lineno);
+      log_backtrace(LOG_WARN,LD_BUG,"");
       return 0; /* just drop it */
     }
     if (!CIRCUIT_IS_ORIGIN(circ)) {
       log_warn(LD_BUG,"outgoing relay cell sent from %s:%d on non-origin "
                "circ. Dropping.", filename, lineno);
+      log_backtrace(LOG_WARN,LD_BUG,"");
       return 0; /* just drop it */
     }
 
@@ -564,11 +689,11 @@ relay_command_to_string(uint8_t command)
  * If you can't send the cell, mark the circuit for close and return -1. Else
  * return 0.
  */
-int
-relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
-                              uint8_t relay_command, const char *payload,
-                              size_t payload_len, crypt_path_t *cpath_layer,
-                              const char *filename, int lineno)
+MOCK_IMPL(int,
+relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
+                               uint8_t relay_command, const char *payload,
+                               size_t payload_len, crypt_path_t *cpath_layer,
+                               const char *filename, int lineno))
 {
   cell_t cell;
   relay_header_t rh;
@@ -580,14 +705,14 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
 
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_RELAY;
-  if (cpath_layer) {
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(cpath_layer);
     cell.circ_id = circ->n_circ_id;
     cell_direction = CELL_DIRECTION_OUT;
-  } else if (! CIRCUIT_IS_ORIGIN(circ)) {
+  } else {
+    tor_assert(! cpath_layer);
     cell.circ_id = TO_OR_CIRCUIT(circ)->p_circ_id;
     cell_direction = CELL_DIRECTION_IN;
-  } else {
-    return -1;
   }
 
   memset(&rh, 0, sizeof(rh));
@@ -601,6 +726,9 @@ relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
+  if (relay_command == RELAY_COMMAND_DROP)
+    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
   if (relay_command == RELAY_COMMAND_END && circ->dirreq_id)
@@ -707,6 +835,16 @@ connection_edge_send_command(edge_connection_t *fromconn,
     return -1;
   }
 
+#ifdef MEASUREMENTS_21206
+  /* Keep track of the number of RELAY_DATA cells sent for directory
+   * connections. */
+  connection_t *linked_conn = TO_CONN(fromconn)->linked_conn;
+
+  if (linked_conn && linked_conn->type == CONN_TYPE_DIR) {
+    ++(TO_DIR_CONN(linked_conn)->data_cells_sent);
+  }
+#endif
+
   return relay_send_command_from_edge(fromconn->stream_id, circ,
                                       relay_command, payload,
                                       payload_len, cpath_layer);
@@ -1490,6 +1628,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
   switch (rh.command) {
     case RELAY_COMMAND_DROP:
+      rep_hist_padding_count_read(PADDING_TYPE_DROP);
 //      log_info(domain,"Got a relay-level padding cell. Dropping.");
       return 0;
     case RELAY_COMMAND_BEGIN:
@@ -1563,6 +1702,16 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       connection_write_to_buf((char*)(cell->payload + RELAY_HEADER_SIZE),
                               rh.length, TO_CONN(conn));
 
+#ifdef MEASUREMENTS_21206
+      /* Count number of RELAY_DATA cells received on a linked directory
+       * connection. */
+      connection_t *linked_conn = TO_CONN(conn)->linked_conn;
+
+      if (linked_conn && linked_conn->type == CONN_TYPE_DIR) {
+        ++(TO_DIR_CONN(linked_conn)->data_cells_received);
+      }
+#endif
+
       if (!optimistic_data) {
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
@@ -2473,7 +2622,7 @@ cell_queues_check_size(void)
   time_t now = time(NULL);
   size_t alloc = cell_queues_get_total_allocation();
   alloc += buf_get_total_allocation();
-  alloc += tor_zlib_get_total_allocation();
+  alloc += tor_compress_get_total_allocation();
   const size_t rend_cache_total = rend_cache_get_total_allocation();
   alloc += rend_cache_total;
   const size_t geoip_client_cache_total =
@@ -2488,9 +2637,7 @@ cell_queues_check_size(void)
       if (rend_cache_total > get_options()->MaxMemInQueues / 5) {
         const size_t bytes_to_remove =
           rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10);
-        rend_cache_clean_v2_descs_as_dir(now, bytes_to_remove);
-        alloc -= rend_cache_total;
-        alloc += rend_cache_get_total_allocation();
+        alloc -= hs_cache_handle_oom(time(NULL), bytes_to_remove);
       }
       if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) {
         const size_t bytes_to_remove =
diff --git a/src/or/relay.h b/src/or/relay.h
index c4f98d92ff..9dc0b5d3a2 100644
--- a/src/or/relay.h
+++ b/src/or/relay.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,10 +20,13 @@ int circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
 
 void relay_header_pack(uint8_t *dest, const relay_header_t *src);
 void relay_header_unpack(relay_header_t *dest, const uint8_t *src);
-int relay_send_command_from_edge_(streamid_t stream_id, circuit_t *circ,
+MOCK_DECL(int,
+relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
                                uint8_t relay_command, const char *payload,
                                size_t payload_len, crypt_path_t *cpath_layer,
-                               const char *filename, int lineno);
+                               const char *filename, int lineno));
+/* Indicates to relay_send_command_from_edge() that it is a control cell. */
+#define CONTROL_CELL_ID 0
 #define relay_send_command_from_edge(stream_id, circ, relay_command, payload, \
                                      payload_len, cpath_layer)          \
   relay_send_command_from_edge_((stream_id), (circ), (relay_command),   \
diff --git a/src/or/rendcache.c b/src/or/rendcache.c
index aa69d735fe..11b60b36a1 100644
--- a/src/or/rendcache.c
+++ b/src/or/rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -86,7 +86,7 @@ rend_cache_get_total_allocation(void)
 }
 
 /** Decrement the total bytes attributed to the rendezvous cache by n. */
-STATIC void
+void
 rend_cache_decrement_allocation(size_t n)
 {
   static int have_underflowed = 0;
@@ -103,7 +103,7 @@ rend_cache_decrement_allocation(size_t n)
 }
 
 /** Increase the total bytes attributed to the rendezvous cache by n. */
-STATIC void
+void
 rend_cache_increment_allocation(size_t n)
 {
   static int have_overflowed = 0;
@@ -462,45 +462,36 @@ rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
 }
 
 /** Remove all old v2 descriptors and those for which this hidden service
- * directory is not responsible for any more.
- *
- * If at all possible, remove at least <b>force_remove</b> bytes of data.
- */
-void
-rend_cache_clean_v2_descs_as_dir(time_t now, size_t force_remove)
+ * directory is not responsible for any more. The cutoff is the time limit for
+ * which we want to keep the cache entry. In other words, any entry created
+ * before will be removed. */
+size_t
+rend_cache_clean_v2_descs_as_dir(time_t cutoff)
 {
   digestmap_iter_t *iter;
-  time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW;
-  const int LAST_SERVED_CUTOFF_STEP = 1800;
-  time_t last_served_cutoff = cutoff;
   size_t bytes_removed = 0;
-  do {
-    for (iter = digestmap_iter_init(rend_cache_v2_dir);
-         !digestmap_iter_done(iter); ) {
-      const char *key;
-      void *val;
-      rend_cache_entry_t *ent;
-      digestmap_iter_get(iter, &key, &val);
-      ent = val;
-      if (ent->parsed->timestamp < cutoff ||
-          ent->last_served < last_served_cutoff) {
-        char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-        base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
-        log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
-                 safe_str_client(key_base32));
-        bytes_removed += rend_cache_entry_allocation(ent);
-        iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
-        rend_cache_entry_free(ent);
-      } else {
-        iter = digestmap_iter_next(rend_cache_v2_dir, iter);
-      }
+
+  for (iter = digestmap_iter_init(rend_cache_v2_dir);
+       !digestmap_iter_done(iter); ) {
+    const char *key;
+    void *val;
+    rend_cache_entry_t *ent;
+    digestmap_iter_get(iter, &key, &val);
+    ent = val;
+    if (ent->parsed->timestamp < cutoff) {
+      char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+      base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN);
+      log_info(LD_REND, "Removing descriptor with ID '%s' from cache",
+               safe_str_client(key_base32));
+      bytes_removed += rend_cache_entry_allocation(ent);
+      iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter);
+      rend_cache_entry_free(ent);
+    } else {
+      iter = digestmap_iter_next(rend_cache_v2_dir, iter);
     }
+  }
 
-    /* In case we didn't remove enough bytes, advance the cutoff a little. */
-    last_served_cutoff += LAST_SERVED_CUTOFF_STEP;
-    if (last_served_cutoff > now)
-      break;
-  } while (bytes_removed < force_remove);
+  return bytes_removed;
 }
 
 /** Lookup in the client cache the given service ID <b>query</b> for
@@ -849,6 +840,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   char want_desc_id[DIGEST_LEN];
   rend_cache_entry_t *e;
   int retval = -1;
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
+
   tor_assert(rend_cache);
   tor_assert(desc);
   tor_assert(desc_id_base32);
@@ -874,11 +867,11 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     log_warn(LD_REND, "Couldn't compute service ID.");
     goto err;
   }
-  if (rend_query->onion_address[0] != '\0' &&
-      strcmp(rend_query->onion_address, service_id)) {
+  if (rend_data->onion_address[0] != '\0' &&
+      strcmp(rend_data->onion_address, service_id)) {
     log_warn(LD_REND, "Received service descriptor for service ID %s; "
              "expected descriptor for service ID %s.",
-             service_id, safe_str(rend_query->onion_address));
+             service_id, safe_str(rend_data->onion_address));
     goto err;
   }
   if (tor_memneq(desc_id, want_desc_id, DIGEST_LEN)) {
@@ -890,14 +883,14 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   /* Decode/decrypt introduction points. */
   if (intro_content && intro_size > 0) {
     int n_intro_points;
-    if (rend_query->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_query->descriptor_cookie,
-                         sizeof(rend_query->descriptor_cookie))) {
+    if (rend_data->auth_type != REND_NO_AUTH &&
+        !tor_mem_is_zero(rend_data->descriptor_cookie,
+                         sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
                                            &ipos_decrypted_size,
-                                           rend_query->descriptor_cookie,
+                                           rend_data->descriptor_cookie,
                                            intro_content,
                                            intro_size) < 0) {
         log_warn(LD_REND, "Failed to decrypt introduction points. We are "
diff --git a/src/or/rendcache.h b/src/or/rendcache.h
index 270b614c38..1bd3be2243 100644
--- a/src/or/rendcache.h
+++ b/src/or/rendcache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -53,10 +53,17 @@ typedef enum {
   REND_CACHE_TYPE_SERVICE = 2,
 } rend_cache_type_t;
 
+/* Return maximum lifetime in seconds of a cache entry. */
+static inline time_t
+rend_cache_max_entry_lifetime(void)
+{
+  return REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW;
+}
+
 void rend_cache_init(void);
 void rend_cache_clean(time_t now, rend_cache_type_t cache_type);
 void rend_cache_failure_clean(time_t now);
-void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove);
+size_t rend_cache_clean_v2_descs_as_dir(time_t cutoff);
 void rend_cache_purge(void);
 void rend_cache_free_all(void);
 int rend_cache_lookup_entry(const char *query, int version,
@@ -77,6 +84,8 @@ void rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
                                    const uint8_t *identity,
                                    const char *service_id);
 void rend_cache_failure_purge(void);
+void rend_cache_decrement_allocation(size_t n);
+void rend_cache_increment_allocation(size_t n);
 
 #ifdef RENDCACHE_PRIVATE
 
@@ -89,8 +98,6 @@ STATIC int cache_failure_intro_lookup(const uint8_t *identity,
                                       const char *service_id,
                                       rend_cache_failure_intro_t
                                       **intro_entry);
-STATIC void rend_cache_decrement_allocation(size_t n);
-STATIC void rend_cache_increment_allocation(size_t n);
 STATIC rend_cache_failure_intro_t *rend_cache_failure_intro_entry_new(
                                       rend_intro_point_failure_t failure);
 STATIC rend_cache_failure_t *rend_cache_failure_entry_new(void);
diff --git a/src/or/rendclient.c b/src/or/rendclient.c
index a93bc94a9c..9e2daf0380 100644
--- a/src/or/rendclient.c
+++ b/src/or/rendclient.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,7 @@
 #include "connection.h"
 #include "connection_edge.h"
 #include "directory.h"
+#include "hs_common.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -104,7 +105,7 @@ rend_client_reextend_intro_circuit(origin_circuit_t *circ)
   if (!extend_info) {
     log_warn(LD_REND,
              "No usable introduction points left for %s. Closing.",
-             safe_str_client(circ->rend_data->onion_address));
+             safe_str_client(rend_data_get_address(circ->rend_data)));
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
     return -1;
   }
@@ -144,18 +145,19 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   off_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
+  const char *onion_address;
 
   tor_assert(introcirc->base_.purpose == CIRCUIT_PURPOSE_C_INTRODUCING);
   tor_assert(rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY);
   tor_assert(introcirc->rend_data);
   tor_assert(rendcirc->rend_data);
-  tor_assert(!rend_cmp_service_ids(introcirc->rend_data->onion_address,
-                                   rendcirc->rend_data->onion_address));
+  tor_assert(!rend_cmp_service_ids(rend_data_get_address(introcirc->rend_data),
+                                  rend_data_get_address(rendcirc->rend_data)));
   assert_circ_anonymity_ok(introcirc, options);
   assert_circ_anonymity_ok(rendcirc, options);
+  onion_address = rend_data_get_address(introcirc->rend_data);
 
-  r = rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1,
-                              &entry);
+  r = rend_cache_lookup_entry(onion_address, -1, &entry);
   /* An invalid onion address is not possible else we have a big issue. */
   tor_assert(r != -EINVAL);
   if (r < 0 || !rend_client_any_intro_points_usable(entry)) {
@@ -164,14 +166,13 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     log_info(LD_REND,
              "query %s didn't have valid rend desc in cache. "
              "Refetching descriptor.",
-             safe_str_client(introcirc->rend_data->onion_address));
+             safe_str_client(onion_address));
     rend_client_refetch_v2_renddesc(introcirc->rend_data);
     {
       connection_t *conn;
 
       while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
-                       AP_CONN_STATE_CIRCUIT_WAIT,
-                       introcirc->rend_data->onion_address))) {
+                       AP_CONN_STATE_CIRCUIT_WAIT, onion_address))) {
         connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
         conn->state = AP_CONN_STATE_RENDDESC_WAIT;
       }
@@ -195,7 +196,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     log_info(LD_REND, "Could not find intro key for %s at %s; we "
              "have a v2 rend desc with %d intro points. "
              "Trying a different intro point...",
-             safe_str_client(introcirc->rend_data->onion_address),
+             safe_str_client(onion_address),
              safe_str_client(extend_info_describe(
                                    introcirc->build_state->chosen_exit)),
              smartlist_len(entry->parsed->intro_nodes));
@@ -235,11 +236,12 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   /* If version is 3, write (optional) auth data and timestamp. */
   if (entry->parsed->protocols & (1<<3)) {
     tmp[0] = 3; /* version 3 of the cell format */
-    tmp[1] = (uint8_t)introcirc->rend_data->auth_type; /* auth type, if any */
+    /* auth type, if any */
+    tmp[1] = (uint8_t) TO_REND_DATA_V2(introcirc->rend_data)->auth_type;
     v3_shift = 1;
-    if (introcirc->rend_data->auth_type != REND_NO_AUTH) {
+    if (tmp[1] != REND_NO_AUTH) {
       set_uint16(tmp+2, htons(REND_DESC_COOKIE_LEN));
-      memcpy(tmp+4, introcirc->rend_data->descriptor_cookie,
+      memcpy(tmp+4, TO_REND_DATA_V2(introcirc->rend_data)->descriptor_cookie,
              REND_DESC_COOKIE_LEN);
       v3_shift += 2+REND_DESC_COOKIE_LEN;
     }
@@ -263,6 +265,11 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
     klen = crypto_pk_asn1_encode(extend_info->onion_key,
                                  tmp+v3_shift+7+DIGEST_LEN+2,
                                  sizeof(tmp)-(v3_shift+7+DIGEST_LEN+2));
+    if (klen < 0) {
+      log_warn(LD_BUG,"Internal error: can't encode public key.");
+      status = -2;
+      goto perm_err;
+    }
     set_uint16(tmp+v3_shift+7+DIGEST_LEN, htons(klen));
     memcpy(tmp+v3_shift+7+DIGEST_LEN+2+klen, rendcirc->rend_data->rend_cookie,
            REND_COOKIE_LEN);
@@ -359,7 +366,7 @@ rend_client_rendcirc_has_opened(origin_circuit_t *circ)
  * Called to close other intro circuits we launched in parallel.
  */
 static void
-rend_client_close_other_intros(const char *onion_address)
+rend_client_close_other_intros(const uint8_t *rend_pk_digest)
 {
   /* abort parallel intro circs, if any */
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) {
@@ -368,8 +375,7 @@ rend_client_close_other_intros(const char *onion_address)
         !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(c);
       if (oc->rend_data &&
-          !rend_cmp_service_ids(onion_address,
-                                oc->rend_data->onion_address)) {
+          rend_circuit_pk_digest_eq(oc, rend_pk_digest)) {
         log_info(LD_REND|LD_CIRC, "Closing introduction circuit %d that we "
                  "built in parallel (Purpose %d).", oc->global_identifier,
                  c->purpose);
@@ -431,7 +437,8 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(circ->rend_data->onion_address);
+    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
+                                                           NULL));
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -440,7 +447,7 @@ rend_client_introduction_acked(origin_circuit_t *circ,
      * If none remain, refetch the service descriptor.
      */
     log_info(LD_REND, "Got nack for %s from %s...",
-        safe_str_client(circ->rend_data->onion_address),
+        safe_str_client(rend_data_get_address(circ->rend_data)),
         safe_str_client(extend_info_describe(circ->build_state->chosen_exit)));
     if (rend_client_report_intro_point_failure(circ->build_state->chosen_exit,
                                              circ->rend_data,
@@ -694,13 +701,15 @@ pick_hsdir(const char *desc_id, const char *desc_id_base32)
  * in the case that no hidden service directory is left to ask for the
  * descriptor, return 0, and in case of a failure -1.  */
 static int
-directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
+directory_get_from_hs_dir(const char *desc_id,
+                          const rend_data_t *rend_query,
                           routerstatus_t *rs_hsdir)
 {
   routerstatus_t *hs_dir = rs_hsdir;
   char *hsdir_fp;
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
   char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64];
+  const rend_data_v2_t *rend_data;
 #ifdef ENABLE_TOR2WEB_MODE
   const int tor2web_mode = get_options()->Tor2webMode;
   const int how_to_fetch = tor2web_mode ? DIRIND_ONEHOP : DIRIND_ANONYMOUS;
@@ -709,6 +718,8 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
 #endif
 
   tor_assert(desc_id);
+  tor_assert(rend_query);
+  rend_data = TO_REND_DATA_V2(rend_query);
 
   base32_encode(desc_id_base32, sizeof(desc_id_base32),
                 desc_id, DIGEST_LEN);
@@ -718,6 +729,9 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
     hs_dir = pick_hsdir(desc_id, desc_id_base32);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
+      control_event_hs_descriptor_failed(rend_query, NULL, "QUERY_NO_HSDIR");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, NULL, NULL);
       return 0;
     }
   }
@@ -731,12 +745,16 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
   /* Encode descriptor cookie for logging purposes. Also, if the cookie is
    * malformed, no fetch is triggered thus this needs to be done before the
    * fetch request. */
-  if (rend_query->auth_type != REND_NO_AUTH) {
+  if (rend_data->auth_type != REND_NO_AUTH) {
     if (base64_encode(descriptor_cookie_base64,
                       sizeof(descriptor_cookie_base64),
-                      rend_query->descriptor_cookie, REND_DESC_COOKIE_LEN,
+                      rend_data->descriptor_cookie,
+                      REND_DESC_COOKIE_LEN,
                       0)<0) {
       log_warn(LD_BUG, "Could not base64-encode descriptor cookie.");
+      control_event_hs_descriptor_failed(rend_query, hsdir_fp, "BAD_DESC");
+      control_event_hs_descriptor_content(rend_data_get_address(rend_query),
+                                          desc_id_base32, hsdir_fp, NULL);
       return 0;
     }
     /* Remove == signs. */
@@ -749,20 +767,22 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
   /* Send fetch request. (Pass query and possibly descriptor cookie so that
    * they can be written to the directory connection and be referred to when
    * the response arrives. */
-  directory_initiate_command_routerstatus_rend(hs_dir,
-                                          DIR_PURPOSE_FETCH_RENDDESC_V2,
-                                          ROUTER_PURPOSE_GENERAL,
-                                          how_to_fetch,
-                                          desc_id_base32,
-                                          NULL, 0, 0,
-                                          rend_query);
+  directory_request_t *req =
+    directory_request_new(DIR_PURPOSE_FETCH_RENDDESC_V2);
+  directory_request_set_routerstatus(req, hs_dir);
+  directory_request_set_indirection(req, how_to_fetch);
+  directory_request_set_resource(req, desc_id_base32);
+  directory_request_set_rend_query(req, rend_query);
+  directory_initiate_request(req);
+  directory_request_free(req);
+
   log_info(LD_REND, "Sending fetch request for v2 descriptor for "
                     "service '%s' with descriptor ID '%s', auth type %d, "
                     "and descriptor cookie '%s' to hidden service "
                     "directory %s",
-           rend_query->onion_address, desc_id_base32,
-           rend_query->auth_type,
-           (rend_query->auth_type == REND_NO_AUTH ? "[none]" :
+           rend_data->onion_address, desc_id_base32,
+           rend_data->auth_type,
+           (rend_data->auth_type == REND_NO_AUTH ? "[none]" :
             escaped_safe_str_client(descriptor_cookie_base64)),
            routerstatus_describe(hs_dir));
   control_event_hs_descriptor_requested(rend_query,
@@ -777,8 +797,8 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query,
  * On success, 1 is returned. If no hidden service is left to ask, return 0.
  * On error, -1 is returned. */
 static int
-fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query,
-                        smartlist_t *hsdirs)
+fetch_v2_desc_by_descid(const char *desc_id,
+                        const rend_data_t *rend_query, smartlist_t *hsdirs)
 {
   int ret;
 
@@ -811,13 +831,12 @@ fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query,
  * On success, 1 is returned. If no hidden service is left to ask, return 0.
  * On error, -1 is returned. */
 static int
-fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
+fetch_v2_desc_by_addr(rend_data_t *rend_query, smartlist_t *hsdirs)
 {
   char descriptor_id[DIGEST_LEN];
   int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS];
   int i, tries_left, ret;
-
-  tor_assert(query);
+  rend_data_v2_t *rend_data = TO_REND_DATA_V2(rend_query);
 
   /* Randomly iterate over the replicas until a descriptor can be fetched
    * from one of the consecutive nodes, or no options are left. */
@@ -831,9 +850,10 @@ fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
     int chosen_replica = replicas_left_to_try[rand_val];
     replicas_left_to_try[rand_val] = replicas_left_to_try[--tries_left];
 
-    ret = rend_compute_v2_desc_id(descriptor_id, query->onion_address,
-                                  query->auth_type == REND_STEALTH_AUTH ?
-                                  query->descriptor_cookie : NULL,
+    ret = rend_compute_v2_desc_id(descriptor_id,
+                                  rend_data->onion_address,
+                                  rend_data->auth_type == REND_STEALTH_AUTH ?
+                                    rend_data->descriptor_cookie : NULL,
                                   time(NULL), chosen_replica);
     if (ret < 0) {
       /* Normally, on failure the descriptor_id is untouched but let's be
@@ -841,18 +861,18 @@ fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs)
       goto end;
     }
 
-    if (tor_memcmp(descriptor_id, query->descriptor_id[chosen_replica],
+    if (tor_memcmp(descriptor_id, rend_data->descriptor_id[chosen_replica],
                    sizeof(descriptor_id)) != 0) {
       /* Not equal from what we currently have so purge the last hid serv
        * request cache and update the descriptor ID with the new value. */
       purge_hid_serv_from_last_hid_serv_requests(
-                                        query->descriptor_id[chosen_replica]);
-      memcpy(query->descriptor_id[chosen_replica], descriptor_id,
-             sizeof(query->descriptor_id[chosen_replica]));
+                                     rend_data->descriptor_id[chosen_replica]);
+      memcpy(rend_data->descriptor_id[chosen_replica], descriptor_id,
+             sizeof(rend_data->descriptor_id[chosen_replica]));
     }
 
     /* Trigger the fetch with the computed descriptor ID. */
-    ret = fetch_v2_desc_by_descid(descriptor_id, query, hsdirs);
+    ret = fetch_v2_desc_by_descid(descriptor_id, rend_query, hsdirs);
     if (ret != 0) {
       /* Either on success or failure, as long as we tried a fetch we are
        * done here. */
@@ -880,16 +900,23 @@ int
 rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs)
 {
   int ret;
+  rend_data_v2_t *rend_data;
+  const char *onion_address;
 
   tor_assert(query);
 
+  /* Get the version 2 data structure of the query. */
+  rend_data = TO_REND_DATA_V2(query);
+  onion_address = rend_data_get_address(query);
+
   /* Depending on what's available in the rend data query object, we will
    * trigger a fetch by HS address or using a descriptor ID. */
 
-  if (query->onion_address[0] != '\0') {
+  if (onion_address[0] != '\0') {
     ret = fetch_v2_desc_by_addr(query, hsdirs);
-  } else if (!tor_digest_is_zero(query->desc_id_fetch)) {
-    ret = fetch_v2_desc_by_descid(query->desc_id_fetch, query, hsdirs);
+  } else if (!tor_digest_is_zero(rend_data->desc_id_fetch)) {
+    ret = fetch_v2_desc_by_descid(rend_data->desc_id_fetch, query,
+                                  hsdirs);
   } else {
     /* Query data is invalid. */
     ret = -1;
@@ -907,10 +934,11 @@ void
 rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
 {
   rend_cache_entry_t *e = NULL;
+  const char *onion_address = rend_data_get_address(rend_query);
 
   tor_assert(rend_query);
   /* Before fetching, check if we already have a usable descriptor here. */
-  if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 &&
+  if (rend_cache_lookup_entry(onion_address, -1, &e) == 0 &&
       rend_client_any_intro_points_usable(e)) {
     log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we "
                       "already have a usable descriptor here. Not fetching.");
@@ -923,7 +951,7 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query)
     return;
   }
   log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s",
-            safe_str_client(rend_query->onion_address));
+            safe_str_client(onion_address));
 
   rend_client_fetch_v2_desc(rend_query, NULL);
   /* We don't need to look the error code because either on failure or
@@ -959,7 +987,7 @@ rend_client_cancel_descriptor_fetches(void)
       } else {
         log_debug(LD_REND, "Marking for close dir conn fetching "
                   "rendezvous descriptor for service %s",
-                  safe_str(rd->onion_address));
+                  safe_str(rend_data_get_address(rd)));
       }
       connection_mark_for_close(conn);
     }
@@ -989,25 +1017,26 @@ rend_client_cancel_descriptor_fetches(void)
  */
 int
 rend_client_report_intro_point_failure(extend_info_t *failed_intro,
-                                       rend_data_t *rend_query,
+                                       rend_data_t *rend_data,
                                        unsigned int failure_type)
 {
   int i, r;
   rend_cache_entry_t *ent;
   connection_t *conn;
+  const char *onion_address = rend_data_get_address(rend_data);
 
-  r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent);
+  r = rend_cache_lookup_entry(onion_address, -1, &ent);
   if (r < 0) {
     /* Either invalid onion address or cache entry not found. */
     switch (-r) {
     case EINVAL:
       log_warn(LD_BUG, "Malformed service ID %s.",
-          escaped_safe_str_client(rend_query->onion_address));
+               escaped_safe_str_client(onion_address));
       return -1;
     case ENOENT:
       log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.",
-          escaped_safe_str_client(rend_query->onion_address));
-      rend_client_refetch_v2_renddesc(rend_query);
+               escaped_safe_str_client(onion_address));
+      rend_client_refetch_v2_renddesc(rend_data);
       return 0;
     default:
       log_warn(LD_BUG, "Unknown cache lookup returned code: %d", r);
@@ -1031,7 +1060,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
       case INTRO_POINT_FAILURE_GENERIC:
         rend_cache_intro_failure_note(failure_type,
                                       (uint8_t *)failed_intro->identity_digest,
-                                      rend_query->onion_address);
+                                      onion_address);
         rend_intro_point_free(intro);
         smartlist_del(ent->parsed->intro_nodes, i);
         break;
@@ -1049,8 +1078,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
           if (zap_intro_point) {
             rend_cache_intro_failure_note(
                 failure_type,
-                (uint8_t *) failed_intro->identity_digest,
-                rend_query->onion_address);
+                (uint8_t *) failed_intro->identity_digest, onion_address);
             rend_intro_point_free(intro);
             smartlist_del(ent->parsed->intro_nodes, i);
           }
@@ -1064,14 +1092,14 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
   if (! rend_client_any_intro_points_usable(ent)) {
     log_info(LD_REND,
              "No more intro points remain for %s. Re-fetching descriptor.",
-             escaped_safe_str_client(rend_query->onion_address));
-    rend_client_refetch_v2_renddesc(rend_query);
+             escaped_safe_str_client(onion_address));
+    rend_client_refetch_v2_renddesc(rend_data);
 
     /* move all pending streams back to renddesc_wait */
     /* NOTE: We can now do this faster, if we use pending_entry_connections */
     while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP,
                                    AP_CONN_STATE_CIRCUIT_WAIT,
-                                   rend_query->onion_address))) {
+                                   onion_address))) {
       connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn));
       conn->state = AP_CONN_STATE_RENDDESC_WAIT;
     }
@@ -1080,7 +1108,7 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro,
   }
   log_info(LD_REND,"%d options left for %s.",
            smartlist_len(ent->parsed->intro_nodes),
-           escaped_safe_str_client(rend_query->onion_address));
+           escaped_safe_str_client(onion_address));
   return 1;
 }
 
@@ -1221,10 +1249,11 @@ rend_client_desc_trynow(const char *query)
     rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data;
     if (!rend_data)
       continue;
-    if (rend_cmp_service_ids(query, rend_data->onion_address))
+    const char *onion_address = rend_data_get_address(rend_data);
+    if (rend_cmp_service_ids(query, onion_address))
       continue;
     assert_connection_ok(base_conn, now);
-    if (rend_cache_lookup_entry(rend_data->onion_address, -1,
+    if (rend_cache_lookup_entry(onion_address, -1,
                                 &entry) == 0 &&
         rend_client_any_intro_points_usable(entry)) {
       /* either this fetch worked, or it failed but there was a
@@ -1259,11 +1288,12 @@ rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
 {
   unsigned int have_onion = 0;
   rend_cache_entry_t *cache_entry = NULL;
+  const char *onion_address = rend_data_get_address(rend_data);
+  rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
 
-  if (*rend_data->onion_address != '\0') {
+  if (onion_address[0] != '\0') {
     /* Ignore return value; we find an entry, or we don't. */
-    (void) rend_cache_lookup_entry(rend_data->onion_address, -1,
-                                   &cache_entry);
+    (void) rend_cache_lookup_entry(onion_address, -1, &cache_entry);
     have_onion = 1;
   }
 
@@ -1277,17 +1307,17 @@ rend_client_note_connection_attempt_ended(const rend_data_t *rend_data)
   /* Remove the HS's entries in last_hid_serv_requests. */
   if (have_onion) {
     unsigned int replica;
-    for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id);
+    for (replica = 0; replica < ARRAY_LENGTH(rend_data_v2->descriptor_id);
          replica++) {
-      const char *desc_id = rend_data->descriptor_id[replica];
+      const char *desc_id = rend_data_v2->descriptor_id[replica];
       purge_hid_serv_from_last_hid_serv_requests(desc_id);
     }
     log_info(LD_REND, "Connection attempt for %s has ended; "
              "cleaning up temporary state.",
-             safe_str_client(rend_data->onion_address));
+             safe_str_client(onion_address));
   } else {
     /* We only have an ID for a fetch. Probably used by HSFETCH. */
-    purge_hid_serv_from_last_hid_serv_requests(rend_data->desc_id_fetch);
+    purge_hid_serv_from_last_hid_serv_requests(rend_data_v2->desc_id_fetch);
   }
 }
 
@@ -1301,12 +1331,13 @@ rend_client_get_random_intro(const rend_data_t *rend_query)
   int ret;
   extend_info_t *result;
   rend_cache_entry_t *entry;
+  const char *onion_address = rend_data_get_address(rend_query);
 
-  ret = rend_cache_lookup_entry(rend_query->onion_address, -1, &entry);
+  ret = rend_cache_lookup_entry(onion_address, -1, &entry);
   if (ret < 0 || !rend_client_any_intro_points_usable(entry)) {
     log_warn(LD_REND,
              "Query '%s' didn't have valid rend desc in cache. Failing.",
-             safe_str_client(rend_query->onion_address));
+             safe_str_client(onion_address));
     /* XXX: Should we refetch the descriptor here if the IPs are not usable
      * anymore ?. */
     return NULL;
diff --git a/src/or/rendclient.h b/src/or/rendclient.h
index b8f8c2f871..ff0f4084fd 100644
--- a/src/or/rendclient.h
+++ b/src/or/rendclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,7 +27,7 @@ void rend_client_cancel_descriptor_fetches(void);
 void rend_client_purge_last_hid_serv_requests(void);
 
 int rend_client_report_intro_point_failure(extend_info_t *failed_intro,
-                                           rend_data_t *rend_query,
+                                           rend_data_t *rend_data,
                                            unsigned int failure_type);
 
 int rend_client_rendezvous_acked(origin_circuit_t *circ,
diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c
index d9d39b1f19..d18356e67a 100644
--- a/src/or/rendcommon.c
+++ b/src/or/rendcommon.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,9 +12,11 @@
 #include "circuitbuild.h"
 #include "config.h"
 #include "control.h"
+#include "hs_common.h"
 #include "rendclient.h"
 #include "rendcommon.h"
 #include "rendmid.h"
+#include "hs_intropoint.h"
 #include "rendservice.h"
 #include "rephist.h"
 #include "router.h"
@@ -475,7 +477,10 @@ rend_encode_v2_descriptors(smartlist_t *descs_out,
     tor_assert(descriptor_cookie);
   }
   /* Obtain service_id from public key. */
-  crypto_pk_get_digest(service_key, service_id);
+  if (crypto_pk_get_digest(service_key, service_id) < 0) {
+    log_warn(LD_BUG, "Couldn't compute service key digest.");
+    return -1;
+  }
   /* Calculate current time-period. */
   time_period = get_time_period(now, period, service_id);
   /* Determine how many seconds the descriptor will be valid. */
@@ -761,7 +766,7 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = rend_mid_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ,payload,length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
@@ -769,7 +774,7 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = rend_mid_introduce(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ,payload,length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
@@ -804,124 +809,6 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
              command);
 }
 
-/** Allocate and return a new rend_data_t with the same
- * contents as <b>query</b>. */
-rend_data_t *
-rend_data_dup(const rend_data_t *data)
-{
-  rend_data_t *data_dup;
-  tor_assert(data);
-  data_dup = tor_memdup(data, sizeof(rend_data_t));
-  data_dup->hsdirs_fp = smartlist_new();
-  SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp,
-                    smartlist_add(data_dup->hsdirs_fp,
-                                  tor_memdup(fp, DIGEST_LEN)));
-  return data_dup;
-}
-
-/** Compute descriptor ID for each replicas and save them. A valid onion
- * address must be present in the <b>rend_data</b>.
- *
- * Return 0 on success else -1. */
-static int
-compute_desc_id(rend_data_t *rend_data)
-{
-  int ret = 0;
-  unsigned replica;
-  time_t now = time(NULL);
-
-  tor_assert(rend_data);
-
-  /* Compute descriptor ID for each replicas. */
-  for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id);
-       replica++) {
-    ret = rend_compute_v2_desc_id(rend_data->descriptor_id[replica],
-                                  rend_data->onion_address,
-                                  rend_data->descriptor_cookie,
-                                  now, replica);
-    if (ret < 0) {
-      goto end;
-    }
-  }
-
- end:
-  return ret;
-}
-
-/** Allocate and initialize a rend_data_t object for a service using the
- * given arguments. Only the <b>onion_address</b> is not optional.
- *
- * Return a valid rend_data_t pointer. */
-rend_data_t *
-rend_data_service_create(const char *onion_address, const char *pk_digest,
-                         const uint8_t *cookie, rend_auth_type_t auth_type)
-{
-  rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data));
-
-  /* We need at least one else the call is wrong. */
-  tor_assert(onion_address != NULL);
-
-  if (pk_digest) {
-    memcpy(rend_data->rend_pk_digest, pk_digest,
-           sizeof(rend_data->rend_pk_digest));
-  }
-  if (cookie) {
-    memcpy(rend_data->rend_cookie, cookie,
-           sizeof(rend_data->rend_cookie));
-  }
-
-  strlcpy(rend_data->onion_address, onion_address,
-          sizeof(rend_data->onion_address));
-  rend_data->auth_type = auth_type;
-  /* Won't be used but still need to initialize it for rend_data dup and
-   * free. */
-  rend_data->hsdirs_fp = smartlist_new();
-
-  return rend_data;
-}
-
-/** Allocate and initialize a rend_data_t object for a client request using
- * the given arguments.  Either an onion address or a descriptor ID is
- * needed. Both can be given but only the onion address will be used to make
- * the descriptor fetch.
- *
- * Return a valid rend_data_t pointer or NULL on error meaning the
- * descriptor IDs couldn't be computed from the given data. */
-rend_data_t *
-rend_data_client_create(const char *onion_address, const char *desc_id,
-                        const char *cookie, rend_auth_type_t auth_type)
-{
-  rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data));
-
-  /* We need at least one else the call is wrong. */
-  tor_assert(onion_address != NULL || desc_id != NULL);
-
-  if (cookie) {
-    memcpy(rend_data->descriptor_cookie, cookie,
-           sizeof(rend_data->descriptor_cookie));
-  }
-  if (desc_id) {
-    memcpy(rend_data->desc_id_fetch, desc_id,
-           sizeof(rend_data->desc_id_fetch));
-  }
-  if (onion_address) {
-    strlcpy(rend_data->onion_address, onion_address,
-            sizeof(rend_data->onion_address));
-    if (compute_desc_id(rend_data) < 0) {
-      goto error;
-    }
-  }
-
-  rend_data->auth_type = auth_type;
-  rend_data->hsdirs_fp = smartlist_new();
-
-  return rend_data;
-
- error:
-  rend_data_free(rend_data);
-  return NULL;
-}
-
 /** Determine the routers that are responsible for <b>id</b> (binary) and
  * add pointers to those routers' routerstatus_t to <b>responsible_dirs</b>.
  * Return -1 if we're returning an empty smartlist, else return 0.
@@ -1116,3 +1003,32 @@ assert_circ_anonymity_ok(origin_circuit_t *circ,
   }
 }
 
+/* Return 1 iff the given <b>digest</b> of a permenanent hidden service key is
+ * equal to the digest in the origin circuit <b>ocirc</b> of its rend data .
+ * If the rend data doesn't exist, 0 is returned. This function is agnostic to
+ * the rend data version. */
+int
+rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                          const uint8_t *digest)
+{
+  size_t rend_pk_digest_len;
+  const uint8_t *rend_pk_digest;
+
+  tor_assert(ocirc);
+  tor_assert(digest);
+
+  if (ocirc->rend_data == NULL) {
+    goto no_match;
+  }
+
+  rend_pk_digest = rend_data_get_pk_digest(ocirc->rend_data,
+                                           &rend_pk_digest_len);
+  if (tor_memeq(rend_pk_digest, digest, rend_pk_digest_len)) {
+    goto match;
+  }
+ no_match:
+  return 0;
+ match:
+  return 1;
+}
+
diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h
index 090e6f25e0..94c2480d86 100644
--- a/src/or/rendcommon.h
+++ b/src/or/rendcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,19 +18,6 @@ typedef enum rend_intro_point_failure_t {
   INTRO_POINT_FAILURE_UNREACHABLE = 2,
 } rend_intro_point_failure_t;
 
-/** Free all storage associated with <b>data</b> */
-static inline void
-rend_data_free(rend_data_t *data)
-{
-  if (!data) {
-    return;
-  }
-  /* Cleanup the HSDir identity digest. */
-  SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d));
-  smartlist_free(data->hsdirs_fp);
-  tor_free(data);
-}
-
 int rend_cmp_service_ids(const char *one, const char *two);
 
 void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
@@ -60,15 +47,8 @@ void rend_get_descriptor_id_bytes(char *descriptor_id_out,
 int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs,
                                          const char *id);
 
-rend_data_t *rend_data_dup(const rend_data_t *data);
-rend_data_t *rend_data_client_create(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *cookie,
-                                     rend_auth_type_t auth_type);
-rend_data_t *rend_data_service_create(const char *onion_address,
-                                      const char *pk_digest,
-                                      const uint8_t *cookie,
-                                      rend_auth_type_t auth_type);
+int rend_circuit_pk_digest_eq(const origin_circuit_t *ocirc,
+                              const uint8_t *digest);
 
 char *rend_auth_encode_cookie(const uint8_t *cookie_in,
                               rend_auth_type_t auth_type);
diff --git a/src/or/rendmid.c b/src/or/rendmid.c
index 441d5043ce..89739e1291 100644
--- a/src/or/rendmid.c
+++ b/src/or/rendmid.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,17 +12,20 @@
 #include "circuitlist.h"
 #include "circuituse.h"
 #include "config.h"
+#include "crypto.h"
 #include "dos.h"
 #include "relay.h"
 #include "rendmid.h"
 #include "rephist.h"
+#include "hs_circuitmap.h"
+#include "hs_intropoint.h"
 
 /** Respond to an ESTABLISH_INTRO cell by checking the signed data and
  * setting the circuit's purpose and service pk digest.
  */
 int
-rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
-                         size_t request_len)
+rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                size_t request_len)
 {
   crypto_pk_t *pk = NULL;
   char buf[DIGEST_LEN+9];
@@ -34,15 +37,14 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   int reason = END_CIRC_REASON_INTERNAL;
 
   log_info(LD_REND,
-           "Received an ESTABLISH_INTRO request on circuit %u",
+           "Received a legacy ESTABLISH_INTRO request on circuit %u",
            (unsigned) circ->p_circ_id);
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-         "Rejecting ESTABLISH_INTRO on non-OR or non-edge circuit.");
+  if (!hs_intro_circuit_is_suitable_for_establish_intro(circ)) {
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
   }
+
   if (request_len < 2+DIGEST_LEN)
     goto truncated;
   /* First 2 bytes: length of asn1-encoded key. */
@@ -96,7 +98,8 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
 
   /* Close any other intro circuits with the same pk. */
   c = NULL;
-  while ((c = circuit_get_intro_point((const uint8_t *)pk_digest))) {
+  while ((c = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                (const uint8_t *)pk_digest))) {
     log_info(LD_REND, "Replacing old circuit for service %s",
              safe_str(serviceid));
     circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED);
@@ -104,16 +107,14 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   }
 
   /* Acknowledge the request. */
-  if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
-                                   RELAY_COMMAND_INTRO_ESTABLISHED,
-                                   "", 0, NULL)<0) {
+  if (hs_intro_send_intro_established_cell(circ) < 0) {
     log_info(LD_GENERAL, "Couldn't send INTRO_ESTABLISHED cell.");
-    goto err;
+    goto err_no_close;
   }
 
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
-  circuit_set_intro_point_digest(circ, (uint8_t *)pk_digest);
+  hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -124,8 +125,9 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
   log_warn(LD_PROTOCOL, "Rejecting truncated ESTABLISH_INTRO cell.");
   reason = END_CIRC_REASON_TORPROTOCOL;
  err:
-  if (pk) crypto_pk_free(pk);
   circuit_mark_for_close(TO_CIRCUIT(circ), reason);
+ err_no_close:
+  if (pk) crypto_pk_free(pk);
   return -1;
 }
 
@@ -134,8 +136,8 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
  * INTRODUCE2 cell.
  */
 int
-rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
-                   size_t request_len)
+rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                          size_t request_len)
 {
   or_circuit_t *intro_circ;
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
@@ -144,26 +146,10 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
   log_info(LD_REND, "Received an INTRODUCE1 request on circuit %u",
            (unsigned)circ->p_circ_id);
 
-  if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_warn(LD_PROTOCOL,
-             "Rejecting INTRODUCE1 on non-OR or non-edge circuit %u.",
-             (unsigned)circ->p_circ_id);
-    goto err;
-  }
-
-  /* We have already done an introduction on this circuit but we just
-     received a request for another one. We block it since this might
-     be an attempt to DoS a hidden service (#15515). */
-  if (circ->already_received_introduce1) {
-    log_fn(LOG_PROTOCOL_WARN, LD_REND,
-           "Blocking multiple introductions on the same circuit. "
-           "Someone might be trying to attack a hidden service through "
-           "this relay.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL);
-    return -1;
-  }
-
-  circ->already_received_introduce1 = 1;
+  /* At this point, we know that the circuit is valid for an INTRODUCE1
+   * because the validation has been made before calling this function. */
+  tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_OR);
+  tor_assert(!circ->base_.n_chan);
 
   /* We could change this to MAX_HEX_NICKNAME_LEN now that 0.0.9.x is
    * obsolete; however, there isn't much reason to do so, and we're going
@@ -182,7 +168,8 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
 
   /* The first 20 bytes are all we look at: they have a hash of the service's
    * PK. */
-  intro_circ = circuit_get_intro_point((const uint8_t*)request);
+  intro_circ = hs_circuitmap_get_intro_circ_v2_relay_side(
+                                                      (const uint8_t*)request);
   if (!intro_circ) {
     log_info(LD_REND,
              "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; "
@@ -203,14 +190,15 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
                                    (char*)request, request_len, NULL)) {
     log_warn(LD_GENERAL,
              "Unable to send INTRODUCE2 cell to Tor client.");
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
   /* And send an ack down the client's circuit.  Empty body means succeeded. */
   if (relay_send_command_from_edge(0,TO_CIRCUIT(circ),
                                    RELAY_COMMAND_INTRODUCE_ACK,
                                    NULL,0,NULL)) {
     log_warn(LD_GENERAL, "Unable to send INTRODUCE_ACK cell to Tor client.");
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+    /* Stop right now, the circuit has been closed. */
     return -1;
   }
 
@@ -222,8 +210,6 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
                                    RELAY_COMMAND_INTRODUCE_ACK,
                                    nak_body, 1, NULL)) {
     log_warn(LD_GENERAL, "Unable to send NAK to Tor client.");
-    /* Is this right? */
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
   }
   return -1;
 }
@@ -270,7 +256,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  if (circuit_get_rendezvous(request)) {
+  if (hs_circuitmap_get_rend_circ_relay_side(request)) {
     log_warn(LD_PROTOCOL,
              "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
     goto err;
@@ -281,12 +267,12 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
                                    RELAY_COMMAND_RENDEZVOUS_ESTABLISHED,
                                    "", 0, NULL)<0) {
     log_warn(LD_PROTOCOL, "Couldn't send RENDEZVOUS_ESTABLISHED cell.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
 
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING);
-  circuit_set_rendezvous_cookie(circ, request);
+  hs_circuitmap_register_rend_circ_relay_side(circ, request);
 
   base16_encode(hexid,9,(char*)request,4);
 
@@ -335,7 +321,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
            "Got request for rendezvous from circuit %u to cookie %s.",
            (unsigned)circ->p_circ_id, hexid);
 
-  rend_circ = circuit_get_rendezvous(request);
+  rend_circ = hs_circuitmap_get_rend_circ_relay_side(request);
   if (!rend_circ) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.",
@@ -358,7 +344,8 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
     log_warn(LD_GENERAL,
              "Unable to send RENDEZVOUS2 cell to client on circuit %u.",
              (unsigned)rend_circ->p_circ_id);
-    goto err;
+    /* Stop right now, the circuit has been closed. */
+    return -1;
   }
 
   /* Join the circuits. */
@@ -369,7 +356,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED);
   circuit_change_purpose(TO_CIRCUIT(rend_circ),
                          CIRCUIT_PURPOSE_REND_ESTABLISHED);
-  circuit_set_rendezvous_cookie(circ, NULL);
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
 
   rend_circ->rend_splice = circ;
   circ->rend_splice = rend_circ;
diff --git a/src/or/rendmid.h b/src/or/rendmid.h
index 10d1287085..daf9e2885e 100644
--- a/src/or/rendmid.h
+++ b/src/or/rendmid.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,10 +12,10 @@
 #ifndef TOR_RENDMID_H
 #define TOR_RENDMID_H
 
-int rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request,
-                             size_t request_len);
-int rend_mid_introduce(or_circuit_t *circ, const uint8_t *request,
-                       size_t request_len);
+int rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
+                                    size_t request_len);
+int rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
+                              size_t request_len);
 int rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
                                   size_t request_len);
 int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
diff --git a/src/or/rendservice.c b/src/or/rendservice.c
index da200d1381..2a3594918e 100644
--- a/src/or/rendservice.c
+++ b/src/or/rendservice.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "config.h"
 #include "control.h"
 #include "directory.h"
+#include "hs_common.h"
 #include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -75,9 +76,11 @@ static ssize_t rend_service_parse_intro_for_v3(
 static int rend_service_check_private_dir(const or_options_t *options,
                                           const rend_service_t *s,
                                           int create);
-static int rend_service_check_private_dir_impl(const or_options_t *options,
-                                               const rend_service_t *s,
-                                               int create);
+static const smartlist_t* rend_get_service_list(
+                                  const smartlist_t* substitute_service_list);
+static smartlist_t* rend_get_service_list_mutable(
+                                  smartlist_t* substitute_service_list);
+static int rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted);
 
 /** Represents the mapping from a virtual port of a rendezvous service to
  * a real port on some IP.
@@ -95,39 +98,6 @@ struct rend_service_port_config_s {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 };
 
-/** Try to maintain this many intro points per service by default. */
-#define NUM_INTRO_POINTS_DEFAULT 3
-/** Maximum number of intro points per service. */
-#define NUM_INTRO_POINTS_MAX 10
-/** Number of extra intro points we launch if our set of intro nodes is
- * empty. See proposal 155, section 4. */
-#define NUM_INTRO_POINTS_EXTRA 2
-
-/** If we can't build our intro circuits, don't retry for this long. */
-#define INTRO_CIRC_RETRY_PERIOD (60*5)
-/** Don't try to build more than this many circuits before giving up
- * for a while.*/
-#define MAX_INTRO_CIRCS_PER_PERIOD 10
-/** How many seconds should we spend trying to connect to a requested
- * rendezvous point before giving up? */
-#define MAX_REND_TIMEOUT 30
-/* Default, minimum and maximum values for the maximum rendezvous failures
- * consensus parameter. */
-#define MAX_REND_FAILURES_DEFAULT 2
-#define MAX_REND_FAILURES_MIN 1
-#define MAX_REND_FAILURES_MAX 10
-
-/** How many times will a hidden service operator attempt to connect to
- * a requested rendezvous point before giving up? */
-static int
-get_max_rend_failures(void)
-{
-  return networkstatus_get_param(NULL, "hs_service_max_rdv_failures",
-                                 MAX_REND_FAILURES_DEFAULT,
-                                 MAX_REND_FAILURES_MIN,
-                                 MAX_REND_FAILURES_MAX);
-}
-
 /* Hidden service directory file names:
  * new file names should be added to rend_service_add_filenames_to_list()
  * for sandboxing purposes. */
@@ -136,18 +106,61 @@ static const char *hostname_fname = "hostname";
 static const char *client_keys_fname = "client_keys";
 static const char *sos_poison_fname = "onion_service_non_anonymous";
 
+/** A list of rend_service_t's for services run on this OP. */
+static smartlist_t *rend_service_list = NULL;
+/** A list of rend_service_t's for services run on this OP which is used as a
+ * staging area before they are put in the main list in order to prune dying
+ * service on config reload. */
+static smartlist_t *rend_service_staging_list = NULL;
+
+/* Like rend_get_service_list_mutable, but returns a read-only list. */
+static const smartlist_t*
+rend_get_service_list(const smartlist_t* substitute_service_list)
+{
+  /* It is safe to cast away the const here, because
+   * rend_get_service_list_mutable does not actually modify the list */
+  return rend_get_service_list_mutable((smartlist_t*)substitute_service_list);
+}
+
+/* Return a mutable list of hidden services.
+ * If substitute_service_list is not NULL, return it.
+ * Otherwise, check if the global rend_service_list is non-NULL, and if so,
+ * return it.
+ * Otherwise, log a BUG message and return NULL.
+ * */
+static smartlist_t*
+rend_get_service_list_mutable(smartlist_t* substitute_service_list)
+{
+  if (substitute_service_list) {
+    return substitute_service_list;
+  }
+
+  /* If no special service list is provided, then just use the global one. */
+
+  if (BUG(!rend_service_list)) {
+    /* No global HS list, which is a programmer error. */
+    return NULL;
+  }
+
+  return rend_service_list;
+}
+
+/** Tells if onion service <b>s</b> is ephemeral.
+ */
+static unsigned int
+rend_service_is_ephemeral(const struct rend_service_t *s)
+{
+  return (s->directory == NULL);
+}
+
 /** Returns a escaped string representation of the service, <b>s</b>.
  */
 static const char *
 rend_service_escaped_dir(const struct rend_service_t *s)
 {
-  return (s->directory) ? escaped(s->directory) : "[EPHEMERAL]";
+  return rend_service_is_ephemeral(s) ? "[EPHEMERAL]" : escaped(s->directory);
 }
 
-/** A list of rend_service_t's for services run on this OP.
- */
-static smartlist_t *rend_service_list = NULL;
-
 /** Return the number of rendezvous services we have configured. */
 int
 num_rend_services(void)
@@ -232,123 +245,131 @@ rend_service_free_all(void)
   rend_service_list = NULL;
 }
 
-/** Validate <b>service</b> and add it to <b>service_list</b>, or to
- * the global rend_service_list if <b>service_list</b> is NULL.
- * Return 0 on success.  On failure, free <b>service</b> and return -1.
- * Takes ownership of <b>service</b>.
- */
+/* Validate a <b>service</b>. Use the <b>service_list</b> to make sure there
+ * is no duplicate entry for the given service object. Return 0 if valid else
+ * -1 if not.*/
 static int
-rend_add_service(smartlist_t *service_list, rend_service_t *service)
+rend_validate_service(const smartlist_t *service_list,
+                      const rend_service_t *service)
 {
-  int i;
-  rend_service_port_config_t *p;
-
-  smartlist_t *s_list;
-  /* If no special service list is provided, then just use the global one. */
-  if (!service_list) {
-    if (BUG(!rend_service_list)) {
-      /* No global HS list, which is a failure. */
-      return -1;
-    }
+  int dupe = 0;
 
-    s_list = rend_service_list;
-  } else {
-    s_list = service_list;
-  }
-
-  service->intro_nodes = smartlist_new();
-  service->expiring_nodes = smartlist_new();
+  tor_assert(service_list);
+  tor_assert(service);
 
   if (service->max_streams_per_circuit < 0) {
     log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max "
-                        "streams per circuit; ignoring.",
+                        "streams per circuit.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (service->max_streams_close_circuit < 0 ||
       service->max_streams_close_circuit > 1) {
     log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid "
-                        "max streams handling; ignoring.",
+                        "max streams handling.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (service->auth_type != REND_NO_AUTH &&
-      (!service->clients ||
-       smartlist_len(service->clients) == 0)) {
-    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but no "
-                        "clients; ignoring.",
+      (!service->clients || smartlist_len(service->clients) == 0)) {
+    log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but "
+                        "no clients.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
+    goto invalid;
   }
 
   if (!service->ports || !smartlist_len(service->ports)) {
-    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured; "
-             "ignoring.",
+    log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured.",
              rend_service_escaped_dir(service));
-    rend_service_free(service);
-    return -1;
-  } else {
-    int dupe = 0;
-    /* XXX This duplicate check has two problems:
-     *
-     * a) It's O(n^2), but the same comment from the bottom of
-     *    rend_config_services() should apply.
-     *
-     * b) We only compare directory paths as strings, so we can't
-     *    detect two distinct paths that specify the same directory
-     *    (which can arise from symlinks, case-insensitivity, bind
-     *    mounts, etc.).
-     *
-     * It also can't detect that two separate Tor instances are trying
-     * to use the same HiddenServiceDir; for that, we would need a
-     * lock file.  But this is enough to detect a simple mistake that
-     * at least one person has actually made.
-     */
-    if (service->directory != NULL) {
-      /* Skip dupe for ephemeral services. */
-      SMARTLIST_FOREACH(s_list, rend_service_t*, ptr,
-                        dupe = dupe ||
-                               !strcmp(ptr->directory, service->directory));
-      if (dupe) {
-        log_warn(LD_REND, "Another hidden service is already configured for "
-                 "directory %s, ignoring.",
-                 rend_service_escaped_dir(service));
-        rend_service_free(service);
-        return -1;
-      }
+    goto invalid;
+  }
+
+  /* XXX This duplicate check has two problems:
+   *
+   * a) It's O(n^2), but the same comment from the bottom of
+   *    rend_config_services() should apply.
+   *
+   * b) We only compare directory paths as strings, so we can't
+   *    detect two distinct paths that specify the same directory
+   *    (which can arise from symlinks, case-insensitivity, bind
+   *    mounts, etc.).
+   *
+   * It also can't detect that two separate Tor instances are trying
+   * to use the same HiddenServiceDir; for that, we would need a
+   * lock file.  But this is enough to detect a simple mistake that
+   * at least one person has actually made.
+   */
+  if (!rend_service_is_ephemeral(service)) {
+    /* Skip dupe for ephemeral services. */
+    SMARTLIST_FOREACH(service_list, rend_service_t *, ptr,
+                      dupe = dupe ||
+                      !strcmp(ptr->directory, service->directory));
+    if (dupe) {
+      log_warn(LD_REND, "Another hidden service is already configured for "
+                        "directory %s.",
+               rend_service_escaped_dir(service));
+      goto invalid;
     }
-    smartlist_add(s_list, service);
-    log_debug(LD_REND,"Configuring service with directory \"%s\"",
-              service->directory);
-    for (i = 0; i < smartlist_len(service->ports); ++i) {
-      p = smartlist_get(service->ports, i);
-      if (!(p->is_unix_addr)) {
-        log_debug(LD_REND,
-                  "Service maps port %d to %s",
-                  p->virtual_port,
-                  fmt_addrport(&p->real_addr, p->real_port));
-      } else {
+  }
+
+  /* Valid. */
+  return 0;
+ invalid:
+  return -1;
+}
+
+/** Add it to <b>service_list</b>, or to the global rend_service_list if
+ * <b>service_list</b> is NULL.  Return 0 on success.  On failure, free
+ * <b>service</b> and return -1. Takes ownership of <b>service</b>. */
+static int
+rend_add_service(smartlist_t *service_list, rend_service_t *service)
+{
+  int i;
+  rend_service_port_config_t *p;
+
+  tor_assert(service);
+
+  smartlist_t *s_list = rend_get_service_list_mutable(service_list);
+  /* We must have a service list, even if it's a temporary one, so we can
+   * check for duplicate services */
+  if (BUG(!s_list)) {
+    return -1;
+  }
+
+  service->intro_nodes = smartlist_new();
+  service->expiring_nodes = smartlist_new();
+
+  log_debug(LD_REND,"Configuring service with directory %s",
+            rend_service_escaped_dir(service));
+  for (i = 0; i < smartlist_len(service->ports); ++i) {
+    p = smartlist_get(service->ports, i);
+    if (!(p->is_unix_addr)) {
+      log_debug(LD_REND,
+                "Service maps port %d to %s",
+                p->virtual_port,
+                fmt_addrport(&p->real_addr, p->real_port));
+    } else {
 #ifdef HAVE_SYS_UN_H
-        log_debug(LD_REND,
-                  "Service maps port %d to socket at \"%s\"",
-                  p->virtual_port, p->unix_addr);
+      log_debug(LD_REND,
+                "Service maps port %d to socket at \"%s\"",
+                p->virtual_port, p->unix_addr);
 #else
-        log_debug(LD_REND,
-                  "Service maps port %d to an AF_UNIX socket, but we "
-                  "have no AF_UNIX support on this platform.  This is "
-                  "probably a bug.",
-                  p->virtual_port);
+      log_warn(LD_BUG,
+               "Service maps port %d to an AF_UNIX socket, but we "
+               "have no AF_UNIX support on this platform.  This is "
+               "probably a bug.",
+               p->virtual_port);
+      rend_service_free(service);
+      return -1;
 #endif /* defined(HAVE_SYS_UN_H) */
-      }
     }
-    return 0;
   }
-  /* NOTREACHED */
+  /* The service passed all the checks */
+  tor_assert(s_list);
+  smartlist_add(s_list, service);
+  return 0;
 }
 
 /** Return a new rend_service_port_config_t with its path set to
@@ -367,9 +388,9 @@ rend_service_port_config_new(const char *socket_path)
   return conf;
 }
 
-/** Parses a real-port to virtual-port mapping separated by the provided
- * separator and returns a new rend_service_port_config_t, or NULL and an
- * optional error string on failure.
+/** Parses a virtual-port to real-port/socket mapping separated by
+ * the provided separator and returns a new rend_service_port_config_t,
+ * or NULL and an optional error string on failure.
  *
  * The format is: VirtualPort SEP (IP|RealPort|IP:RealPort|'socket':path)?
  *
@@ -394,14 +415,12 @@ rend_service_parse_port_config(const char *string, const char *sep,
   smartlist_split_string(sl, string, sep,
                          SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
   if (smartlist_len(sl) < 1 || BUG(smartlist_len(sl) > 2)) {
-    if (err_msg_out)
-      err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
+    err_msg = tor_strdup("Bad syntax in hidden service port configuration.");
     goto err;
   }
   virtport = (int)tor_parse_long(smartlist_get(sl,0), 10, 1, 65535, NULL,NULL);
   if (!virtport) {
-    if (err_msg_out)
-      tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
+    tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service "
                    "port configuration", escaped(smartlist_get(sl,0)));
 
     goto err;
@@ -429,10 +448,8 @@ rend_service_parse_port_config(const char *string, const char *sep,
     } else if (strchr(addrport, ':') || strchr(addrport, '.')) {
       /* else try it as an IP:port pair if it has a : or . in it */
       if (tor_addr_port_lookup(addrport, &addr, &p)<0) {
-        if (err_msg_out)
-          err_msg = tor_strdup("Unparseable address in hidden service port "
-                               "configuration.");
-
+        err_msg = tor_strdup("Unparseable address in hidden service port "
+                             "configuration.");
         goto err;
       }
       realport = p?p:virtport;
@@ -440,11 +457,9 @@ rend_service_parse_port_config(const char *string, const char *sep,
       /* No addr:port, no addr -- must be port. */
       realport = (int)tor_parse_long(addrport, 10, 1, 65535, NULL, NULL);
       if (!realport) {
-        if (err_msg_out)
-          tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
-                       "hidden service port configuration.",
-                       escaped(addrport));
-
+        tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in "
+                     "hidden service port configuration.",
+                     escaped(addrport));
         goto err;
       }
       tor_addr_from_ipv4h(&addr, 0x7F000001u); /* Default to 127.0.0.1 */
@@ -463,7 +478,11 @@ rend_service_parse_port_config(const char *string, const char *sep,
 
  err:
   tor_free(addrport);
-  if (err_msg_out) *err_msg_out = err_msg;
+  if (err_msg_out != NULL) {
+    *err_msg_out = err_msg;
+  } else {
+    tor_free(err_msg);
+  }
   SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
   smartlist_free(sl);
 
@@ -503,32 +522,138 @@ rend_service_check_dir_and_add(smartlist_t *service_list,
     return -1;
   }
 
-  if (validate_only) {
-    rend_service_free(service);
-    return 0;
-  } else {
-    /* Use service_list for unit tests */
-    smartlist_t *s_list = NULL;
-    /* If no special service list is provided, then just use the global one. */
-    if (!service_list) {
-      if (BUG(!rend_service_list)) {
-        /* No global HS list, which is a failure, because we plan on adding to
-         * it */
-        return -1;
-      }
-      s_list = rend_service_list;
-    } else {
-      s_list = service_list;
+  smartlist_t *s_list = rend_get_service_list_mutable(service_list);
+  /* We must have a service list, even if it's a temporary one, so we can
+   * check for duplicate services */
+  if (BUG(!s_list)) {
+    return -1;
+  }
+  return rend_add_service(s_list, service);
+}
+
+/* Helper: Actual implementation of the pruning on reload which we've
+ * decoupled in order to make the unit test workeable without ugly hacks.
+ * Furthermore, this function does NOT free any memory but will nullify the
+ * temporary list pointer whatever happens. */
+STATIC void
+rend_service_prune_list_impl_(void)
+{
+  origin_circuit_t *ocirc = NULL;
+  smartlist_t *surviving_services, *old_service_list, *new_service_list;
+
+  /* When pruning our current service list, we must have a staging list that
+   * contains what we want to check else it's a code flow error. */
+  tor_assert(rend_service_staging_list);
+
+  /* We are about to prune the current list of its dead service so set the
+   * semantic for that list to be the "old" one. */
+  old_service_list = rend_service_list;
+  /* The staging list is now the "new" list so set this semantic. */
+  new_service_list = rend_service_staging_list;
+  /* After this, whatever happens, we'll use our new list. */
+  rend_service_list = new_service_list;
+  /* Finally, nullify the staging list pointer as we don't need it anymore
+   * and it needs to be NULL before the next reload. */
+  rend_service_staging_list = NULL;
+  /* Nothing to prune if we have no service list so stop right away. */
+  if (!old_service_list) {
+    return;
+  }
+
+  /* This contains all _existing_ services that survives the relaod that is
+   * that haven't been removed from the configuration. The difference between
+   * this list and the new service list is that the new list can possibly
+   * contain newly configured service that have no introduction points opened
+   * yet nor key material loaded or generated. */
+  surviving_services = smartlist_new();
+
+  /* Preserve the existing ephemeral services.
+   *
+   * This is the ephemeral service equivalent of the "Copy introduction
+   * points to new services" block, except there's no copy required since
+   * the service structure isn't regenerated.
+   *
+   * After this is done, all ephemeral services will be:
+   *  * Removed from old_service_list, so the equivalent non-ephemeral code
+   *    will not attempt to preserve them.
+   *  * Added to the new_service_list (that previously only had the
+   *    services listed in the configuration).
+   *  * Added to surviving_services, which is the list of services that
+   *    will NOT have their intro point closed.
+   */
+  SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+    if (rend_service_is_ephemeral(old)) {
+      SMARTLIST_DEL_CURRENT(old_service_list, old);
+      smartlist_add(surviving_services, old);
+      smartlist_add(new_service_list, old);
     }
-    /* s_list can not be NULL here - if both service_list and rend_service_list
-     * are NULL, and validate_only is false, we exit earlier in the function
-     */
-    if (BUG(!s_list)) {
-      return -1;
+  } SMARTLIST_FOREACH_END(old);
+
+  /* Copy introduction points to new services. This is O(n^2), but it's only
+   * called on reconfigure, so it's ok performance wise. */
+  SMARTLIST_FOREACH_BEGIN(new_service_list, rend_service_t *, new) {
+    SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
+      /* Skip ephemeral services as we only want to copy introduction points
+       * from current services to newly configured one that already exists.
+       * The same directory means it's the same service. */
+      if (rend_service_is_ephemeral(new) || rend_service_is_ephemeral(old) ||
+          strcmp(old->directory, new->directory)) {
+        continue;
+      }
+      smartlist_add_all(new->intro_nodes, old->intro_nodes);
+      smartlist_clear(old->intro_nodes);
+      smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
+      smartlist_clear(old->expiring_nodes);
+      /* This regular service will survive the closing IPs step after. */
+      smartlist_add(surviving_services, old);
+      break;
+    } SMARTLIST_FOREACH_END(old);
+  } SMARTLIST_FOREACH_END(new);
+
+  /* For every service introduction circuit we can find, see if we have a
+   * matching surviving configured service. If not, close the circuit. */
+  while ((ocirc = circuit_get_next_service_intro_circ(ocirc))) {
+    int keep_it = 0;
+    tor_assert(ocirc->rend_data);
+    SMARTLIST_FOREACH_BEGIN(surviving_services, const rend_service_t *, s) {
+      if (rend_circuit_pk_digest_eq(ocirc, (uint8_t *) s->pk_digest)) {
+        /* Keep this circuit as we have a matching configured service. */
+        keep_it = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(s);
+    if (keep_it) {
+      continue;
     }
-    /* Ignore service failures until 030 */
-    rend_add_service(s_list, service);
-    return 0;
+    log_info(LD_REND, "Closing intro point %s for service %s.",
+             safe_str_client(extend_info_describe(
+                                        ocirc->build_state->chosen_exit)),
+             safe_str_client(rend_data_get_address(ocirc->rend_data)));
+    /* Reason is FINISHED because service has been removed and thus the
+     * circuit is considered old/uneeded. */
+    circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED);
+  }
+  smartlist_free(surviving_services);
+}
+
+/* Try to prune our main service list using the temporary one that we just
+ * loaded and parsed successfully. The pruning process decides which onion
+ * services to keep and which to discard after a reload. */
+void
+rend_service_prune_list(void)
+{
+  smartlist_t *old_service_list = rend_service_list;
+  /* Don't try to prune anything if we have no staging list. */
+  if (!rend_service_staging_list) {
+    return;
+  }
+  rend_service_prune_list_impl_();
+  if (old_service_list) {
+    /* Every remaining service in the old list have been removed from the
+     * configuration so clean them up safely. */
+    SMARTLIST_FOREACH(old_service_list, rend_service_t *, s,
+                      rend_service_free(s));
+    smartlist_free(old_service_list);
   }
 }
 
@@ -543,22 +668,30 @@ rend_config_services(const or_options_t *options, int validate_only)
   config_line_t *line;
   rend_service_t *service = NULL;
   rend_service_port_config_t *portcfg;
-  smartlist_t *old_service_list = NULL;
   int ok = 0;
+  int rv = -1;
 
-  if (!validate_only) {
-    old_service_list = rend_service_list;
-    rend_service_list = smartlist_new();
+  /* Use the staging service list so that we can check then do the pruning
+   * process using the main list at the end. */
+  if (rend_service_staging_list == NULL) {
+    rend_service_staging_list = smartlist_new();
   }
 
   for (line = options->RendConfigLines; line; line = line->next) {
     if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* register the service we just finished parsing
-       * this code registers every service except the last one parsed,
-       * which is registered below the loop */
-      if (rend_service_check_dir_and_add(NULL, options, service,
-                                         validate_only) < 0) {
-          return -1;
+      if (service) {
+        /* Validate and register the service we just finished parsing this
+         * code registers every service except the last one parsed, which is
+         * validated and registered below the loop. */
+        if (rend_validate_service(rend_service_staging_list, service) < 0) {
+          goto free_and_return;
+        }
+        if (rend_service_check_dir_and_add(rend_service_staging_list, options,
+                                           service, validate_only) < 0) {
+          /* The above frees the service on error so nullify the pointer. */
+          service = NULL;
+          goto free_and_return;
+        }
       }
       service = tor_malloc_zero(sizeof(rend_service_t));
       service->directory = tor_strdup(line->value);
@@ -570,8 +703,7 @@ rend_config_services(const or_options_t *options, int validate_only)
     if (!service) {
       log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
                line->key);
-      rend_service_free(service);
-      return -1;
+      goto free_and_return;
     }
     if (!strcasecmp(line->key, "HiddenServicePort")) {
       char *err_msg = NULL;
@@ -580,8 +712,7 @@ rend_config_services(const or_options_t *options, int validate_only)
         if (err_msg)
           log_warn(LD_CONFIG, "%s", err_msg);
         tor_free(err_msg);
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       tor_assert(!err_msg);
       smartlist_add(service->ports, portcfg);
@@ -592,12 +723,12 @@ rend_config_services(const or_options_t *options, int validate_only)
         log_warn(LD_CONFIG,
                  "HiddenServiceAllowUnknownPorts should be 0 or 1, not %s",
                  line->value);
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       log_info(LD_CONFIG,
                "HiddenServiceAllowUnknownPorts=%d for %s",
-               (int)service->allow_unknown_ports, service->directory);
+               (int)service->allow_unknown_ports,
+               rend_service_escaped_dir(service));
     } else if (!strcasecmp(line->key,
                            "HiddenServiceDirGroupReadable")) {
         service->dir_group_readable = (int)tor_parse_long(line->value,
@@ -606,12 +737,12 @@ rend_config_services(const or_options_t *options, int validate_only)
             log_warn(LD_CONFIG,
                      "HiddenServiceDirGroupReadable should be 0 or 1, not %s",
                      line->value);
-            rend_service_free(service);
-            return -1;
+            goto free_and_return;
         }
         log_info(LD_CONFIG,
                  "HiddenServiceDirGroupReadable=%d for %s",
-                 service->dir_group_readable, service->directory);
+                 service->dir_group_readable,
+                 rend_service_escaped_dir(service));
     } else if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
       service->max_streams_per_circuit = (int)tor_parse_long(line->value,
                                                     10, 0, 65535, &ok, NULL);
@@ -619,12 +750,12 @@ rend_config_services(const or_options_t *options, int validate_only)
         log_warn(LD_CONFIG,
                  "HiddenServiceMaxStreams should be between 0 and %d, not %s",
                  65535, line->value);
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       log_info(LD_CONFIG,
                "HiddenServiceMaxStreams=%d for %s",
-               service->max_streams_per_circuit, service->directory);
+               service->max_streams_per_circuit,
+               rend_service_escaped_dir(service));
     } else if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
       service->max_streams_close_circuit = (int)tor_parse_long(line->value,
                                                         10, 0, 1, &ok, NULL);
@@ -633,28 +764,26 @@ rend_config_services(const or_options_t *options, int validate_only)
                  "HiddenServiceMaxStreamsCloseCircuit should be 0 or 1, "
                  "not %s",
                  line->value);
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       log_info(LD_CONFIG,
                "HiddenServiceMaxStreamsCloseCircuit=%d for %s",
-               (int)service->max_streams_close_circuit, service->directory);
+               (int)service->max_streams_close_circuit,
+               rend_service_escaped_dir(service));
     } else if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
       service->n_intro_points_wanted =
         (unsigned int) tor_parse_long(line->value, 10,
-                                      NUM_INTRO_POINTS_DEFAULT,
-                                      NUM_INTRO_POINTS_MAX, &ok, NULL);
+                                      0, NUM_INTRO_POINTS_MAX, &ok, NULL);
       if (!ok) {
         log_warn(LD_CONFIG,
                  "HiddenServiceNumIntroductionPoints "
                  "should be between %d and %d, not %s",
-                 NUM_INTRO_POINTS_DEFAULT, NUM_INTRO_POINTS_MAX,
-                 line->value);
-        rend_service_free(service);
-        return -1;
+                 0, NUM_INTRO_POINTS_MAX, line->value);
+        goto free_and_return;
       }
       log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
-               service->n_intro_points_wanted, service->directory);
+               service->n_intro_points_wanted,
+               rend_service_escaped_dir(service));
     } else if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
       /* Parse auth type and comma-separated list of client names and add a
        * rend_authorized_client_t for each client to the service's list
@@ -665,8 +794,7 @@ rend_config_services(const or_options_t *options, int validate_only)
       if (service->auth_type != REND_NO_AUTH) {
         log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient "
                  "lines for a single service.");
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       type_names_split = smartlist_new();
       smartlist_split_string(type_names_split, line->value, " ", 0, 2);
@@ -674,9 +802,7 @@ rend_config_services(const or_options_t *options, int validate_only)
         log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This "
                          "should have been prevented when parsing the "
                          "configuration.");
-        smartlist_free(type_names_split);
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       authname = smartlist_get(type_names_split, 0);
       if (!strcasecmp(authname, "basic")) {
@@ -690,8 +816,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                  (char *) smartlist_get(type_names_split, 0));
         SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
         smartlist_free(type_names_split);
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
       service->clients = smartlist_new();
       if (smartlist_len(type_names_split) < 2) {
@@ -728,8 +853,7 @@ rend_config_services(const or_options_t *options, int validate_only)
                    client_name, REND_CLIENTNAME_MAX_LEN);
           SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
           smartlist_free(clients);
-          rend_service_free(service);
-          return -1;
+          goto free_and_return;
         }
         client = tor_malloc_zero(sizeof(rend_authorized_client_t));
         client->client_name = tor_strdup(client_name);
@@ -751,109 +875,56 @@ rend_config_services(const or_options_t *options, int validate_only)
                  smartlist_len(service->clients),
                  service->auth_type == REND_BASIC_AUTH ? 512 : 16,
                  service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
     } else {
       tor_assert(!strcasecmp(line->key, "HiddenServiceVersion"));
       if (strcmp(line->value, "2")) {
         log_warn(LD_CONFIG,
                  "The only supported HiddenServiceVersion is 2.");
-        rend_service_free(service);
-        return -1;
+        goto free_and_return;
       }
     }
   }
+  /* Validate the last service that we just parsed. */
+  if (service &&
+      rend_validate_service(rend_service_staging_list, service) < 0) {
+    goto free_and_return;
+  }
   /* register the final service after we have finished parsing all services
    * this code only registers the last service, other services are registered
    * within the loop. It is ok for this service to be NULL, it is ignored. */
-  if (rend_service_check_dir_and_add(NULL, options, service,
-                                     validate_only) < 0) {
-    return -1;
+  if (rend_service_check_dir_and_add(rend_service_staging_list, options,
+                                     service, validate_only) < 0) {
+    /* Service object is freed on error so nullify pointer. */
+    service = NULL;
+    goto free_and_return;
+  }
+  /* The service is in the staging list so nullify pointer to avoid double
+   * free of this object in case of error because we lost ownership of it at
+   * this point. */
+  service = NULL;
+
+  /* Free the newly added services if validating */
+  if (validate_only) {
+    rv = 0;
+    goto free_and_return;
   }
 
-  /* If this is a reload and there were hidden services configured before,
-   * keep the introduction points that are still needed and close the
-   * other ones. */
-  if (old_service_list && !validate_only) {
-    smartlist_t *surviving_services = smartlist_new();
-
-    /* Preserve the existing ephemeral services.
-     *
-     * This is the ephemeral service equivalent of the "Copy introduction
-     * points to new services" block, except there's no copy required since
-     * the service structure isn't regenerated.
-     *
-     * After this is done, all ephemeral services will be:
-     *  * Removed from old_service_list, so the equivalent non-ephemeral code
-     *    will not attempt to preserve them.
-     *  * Added to the new rend_service_list (that previously only had the
-     *    services listed in the configuration).
-     *  * Added to surviving_services, which is the list of services that
-     *    will NOT have their intro point closed.
-     */
-    SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, {
-      if (!old->directory) {
-        SMARTLIST_DEL_CURRENT(old_service_list, old);
-        smartlist_add(surviving_services, old);
-        smartlist_add(rend_service_list, old);
-      }
-    });
-
-    /* Copy introduction points to new services. */
-    /* XXXX This is O(n^2), but it's only called on reconfigure, so it's
-     * probably ok? */
-    SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, new) {
-      SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) {
-        if (new->directory && old->directory &&
-            !strcmp(old->directory, new->directory)) {
-          smartlist_add_all(new->intro_nodes, old->intro_nodes);
-          smartlist_clear(old->intro_nodes);
-          smartlist_add_all(new->expiring_nodes, old->expiring_nodes);
-          smartlist_clear(old->expiring_nodes);
-          smartlist_add(surviving_services, old);
-          break;
-        }
-      } SMARTLIST_FOREACH_END(old);
-    } SMARTLIST_FOREACH_END(new);
-
-    /* Close introduction circuits of services we don't serve anymore. */
-    /* XXXX it would be nicer if we had a nicer abstraction to use here,
-     * so we could just iterate over the list of services to close, but
-     * once again, this isn't critical-path code. */
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-      if (!circ->marked_for_close &&
-          circ->state == CIRCUIT_STATE_OPEN &&
-          (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
-           circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
-        origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
-        int keep_it = 0;
-        tor_assert(oc->rend_data);
-        SMARTLIST_FOREACH(surviving_services, rend_service_t *, ptr, {
-          if (tor_memeq(ptr->pk_digest, oc->rend_data->rend_pk_digest,
-                      DIGEST_LEN)) {
-            keep_it = 1;
-            break;
-          }
-        });
-        if (keep_it)
-          continue;
-        log_info(LD_REND, "Closing intro point %s for service %s.",
-                 safe_str_client(extend_info_describe(
-                                            oc->build_state->chosen_exit)),
-                 oc->rend_data->onion_address);
-        circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
-        /* XXXX Is there another reason we should use here? */
-      }
-    }
-    SMARTLIST_FOREACH_END(circ);
-    smartlist_free(surviving_services);
-    SMARTLIST_FOREACH(old_service_list, rend_service_t *, ptr,
-                      rend_service_free(ptr));
-    smartlist_free(old_service_list);
-  }
+  /* This could be a reload of configuration so try to prune the main list
+   * using the staging one. And we know we are not in validate mode here.
+   * After this, the main and staging list will point to the right place and
+   * be in a quiescent usable state. */
+  rend_service_prune_list();
 
   return 0;
+ free_and_return:
+  rend_service_free(service);
+  SMARTLIST_FOREACH(rend_service_staging_list, rend_service_t *, ptr,
+                    rend_service_free(ptr));
+  smartlist_free(rend_service_staging_list);
+  rend_service_staging_list = NULL;
+  return rv;
 }
 
 /** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, using
@@ -951,7 +1022,7 @@ rend_service_del_ephemeral(const char *service_id)
              "removal.");
     return -1;
   }
-  if (s->directory) {
+  if (!rend_service_is_ephemeral(s)) {
     log_warn(LD_CONFIG, "Requested non-ephemeral Onion Service for removal.");
     return -1;
   }
@@ -968,12 +1039,13 @@ rend_service_del_ephemeral(const char *service_id)
          circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
       tor_assert(oc->rend_data);
-      if (!tor_memeq(s->pk_digest, oc->rend_data->rend_pk_digest, DIGEST_LEN))
+      if (!rend_circuit_pk_digest_eq(oc, (uint8_t *) s->pk_digest)) {
         continue;
+      }
       log_debug(LD_REND, "Closing intro point %s for service %s.",
                 safe_str_client(extend_info_describe(
                                           oc->build_state->chosen_exit)),
-                oc->rend_data->onion_address);
+                rend_data_get_address(oc->rend_data));
       circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED);
     }
   } SMARTLIST_FOREACH_END(circ);
@@ -985,6 +1057,45 @@ rend_service_del_ephemeral(const char *service_id)
   return 0;
 }
 
+/* There can be 1 second's delay due to second_elapsed_callback, and perhaps
+ * another few seconds due to blocking calls. */
+#define INTRO_CIRC_RETRY_PERIOD_SLOP 10
+
+/** Log information about the intro point creation rate and current intro
+ * points for service, upgrading the log level from min_severity to warn if
+ * we have stopped launching new intro point circuits. */
+static void
+rend_log_intro_limit(const rend_service_t *service, int min_severity)
+{
+  int exceeded_limit = (service->n_intro_circuits_launched >=
+                        rend_max_intro_circs_per_period(
+                                              service->n_intro_points_wanted));
+  int severity = min_severity;
+  /* We stopped creating circuits */
+  if (exceeded_limit) {
+    severity = LOG_WARN;
+  }
+  time_t intro_period_elapsed = time(NULL) - service->intro_period_started;
+  tor_assert_nonfatal(intro_period_elapsed >= 0);
+  {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(INTRO_CIRC_RETRY_PERIOD);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_fn(severity, LD_REND,
+             "Hidden service %s %s %d intro points in the last %d seconds. "
+             "Intro circuit launches are limited to %d per %d seconds.%s",
+             service->service_id,
+             exceeded_limit ? "exceeded launch limit with" : "launched",
+             service->n_intro_circuits_launched,
+             (int)intro_period_elapsed,
+             rend_max_intro_circs_per_period(service->n_intro_points_wanted),
+             INTRO_CIRC_RETRY_PERIOD, msg);
+      rend_service_dump_stats(severity);
+      tor_free(msg);
+    }
+  }
+}
+
 /** Replace the old value of <b>service</b>-\>desc with one that reflects
  * the other fields in service.
  */
@@ -992,7 +1103,6 @@ static void
 rend_service_update_descriptor(rend_service_t *service)
 {
   rend_service_descriptor_t *d;
-  origin_circuit_t *circ;
   int i;
 
   rend_service_descriptor_free(service->desc);
@@ -1013,9 +1123,10 @@ rend_service_update_descriptor(rend_service_t *service)
     /* This intro point won't be listed in the descriptor... */
     intro_svc->listed_in_last_desc = 0;
 
-    circ = find_intro_circuit(intro_svc, service->pk_digest);
-    if (!circ || circ->base_.purpose != CIRCUIT_PURPOSE_S_INTRO) {
-      /* This intro point's circuit isn't finished yet.  Don't list it. */
+    /* circuit_established is set in rend_service_intro_established(), and
+     * checked every second in rend_consider_services_intro_points(), so it's
+     * safe to use it here */
+    if (!intro_svc->circuit_established) {
       continue;
     }
 
@@ -1037,6 +1148,26 @@ rend_service_update_descriptor(rend_service_t *service)
       intro_svc->time_published = time(NULL);
     }
   }
+
+  /* Check that we have the right number of intro points */
+  unsigned int have_intro = (unsigned int)smartlist_len(d->intro_nodes);
+  if (have_intro != service->n_intro_points_wanted) {
+    int severity;
+    /* Getting less than we wanted or more than we're allowed is serious */
+    if (have_intro < service->n_intro_points_wanted ||
+        have_intro > NUM_INTRO_POINTS_MAX) {
+      severity = LOG_WARN;
+    } else {
+      /* Getting more than we wanted is weird, but less of a problem */
+      severity = LOG_NOTICE;
+    }
+    log_fn(severity, LD_REND, "Hidden service %s wanted %d intro points, but "
+           "descriptor was updated with %d instead.",
+           service->service_id,
+           service->n_intro_points_wanted, have_intro);
+    /* Now log an informative message about how we might have got here. */
+    rend_log_intro_limit(service, severity);
+  }
 }
 
 /* Allocate and return a string containing the path to file_name in
@@ -1067,7 +1198,7 @@ rend_service_sos_poison_path(const rend_service_t *service)
   return rend_service_path(service, sos_poison_fname);
 }
 
-/** Return True if hidden services <b>service> has been poisoned by single
+/** Return True if hidden services <b>service</b> has been poisoned by single
  * onion mode. */
 static int
 service_is_single_onion_poisoned(const rend_service_t *service)
@@ -1080,7 +1211,7 @@ service_is_single_onion_poisoned(const rend_service_t *service)
     return 0;
   }
 
-  if (!service->directory) {
+  if (rend_service_is_ephemeral(service)) {
     return 0;
   }
 
@@ -1132,8 +1263,13 @@ rend_service_verify_single_onion_poison(const rend_service_t* s,
   }
 
   /* Ephemeral services are checked at ADD_ONION time */
-  if (!s->directory) {
-    return 0;
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
   }
 
   /* Services without keys are always ok - their keys will only ever be used
@@ -1176,7 +1312,7 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   int retval = -1;
   char *poison_fname = NULL;
 
-  if (!service->directory) {
+  if (rend_service_is_ephemeral(service)) {
     log_info(LD_REND, "Ephemeral HS started in non-anonymous mode.");
     return 0;
   }
@@ -1189,7 +1325,8 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   }
 
   /* Make sure the directory was created before calling this function. */
-  if (BUG(rend_service_check_private_dir_impl(options, service, 0) < 0))
+  if (BUG(hs_check_service_private_dir(options->User, service->directory,
+                                       service->dir_group_readable, 0) < 0))
     return -1;
 
   poison_fname = rend_service_sos_poison_path(service);
@@ -1226,7 +1363,7 @@ poison_new_single_onion_hidden_service_dir_impl(const rend_service_t *service,
   return retval;
 }
 
-/** We just got launched in Single Onion Mode. That's a non-anoymous mode for
+/** We just got launched in Single Onion Mode. That's a non-anonymous mode for
  * hidden services. If s is new, we should mark its hidden service
  * directory appropriately so that it is never launched as a location-private
  * hidden service. (New directories don't have private key files.)
@@ -1243,6 +1380,16 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
   /* We must only poison directories if we're in Single Onion mode */
   tor_assert(rend_service_non_anonymous_mode_enabled(options));
 
+  /* Ephemeral services aren't allowed in non-anonymous mode */
+  if (BUG(rend_service_is_ephemeral(s))) {
+    return -1;
+  }
+
+  /* Service is expected to have a directory */
+  if (BUG(!s->directory)) {
+    return -1;
+  }
+
   if (!rend_service_private_key_exists(s)) {
     if (poison_new_single_onion_hidden_service_dir_impl(s, options)
         < 0) {
@@ -1262,22 +1409,17 @@ rend_service_poison_new_single_onion_dir(const rend_service_t *s,
 int
 rend_service_load_all_keys(const smartlist_t *service_list)
 {
-  const smartlist_t *s_list = NULL;
-  /* If no special service list is provided, then just use the global one. */
-  if (!service_list) {
-    if (BUG(!rend_service_list)) {
-      return -1;
-    }
-    s_list = rend_service_list;
-  } else {
-    s_list = service_list;
+  /* Use service_list for unit tests */
+  const smartlist_t *s_list = rend_get_service_list(service_list);
+  if (BUG(!s_list)) {
+    return -1;
   }
 
   SMARTLIST_FOREACH_BEGIN(s_list, rend_service_t *, s) {
     if (s->private_key)
       continue;
-    log_info(LD_REND, "Loading hidden-service keys from \"%s\"",
-             s->directory);
+    log_info(LD_REND, "Loading hidden-service keys from %s",
+             rend_service_escaped_dir(s));
 
     if (rend_service_load_keys(s) < 0)
       return -1;
@@ -1309,9 +1451,9 @@ rend_services_add_filenames_to_lists(smartlist_t *open_lst,
   if (!rend_service_list)
     return;
   SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) {
-    if (s->directory) {
+    if (!rend_service_is_ephemeral(s)) {
       rend_service_add_filenames_to_list(open_lst, s);
-      smartlist_add(stat_lst, tor_strdup(s->directory));
+      smartlist_add_strdup(stat_lst, s->directory);
     }
   } SMARTLIST_FOREACH_END(s);
 }
@@ -1334,32 +1476,6 @@ rend_service_derive_key_digests(struct rend_service_t *s)
   return 0;
 }
 
-/* Implements the directory check from rend_service_check_private_dir,
- * without doing the single onion poison checks. */
-static int
-rend_service_check_private_dir_impl(const or_options_t *options,
-                                    const rend_service_t *s,
-                                    int create)
-{
-  cpd_check_t  check_opts = CPD_NONE;
-  if (create) {
-    check_opts |= CPD_CREATE;
-  } else {
-    check_opts |= CPD_CHECK_MODE_ONLY;
-    check_opts |= CPD_CHECK;
-  }
-  if (s->dir_group_readable) {
-    check_opts |= CPD_GROUP_READ;
-  }
-  /* Check/create directory */
-  if (check_private_dir(s->directory, check_opts, options->User) < 0) {
-    log_warn(LD_REND, "Checking service directory %s failed.", s->directory);
-    return -1;
-  }
-
-  return 0;
-}
-
 /** Make sure that the directory for <b>s</b> is private, using the config in
  * <b>options</b>.
  * If <b>create</b> is true:
@@ -1380,7 +1496,8 @@ rend_service_check_private_dir(const or_options_t *options,
   }
 
   /* Check/create directory */
-  if (rend_service_check_private_dir_impl(options, s, create) < 0) {
+  if (hs_check_service_private_dir(options->User, s->directory,
+                                   s->dir_group_readable, create) < 0) {
     return -1;
   }
 
@@ -1806,7 +1923,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   const or_options_t *options = get_options();
   char *err_msg = NULL;
   int err_msg_severity = LOG_WARN;
-  const char *stage_descr = NULL;
+  const char *stage_descr = NULL, *rend_pk_digest;
   int reason = END_CIRC_REASON_TORPROTOCOL;
   /* Service/circuit/key stuff we can learn before parsing */
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
@@ -1841,14 +1958,15 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   assert_circ_anonymity_ok(circuit, options);
   tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   /* We'll use this in a bazillion log messages */
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
   /* look up service depending on circuit. */
-  service =
-    rend_service_get_by_pk_digest(circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_BUG,
              "Internal error: Got an INTRODUCE2 cell on an intro "
@@ -2041,7 +2159,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   /* Launch a circuit to the client's chosen rendezvous point.
    */
-  int max_rend_failures=get_max_rend_failures();
+  int max_rend_failures=hs_get_service_max_rend_failures();
   for (i=0;i<max_rend_failures;i++) {
     int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
     if (circ_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME;
@@ -2073,8 +2191,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
   /* Fill in the circuit's state. */
 
   launched->rend_data =
-    rend_data_service_create(service->service_id,
-                             circuit->rend_data->rend_pk_digest,
+    rend_data_service_create(service->service_id, rend_pk_digest,
                              parsed_req->rc, service->auth_type);
 
   launched->build_state->service_pending_final_cpath_ref =
@@ -2705,7 +2822,14 @@ rend_service_decrypt_intro(
   /* Check that this cell actually matches this service key */
 
   /* first DIGEST_LEN bytes of request is intro or service pk digest */
-  crypto_pk_get_digest(key, (char *)key_digest);
+  if (crypto_pk_get_digest(key, (char *)key_digest) < 0) {
+    if (err_msg_out)
+      *err_msg_out = tor_strdup("Couldn't compute RSA digest.");
+    log_warn(LD_BUG, "Couldn't compute key digest.");
+    status = -7;
+    goto err;
+  }
+
   if (tor_memneq(key_digest, intro->pk, DIGEST_LEN)) {
     if (err_msg_out) {
       base32_encode(service_id, REND_SERVICE_ID_LEN_BASE32 + 1,
@@ -2944,10 +3068,10 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   }
   oldcirc->hs_service_side_rend_circ_has_been_relaunched = 1;
 
-  /* We check failure_count >= get_max_rend_failures()-1 below, and the -1
-   * is because we increment the failure count for our current failure
+  /* We check failure_count >= hs_get_service_max_rend_failures()-1 below, and
+   * the -1 is because we increment the failure count for our current failure
    * *after* this clause. */
-  int max_rend_failures = get_max_rend_failures() - 1;
+  int max_rend_failures = hs_get_service_max_rend_failures() - 1;
 
   if (!oldcirc->build_state ||
       oldcirc->build_state->failure_count >= max_rend_failures ||
@@ -3113,15 +3237,67 @@ count_intro_point_circuits(const rend_service_t *service)
          circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) {
       origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
       if (oc->rend_data &&
-          !rend_cmp_service_ids(service->service_id,
-                                oc->rend_data->onion_address))
+          rend_circuit_pk_digest_eq(oc, (uint8_t *) service->pk_digest)) {
         num_ipos++;
+      }
     }
   }
   SMARTLIST_FOREACH_END(circ);
   return num_ipos;
 }
 
+/* Given a buffer of at least RELAY_PAYLOAD_SIZE bytes in <b>cell_body_out</b>,
+   write the body of a legacy ESTABLISH_INTRO cell in it. Use <b>intro_key</b>
+   as the intro point auth key, and <b>rend_circ_nonce</b> as the circuit
+   crypto material. On success, fill <b>cell_body_out</b> and return the number
+   of bytes written. On fail, return -1.
+ */
+STATIC ssize_t
+encode_establish_intro_cell_legacy(char *cell_body_out,
+                                 size_t cell_body_out_len,
+                                 crypto_pk_t *intro_key, char *rend_circ_nonce)
+{
+  int retval = -1;
+  int r;
+  int len = 0;
+  char auth[DIGEST_LEN + 9];
+
+  tor_assert(intro_key);
+  tor_assert(rend_circ_nonce);
+
+  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
+  r = crypto_pk_asn1_encode(intro_key, cell_body_out+2,
+                            RELAY_PAYLOAD_SIZE-2);
+  if (r < 0) {
+    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
+    goto err;
+  }
+  len = r;
+  set_uint16(cell_body_out, htons((uint16_t)len));
+  len += 2;
+  memcpy(auth, rend_circ_nonce, DIGEST_LEN);
+  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
+  if (crypto_digest(cell_body_out+len, auth, DIGEST_LEN+9))
+    goto err;
+  len += 20;
+  note_crypto_pk_op(REND_SERVER);
+  r = crypto_pk_private_sign_digest(intro_key, cell_body_out+len,
+                                    cell_body_out_len - len,
+                                    cell_body_out, len);
+  if (r<0) {
+    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
+    goto err;
+  }
+  len += r;
+
+  retval = len;
+
+ err:
+  memwipe(auth, 0, sizeof(auth));
+
+  return retval;
+}
+
 /** Called when we're done building a circuit to an introduction point:
  *  sends a RELAY_ESTABLISH_INTRO cell.
  */
@@ -3129,23 +3305,23 @@ void
 rend_service_intro_has_opened(origin_circuit_t *circuit)
 {
   rend_service_t *service;
-  size_t len;
-  int r;
   char buf[RELAY_PAYLOAD_SIZE];
-  char auth[DIGEST_LEN + 9];
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+  unsigned int expiring_nodes_len, num_ip_circuits, valid_ip_circuits = 0;
   int reason = END_CIRC_REASON_TORPROTOCOL;
+  const char *rend_pk_digest;
 
   tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO);
   assert_circ_anonymity_ok(circuit, get_options());
   tor_assert(circuit->cpath);
   tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only on supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %u.",
              safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id);
@@ -3153,13 +3329,22 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
     goto err;
   }
 
+  /* Take the current amount of expiring nodes and the current amount of IP
+   * circuits and compute how many valid IP circuits we have. */
+  expiring_nodes_len = (unsigned int) smartlist_len(service->expiring_nodes);
+  num_ip_circuits = count_intro_point_circuits(service);
+  /* Let's avoid an underflow. The valid_ip_circuits is initialized to 0 in
+   * case this condition turns out false because it means that all circuits
+   * are expiring so we need to keep this circuit. */
+  if (num_ip_circuits > expiring_nodes_len) {
+    valid_ip_circuits = num_ip_circuits - expiring_nodes_len;
+  }
+
   /* If we already have enough introduction circuits for this service,
    * redefine this one as a general circuit or close it, depending.
-   * Substract the amount of expiring nodes here since the circuits are
+   * Substract the amount of expiring nodes here because the circuits are
    * still opened. */
-  if ((count_intro_point_circuits(service) -
-       smartlist_len(service->expiring_nodes)) >
-      service->n_intro_points_wanted) {
+  if (valid_ip_circuits > service->n_intro_points_wanted) {
     const or_options_t *options = get_options();
     /* Remove the intro point associated with this circuit, it's being
      * repurposed or closed thus cleanup memory. */
@@ -3186,9 +3371,8 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
       circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_C_GENERAL);
 
       {
-        rend_data_t *rend_data = circuit->rend_data;
+        rend_data_free(circuit->rend_data);
         circuit->rend_data = NULL;
-        rend_data_free(rend_data);
       }
       {
         crypto_pk_t *intro_key = circuit->intro_key;
@@ -3206,42 +3390,25 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
            (unsigned)circuit->base_.n_circ_id, serviceid);
   circuit_log_path(LOG_INFO, LD_REND, circuit);
 
-  /* Use the intro key instead of the service key in ESTABLISH_INTRO. */
-  crypto_pk_t *intro_key = circuit->intro_key;
-  /* Build the payload for a RELAY_ESTABLISH_INTRO cell. */
-  r = crypto_pk_asn1_encode(intro_key, buf+2,
-                            RELAY_PAYLOAD_SIZE-2);
-  if (r < 0) {
-    log_warn(LD_BUG, "Internal error; failed to establish intro point.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
-  }
-  len = r;
-  set_uint16(buf, htons((uint16_t)len));
-  len += 2;
-  memcpy(auth, circuit->cpath->prev->rend_circ_nonce, DIGEST_LEN);
-  memcpy(auth+DIGEST_LEN, "INTRODUCE", 9);
-  if (crypto_digest(buf+len, auth, DIGEST_LEN+9))
-    goto err;
-  len += 20;
-  note_crypto_pk_op(REND_SERVER);
-  r = crypto_pk_private_sign_digest(intro_key, buf+len, sizeof(buf)-len,
-                                    buf, len);
-  if (r<0) {
-    log_warn(LD_BUG, "Internal error: couldn't sign introduction request.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
-  }
-  len += r;
+  /* Send the ESTABLISH_INTRO cell */
+  {
+    ssize_t len;
+    len = encode_establish_intro_cell_legacy(buf, sizeof(buf),
+                                      circuit->intro_key,
+                                      circuit->cpath->prev->rend_circ_nonce);
+    if (len < 0) {
+      reason = END_CIRC_REASON_INTERNAL;
+      goto err;
+    }
 
-  if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
-                                   RELAY_COMMAND_ESTABLISH_INTRO,
-                                   buf, len, circuit->cpath->prev)<0) {
-    log_info(LD_GENERAL,
+    if (relay_send_command_from_edge(0, TO_CIRCUIT(circuit),
+                                     RELAY_COMMAND_ESTABLISH_INTRO,
+                                     buf, len, circuit->cpath->prev)<0) {
+      log_info(LD_GENERAL,
              "Couldn't send introduction request for service %s on circuit %u",
              serviceid, (unsigned)circuit->base_.n_circ_id);
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+      goto done;
+    }
   }
 
   /* We've attempted to use this circuit */
@@ -3253,7 +3420,6 @@ rend_service_intro_has_opened(origin_circuit_t *circuit)
   circuit_mark_for_close(TO_CIRCUIT(circuit), reason);
  done:
   memwipe(buf, 0, sizeof(buf));
-  memwipe(auth, 0, sizeof(auth));
   memwipe(serviceid, 0, sizeof(serviceid));
 
   return;
@@ -3272,22 +3438,24 @@ rend_service_intro_established(origin_circuit_t *circuit,
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
   (void) request;
   (void) request_len;
+  tor_assert(circuit->rend_data);
+  /* XXX: This is version 2 specific (only supported one for now). */
+  const char *rend_pk_digest =
+    (char *) rend_data_get_pk_digest(circuit->rend_data, NULL);
 
   if (circuit->base_.purpose != CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) {
     log_warn(LD_PROTOCOL,
              "received INTRO_ESTABLISHED cell on non-intro circuit.");
     goto err;
   }
-  tor_assert(circuit->rend_data);
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Unknown service on introduction circuit %u.",
              (unsigned)circuit->base_.n_circ_id);
     goto err;
   }
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32 + 1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
   /* We've just successfully established a intro circuit to one of our
    * introduction point, account for it. */
   intro = find_intro_point(circuit);
@@ -3330,6 +3498,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
   char hexcookie[9];
   int reason;
+  const char *rend_cookie, *rend_pk_digest;
 
   tor_assert(circuit->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   tor_assert(circuit->cpath);
@@ -3337,6 +3506,11 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   assert_circ_anonymity_ok(circuit, get_options());
   tor_assert(circuit->rend_data);
 
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circuit->rend_data,
+                                                    NULL);
+  rend_cookie = circuit->rend_data->rend_cookie;
+
   /* Declare the circuit dirty to avoid reuse, and for path-bias */
   if (!circuit->base_.timestamp_dirty)
     circuit->base_.timestamp_dirty = time(NULL);
@@ -3346,9 +3520,9 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
 
   hop = circuit->build_state->service_pending_final_cpath_ref->cpath;
 
-  base16_encode(hexcookie,9,circuit->rend_data->rend_cookie,4);
+  base16_encode(hexcookie,9, rend_cookie,4);
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circuit->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
 
   log_info(LD_REND,
            "Done building circuit %u to rendezvous with "
@@ -3377,8 +3551,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   circuit->build_state->pending_final_cpath = hop;
   circuit->build_state->service_pending_final_cpath_ref->cpath = NULL;
 
-  service = rend_service_get_by_pk_digest(
-                circuit->rend_data->rend_pk_digest);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_GENERAL, "Internal error: unrecognized service ID on "
              "rendezvous circuit.");
@@ -3387,7 +3560,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   }
 
   /* All we need to do is send a RELAY_RENDEZVOUS1 cell... */
-  memcpy(buf, circuit->rend_data->rend_cookie, REND_COOKIE_LEN);
+  memcpy(buf, rend_cookie, REND_COOKIE_LEN);
   if (crypto_dh_get_public(hop->rend_dh_handshake_state,
                            buf+REND_COOKIE_LEN, DH_KEY_LEN)<0) {
     log_warn(LD_GENERAL,"Couldn't get DH public key.");
@@ -3403,8 +3576,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
                                    buf, REND_COOKIE_LEN+DH_KEY_LEN+DIGEST_LEN,
                                    circuit->cpath->prev)<0) {
     log_warn(LD_GENERAL, "Couldn't send RENDEZVOUS1 cell.");
-    reason = END_CIRC_REASON_INTERNAL;
-    goto err;
+    goto done;
   }
 
   crypto_dh_free(hop->rend_dh_handshake_state);
@@ -3451,8 +3623,8 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
   origin_circuit_t *circ = NULL;
 
   tor_assert(intro);
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest,
-                                                  CIRCUIT_PURPOSE_S_INTRO))) {
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest, CIRCUIT_PURPOSE_S_INTRO))) {
     if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
                 intro->extend_info->identity_digest, DIGEST_LEN) &&
         circ->rend_data) {
@@ -3461,8 +3633,9 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest)
   }
 
   circ = NULL;
-  while ((circ = circuit_get_next_by_pk_and_purpose(circ,pk_digest,
-                                        CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
+  while ((circ = circuit_get_next_by_pk_and_purpose(circ,
+                         (uint8_t *) pk_digest,
+                         CIRCUIT_PURPOSE_S_ESTABLISH_INTRO))) {
     if (tor_memeq(circ->build_state->chosen_exit->identity_digest,
                 intro->extend_info->identity_digest, DIGEST_LEN) &&
         circ->rend_data) {
@@ -3501,7 +3674,7 @@ find_intro_point(origin_circuit_t *circ)
   tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
              TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO);
   tor_assert(circ->rend_data);
-  serviceid = circ->rend_data->onion_address;
+  serviceid = rend_data_get_address(circ->rend_data);
 
   SMARTLIST_FOREACH(rend_service_list, rend_service_t *, s,
     if (tor_memeq(s->service_id, serviceid, REND_SERVICE_ID_LEN_BASE32)) {
@@ -3577,13 +3750,16 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
        * request. Lookup is made in rend_service_desc_has_uploaded(). */
       rend_data = rend_data_client_create(service_id, desc->desc_id, NULL,
                                           REND_NO_AUTH);
-      directory_initiate_command_routerstatus_rend(hs_dir,
-                                              DIR_PURPOSE_UPLOAD_RENDDESC_V2,
-                                                   ROUTER_PURPOSE_GENERAL,
-                                                   DIRIND_ANONYMOUS, NULL,
-                                                   desc->desc_str,
-                                                   strlen(desc->desc_str),
-                                                   0, rend_data);
+      directory_request_t *req =
+        directory_request_new(DIR_PURPOSE_UPLOAD_RENDDESC_V2);
+      directory_request_set_routerstatus(req, hs_dir);
+      directory_request_set_indirection(req, DIRIND_ANONYMOUS);
+      directory_request_set_payload(req,
+                                    desc->desc_str, strlen(desc->desc_str));
+      directory_request_set_rend_query(req, rend_data);
+      directory_initiate_request(req);
+      directory_request_free(req);
+
       rend_data_free(rend_data);
       base32_encode(desc_id_base32, sizeof(desc_id_base32),
                     desc->desc_id, DIGEST_LEN);
@@ -3815,6 +3991,19 @@ remove_invalid_intro_points(rend_service_t *service,
 {
   tor_assert(service);
 
+  /* Remove any expired nodes that doesn't have a circuit. */
+  SMARTLIST_FOREACH_BEGIN(service->expiring_nodes, rend_intro_point_t *,
+                          intro) {
+    origin_circuit_t *intro_circ =
+      find_intro_circuit(intro, service->pk_digest);
+    if (intro_circ) {
+      continue;
+    }
+    /* No more circuit, cleanup the into point object. */
+    SMARTLIST_DEL_CURRENT(service->expiring_nodes, intro);
+    rend_intro_point_free(intro);
+  } SMARTLIST_FOREACH_END(intro);
+
   SMARTLIST_FOREACH_BEGIN(service->intro_nodes, rend_intro_point_t *,
                           intro) {
     /* Find the introduction point node object. */
@@ -3890,10 +4079,13 @@ void
 rend_service_desc_has_uploaded(const rend_data_t *rend_data)
 {
   rend_service_t *service;
+  const char *onion_address;
 
   tor_assert(rend_data);
 
-  service = rend_service_get_by_service_id(rend_data->onion_address);
+  onion_address = rend_data_get_address(rend_data);
+
+  service = rend_service_get_by_service_id(onion_address);
   if (service == NULL) {
     return;
   }
@@ -3911,6 +4103,23 @@ rend_service_desc_has_uploaded(const rend_data_t *rend_data)
   } SMARTLIST_FOREACH_END(intro);
 }
 
+/** Don't try to build more than this many circuits before giving up
+ * for a while. Dynamically calculated based on the configured number of
+ * introduction points for the service, n_intro_points_wanted. */
+static int
+rend_max_intro_circs_per_period(unsigned int n_intro_points_wanted)
+{
+  /* Allow all but one of the initial connections to fail and be
+   * retried. (If all fail, we *want* to wait, because something is broken.) */
+  tor_assert(n_intro_points_wanted <= NUM_INTRO_POINTS_MAX);
+
+  /* For the normal use case, 3 intro points plus 2 extra for performance and
+   * allow that twice because once every 24h or so, we can do it twice for two
+   * descriptors that is the current one and the next one. So (3 + 2) * 2 ==
+   * 12 allowed attempts for one period. */
+  return ((n_intro_points_wanted + NUM_INTRO_POINTS_EXTRA) * 2);
+}
+
 /** For every service, check how many intro points it currently has, and:
  *  - Invalidate introdution points based on specific criteria, see
  *  remove_invalid_intro_points comments.
@@ -3955,23 +4164,29 @@ rend_consider_services_intro_points(void)
     smartlist_clear(exclude_nodes);
     smartlist_clear(retry_nodes);
 
+    /* Cleanup the invalid intro points and save the node objects, if any,
+     * in the exclude_nodes and retry_nodes lists. */
+    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
+
     /* This retry period is important here so we don't stress circuit
      * creation. */
+
     if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) {
-      /* One period has elapsed; we can try building circuits again. */
+      /* One period has elapsed:
+       *  - if we stopped, we can try building circuits again,
+       *  - if we haven't, we reset the circuit creation counts. */
+      rend_log_intro_limit(service, LOG_INFO);
       service->intro_period_started = now;
       service->n_intro_circuits_launched = 0;
     } else if (service->n_intro_circuits_launched >=
-               MAX_INTRO_CIRCS_PER_PERIOD) {
+               rend_max_intro_circs_per_period(
+                                      service->n_intro_points_wanted)) {
       /* We have failed too many times in this period; wait for the next
-       * one before we try again. */
+       * one before we try to initiate any more connections. */
+      rend_log_intro_limit(service, LOG_WARN);
       continue;
     }
 
-    /* Cleanup the invalid intro points and save the node objects, if apply,
-     * in the exclude_nodes and retry_nodes list. */
-    remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now);
-
     /* Let's try to rebuild circuit on the nodes we want to retry on. */
     SMARTLIST_FOREACH_BEGIN(retry_nodes, rend_intro_point_t *, intro) {
       r = rend_service_launch_establish_intro(service, intro);
@@ -3991,17 +4206,17 @@ rend_consider_services_intro_points(void)
     /* Avoid mismatched signed comparaison below. */
     intro_nodes_len = (unsigned int) smartlist_len(service->intro_nodes);
 
-    /* Quiescent state, no node expiring and we have more or the amount of
-     * wanted node for this service. Proceed to the next service. Could be
-     * more because we launch two preemptive circuits if our intro nodes
-     * list is empty. */
-    if (smartlist_len(service->expiring_nodes) == 0 &&
-        intro_nodes_len >= service->n_intro_points_wanted) {
+    /* Quiescent state, we have more or the equal amount of wanted node for
+     * this service. Proceed to the next service. We can have more nodes
+     * because we launch extra preemptive circuits if our intro nodes list was
+     * originally empty for performance reasons. */
+    if (intro_nodes_len >= service->n_intro_points_wanted) {
       continue;
     }
 
-    /* Number of intro points we want to open which is the wanted amount
-     * minus the current amount of valid nodes. */
+    /* Number of intro points we want to open which is the wanted amount minus
+     * the current amount of valid nodes. We know that this won't underflow
+     * because of the check above. */
     n_intro_points_to_open = service->n_intro_points_wanted - intro_nodes_len;
     if (intro_nodes_len == 0) {
       /* We want to end up with n_intro_points_wanted intro points, but if
@@ -4021,8 +4236,6 @@ rend_consider_services_intro_points(void)
       const node_t *node;
       rend_intro_point_t *intro;
       router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC;
-      if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION)
-        flags |= CRN_ALLOW_INVALID;
       router_crn_flags_t direct_flags = flags;
       direct_flags |= CRN_PREF_ADDR;
       direct_flags |= CRN_DIRECT_CONN;
@@ -4058,6 +4271,9 @@ rend_consider_services_intro_points(void)
        * even if we are a single onion service and intend to connect to it
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
+      if (BUG(intro->extend_info == NULL)) {
+        break;
+      }
       intro->intro_key = crypto_pk_new();
       const int fail = crypto_pk_generate_key(intro->intro_key);
       tor_assert(!fail);
@@ -4193,8 +4409,8 @@ rend_service_dump_stats(int severity)
 
   for (i=0; i < smartlist_len(rend_service_list); ++i) {
     service = smartlist_get(rend_service_list, i);
-    tor_log(severity, LD_GENERAL, "Service configured in \"%s\":",
-        service->directory);
+    tor_log(severity, LD_GENERAL, "Service configured in %s:",
+            rend_service_escaped_dir(service));
     for (j=0; j < smartlist_len(service->intro_nodes); ++j) {
       intro = smartlist_get(service->intro_nodes, j);
       safe_name = safe_str_client(intro->extend_info->nickname);
@@ -4280,14 +4496,16 @@ rend_service_set_connection_addr_port(edge_connection_t *conn,
   smartlist_t *matching_ports;
   rend_service_port_config_t *chosen_port;
   unsigned int warn_once = 0;
+  const char *rend_pk_digest;
 
   tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED);
   tor_assert(circ->rend_data);
   log_debug(LD_REND,"beginning to hunt for addr/port");
+  /* XXX: This is version 2 specific (only one supported). */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(circ->rend_data, NULL);
   base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
-                circ->rend_data->rend_pk_digest, REND_SERVICE_ID_LEN);
-  service = rend_service_get_by_pk_digest(
-                circ->rend_data->rend_pk_digest);
+                rend_pk_digest, REND_SERVICE_ID_LEN);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
   if (!service) {
     log_warn(LD_REND, "Couldn't find any service associated with pk %s on "
              "rendezvous circuit %u; closing.",
@@ -4409,3 +4627,19 @@ rend_service_non_anonymous_mode_enabled(const or_options_t *options)
   return options->HiddenServiceNonAnonymousMode ? 1 : 0;
 }
 
+#ifdef TOR_UNIT_TESTS
+
+STATIC void
+set_rend_service_list(smartlist_t *new_list)
+{
+  rend_service_list = new_list;
+}
+
+STATIC void
+set_rend_rend_service_staging_list(smartlist_t *new_list)
+{
+  rend_service_staging_list = new_list;
+}
+
+#endif /* TOR_UNIT_TESTS */
+
diff --git a/src/or/rendservice.h b/src/or/rendservice.h
index 3b185672f6..1583a6010b 100644
--- a/src/or/rendservice.h
+++ b/src/or/rendservice.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -129,10 +129,23 @@ STATIC int rend_service_verify_single_onion_poison(
 STATIC int rend_service_poison_new_single_onion_dir(
                                                   const rend_service_t *s,
                                                   const or_options_t* options);
-#endif
+STATIC ssize_t encode_establish_intro_cell_legacy(char *cell_body_out,
+                                                  size_t cell_body_out_len,
+                                                  crypto_pk_t *intro_key,
+                                                  char *rend_circ_nonce);
+#ifdef TOR_UNIT_TESTS
+
+STATIC void set_rend_service_list(smartlist_t *new_list);
+STATIC void set_rend_rend_service_staging_list(smartlist_t *new_list);
+STATIC void rend_service_prune_list_impl_(void);
+
+#endif /* TOR_UNIT_TESTS */
+
+#endif /* RENDSERVICE_PRIVATE */
 
 int num_rend_services(void);
 int rend_config_services(const or_options_t *options, int validate_only);
+void rend_service_prune_list(void);
 int rend_service_load_all_keys(const smartlist_t *service_list);
 void rend_services_add_filenames_to_lists(smartlist_t *open_lst,
                                           smartlist_t *stat_lst);
diff --git a/src/or/rephist.c b/src/or/rephist.c
index f0bac57898..ffc1867955 100644
--- a/src/or/rephist.c
+++ b/src/or/rephist.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -84,9 +84,13 @@
 #include "router.h"
 #include "routerlist.h"
 #include "ht.h"
+#include "channelpadding.h"
+
+#include "channelpadding.h"
+#include "connection_or.h"
 
 static void bw_arrays_init(void);
-static void predicted_ports_init(void);
+static void predicted_ports_alloc(void);
 
 /** Total number of bytes currently allocated in fields used by rephist.c. */
 uint64_t rephist_total_alloc=0;
@@ -165,6 +169,44 @@ typedef struct or_history_t {
   digestmap_t *link_history_map;
 } or_history_t;
 
+/**
+ * This structure holds accounting needed to calculate the padding overhead.
+ */
+typedef struct padding_counts_t {
+  /** Total number of cells we have received, including padding */
+  uint64_t read_cell_count;
+  /** Total number of cells we have sent, including padding */
+  uint64_t write_cell_count;
+  /** Total number of CELL_PADDING cells we have received */
+  uint64_t read_pad_cell_count;
+  /** Total number of CELL_PADDING cells we have sent */
+  uint64_t write_pad_cell_count;
+  /** Total number of read cells on padding-enabled conns */
+  uint64_t enabled_read_cell_count;
+  /** Total number of sent cells on padding-enabled conns */
+  uint64_t enabled_write_cell_count;
+  /** Total number of read CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_read_pad_cell_count;
+  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
+  uint64_t enabled_write_pad_cell_count;
+  /** Total number of RELAY_DROP cells we have received */
+  uint64_t read_drop_cell_count;
+  /** Total number of RELAY_DROP cells we have sent */
+  uint64_t write_drop_cell_count;
+  /** The maximum number of padding timers we've seen in 24 hours */
+  uint64_t maximum_chanpad_timers;
+  /** When did we first copy padding_current into padding_published? */
+  char first_published_at[ISO_TIME_LEN+1];
+} padding_counts_t;
+
+/** Holds the current values of our padding statistics.
+ * It is not published until it is transferred to padding_published. */
+static padding_counts_t padding_current;
+
+/** Remains fixed for a 24 hour period, and then is replaced
+ * by a redacted copy of padding_current */
+static padding_counts_t padding_published;
+
 /** When did we last multiply all routers' weighted_run_length and
  * total_run_weights by STABILITY_ALPHA? */
 static time_t stability_last_downrated = 0;
@@ -264,7 +306,7 @@ rep_hist_init(void)
 {
   history_map = digestmap_new();
   bw_arrays_init();
-  predicted_ports_init();
+  predicted_ports_alloc();
 }
 
 /** Helper: note that we are no longer connected to the router with history
@@ -905,9 +947,9 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down)
     base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
 
     if (missing_means_down && hist->start_of_run &&
-        !router_get_by_id_digest(digest)) {
+        !connection_or_digest_is_known_relay(digest)) {
       /* We think this relay is running, but it's not listed in our
-       * routerlist. Somehow it fell out without telling us it went
+       * consensus. Somehow it fell out without telling us it went
        * down. Complain and also correct it. */
       log_info(LD_HIST,
                "Relay '%s' is listed as up in rephist, but it's not in "
@@ -1758,6 +1800,40 @@ typedef struct predicted_port_t {
 
 /** A list of port numbers that have been used recently. */
 static smartlist_t *predicted_ports_list=NULL;
+/** How long do we keep predicting circuits? */
+static int prediction_timeout=0;
+/** When was the last time we added a prediction entry (HS or port) */
+static time_t last_prediction_add_time=0;
+
+/**
+ * How much time left until we stop predicting circuits?
+ */
+int
+predicted_ports_prediction_time_remaining(time_t now)
+{
+  time_t idle_delta = now - last_prediction_add_time;
+
+  /* Protect against overflow of return value. This can happen if the clock
+   * jumps backwards in time. Update the last prediction time (aka last
+   * active time) to prevent it. This update is preferable to using monotonic
+   * time because it prevents clock jumps into the past from simply causing
+   * very long idle timeouts while the monotonic time stands still. */
+  if (last_prediction_add_time > now) {
+    last_prediction_add_time = now;
+    idle_delta = 0;
+  }
+
+  /* Protect against underflow of the return value. This can happen for very
+   * large periods of inactivity/system sleep. */
+  if (idle_delta > prediction_timeout)
+    return 0;
+
+  if (BUG((prediction_timeout - idle_delta) > INT_MAX)) {
+    return INT_MAX;
+  }
+
+  return (int)(prediction_timeout - idle_delta);
+}
 
 /** We just got an application request for a connection with
  * port <b>port</b>. Remember it for the future, so we can keep
@@ -1767,21 +1843,40 @@ static void
 add_predicted_port(time_t now, uint16_t port)
 {
   predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t));
+
+  //  If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
   pp->port = port;
   pp->time = now;
   rephist_total_alloc += sizeof(*pp);
   smartlist_add(predicted_ports_list, pp);
 }
 
-/** Initialize whatever memory and structs are needed for predicting
+/**
+ * Allocate whatever memory and structs are needed for predicting
  * which ports will be used. Also seed it with port 80, so we'll build
  * circuits on start-up.
  */
 static void
-predicted_ports_init(void)
+predicted_ports_alloc(void)
 {
   predicted_ports_list = smartlist_new();
-  add_predicted_port(time(NULL), 80); /* add one to kickstart us */
+}
+
+void
+predicted_ports_init(void)
+{
+  add_predicted_port(time(NULL), 443); // Add a port to get us started
 }
 
 /** Free whatever memory is needed for predicting which ports will
@@ -1812,6 +1907,12 @@ rep_hist_note_used_port(time_t now, uint16_t port)
   SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
     if (pp->port == port) {
       pp->time = now;
+
+      last_prediction_add_time = now;
+      log_info(LD_CIRC,
+               "New port prediction added. Will continue predictive circ "
+               "building for %d more seconds.",
+               predicted_ports_prediction_time_remaining(now));
       return;
     }
   } SMARTLIST_FOREACH_END(pp);
@@ -1828,7 +1929,8 @@ rep_hist_get_predicted_ports(time_t now)
   int predicted_circs_relevance_time;
   smartlist_t *out = smartlist_new();
   tor_assert(predicted_ports_list);
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
+
+  predicted_circs_relevance_time = prediction_timeout;
 
   /* clean out obsolete entries */
   SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
@@ -1888,6 +1990,18 @@ static time_t predicted_internal_capacity_time = 0;
 void
 rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity)
 {
+  // If the list is empty, re-randomize predicted ports lifetime
+  if (!any_predicted_circuits(now)) {
+    prediction_timeout = channelpadding_get_circuits_available_timeout();
+  }
+
+  last_prediction_add_time = now;
+
+  log_info(LD_CIRC,
+          "New port prediction added. Will continue predictive circ building "
+          "for %d more seconds.",
+          predicted_ports_prediction_time_remaining(now));
+
   predicted_internal_time = now;
   if (need_uptime)
     predicted_internal_uptime_time = now;
@@ -1901,7 +2015,8 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime,
                                 int *need_capacity)
 {
   int predicted_circs_relevance_time;
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
+
+  predicted_circs_relevance_time = prediction_timeout;
 
   if (!predicted_internal_time) { /* initialize it */
     predicted_internal_time = now;
@@ -1923,7 +2038,7 @@ int
 any_predicted_circuits(time_t now)
 {
   int predicted_circs_relevance_time;
-  predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime;
+  predicted_circs_relevance_time = prediction_timeout;
 
   return smartlist_len(predicted_ports_list) ||
          predicted_internal_time + predicted_circs_relevance_time >= now;
@@ -3210,8 +3325,7 @@ rep_hist_hs_stats_write(time_t now)
   return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
 }
 
-#define MAX_LINK_PROTO_TO_LOG 4
-static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2];
+static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
 
 /** Note that we negotiated link protocol version <b>link_proto</b>, on
  * a connection that started here iff <b>started_here</b> is true.
@@ -3220,7 +3334,7 @@ void
 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
 {
   started_here = !!started_here; /* force to 0 or 1 */
-  if (link_proto > MAX_LINK_PROTO_TO_LOG) {
+  if (link_proto > MAX_LINK_PROTO) {
     log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
     return;
   }
@@ -3228,6 +3342,165 @@ rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
   link_proto_count[link_proto][started_here]++;
 }
 
+/**
+ * Update the maximum count of total pending channel padding timers
+ * in this period.
+ */
+void
+rep_hist_padding_count_timers(uint64_t num_timers)
+{
+  if (num_timers > padding_current.maximum_chanpad_timers) {
+    padding_current.maximum_chanpad_timers = num_timers;
+  }
+}
+
+/**
+ * Count a cell that we sent for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_write(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.write_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.write_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_write_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_write_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Count a cell that we've received for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_read(padding_type_t type)
+{
+  switch (type) {
+    case PADDING_TYPE_DROP:
+      padding_current.read_drop_cell_count++;
+      break;
+    case PADDING_TYPE_CELL:
+      padding_current.read_pad_cell_count++;
+      break;
+    case PADDING_TYPE_TOTAL:
+      padding_current.read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_TOTAL:
+      padding_current.enabled_read_cell_count++;
+      break;
+    case PADDING_TYPE_ENABLED_CELL:
+      padding_current.enabled_read_pad_cell_count++;
+      break;
+  }
+}
+
+/**
+ * Reset our current padding statistics. Called once every 24 hours.
+ */
+void
+rep_hist_reset_padding_counts(void)
+{
+  memset(&padding_current, 0, sizeof(padding_current));
+}
+
+/**
+ * Copy our current cell counts into a structure for listing in our
+ * extra-info descriptor. Also perform appropriate rounding and redaction.
+ *
+ * This function is called once every 24 hours.
+ */
+#define MIN_CELL_COUNTS_TO_PUBLISH 1
+#define ROUND_CELL_COUNTS_TO 10000
+void
+rep_hist_prep_published_padding_counts(time_t now)
+{
+  memcpy(&padding_published, &padding_current, sizeof(padding_published));
+
+  if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH ||
+      padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) {
+    memset(&padding_published, 0, sizeof(padding_published));
+    return;
+  }
+
+  format_iso_time(padding_published.first_published_at, now);
+#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
+                                      ROUND_CELL_COUNTS_TO)
+  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
+  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
+#undef ROUND_AND_SET_COUNT
+}
+
+/**
+ * Returns an allocated string for extra-info documents for publishing
+ * padding statistics from the last 24 hour interval.
+ */
+char *
+rep_hist_get_padding_count_lines(void)
+{
+  char *result = NULL;
+
+  if (!padding_published.read_cell_count ||
+          !padding_published.write_cell_count) {
+    return NULL;
+  }
+
+  tor_asprintf(&result, "padding-counts %s (%d s)"
+                        " bin-size="U64_FORMAT
+                        " write-drop="U64_FORMAT
+                        " write-pad="U64_FORMAT
+                        " write-total="U64_FORMAT
+                        " read-drop="U64_FORMAT
+                        " read-pad="U64_FORMAT
+                        " read-total="U64_FORMAT
+                        " enabled-read-pad="U64_FORMAT
+                        " enabled-read-total="U64_FORMAT
+                        " enabled-write-pad="U64_FORMAT
+                        " enabled-write-total="U64_FORMAT
+                        " max-chanpad-timers="U64_FORMAT
+                        "\n",
+               padding_published.first_published_at,
+               REPHIST_CELL_PADDING_COUNTS_INTERVAL,
+               U64_PRINTF_ARG(ROUND_CELL_COUNTS_TO),
+               U64_PRINTF_ARG(padding_published.write_drop_cell_count),
+               U64_PRINTF_ARG(padding_published.write_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.write_cell_count),
+               U64_PRINTF_ARG(padding_published.read_drop_cell_count),
+               U64_PRINTF_ARG(padding_published.read_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.read_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_read_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_read_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_write_pad_cell_count),
+               U64_PRINTF_ARG(padding_published.enabled_write_cell_count),
+               U64_PRINTF_ARG(padding_published.maximum_chanpad_timers)
+               );
+
+  return result;
+}
+
 /** Log a heartbeat message explaining how many connections of each link
  * protocol version we have used.
  */
diff --git a/src/or/rephist.h b/src/or/rephist.h
index ff4810a56d..2b1c2e7ec7 100644
--- a/src/or/rephist.h
+++ b/src/or/rephist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,6 +48,7 @@ double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when);
 long rep_hist_get_weighted_time_known(const char *id, time_t when);
 int rep_hist_have_measured_enough_stability(void);
 
+void predicted_ports_init(void);
 void rep_hist_note_used_port(time_t now, uint16_t port);
 smartlist_t *rep_hist_get_predicted_ports(time_t now);
 void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports);
@@ -59,6 +60,7 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime,
 
 int any_predicted_circuits(time_t now);
 int rep_hist_circbuilding_dormant(time_t now);
+int predicted_ports_prediction_time_remaining(time_t now);
 
 void note_crypto_pk_op(pk_op_t operation);
 void dump_pk_ops(int severity);
@@ -119,5 +121,30 @@ extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1];
 extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1];
 #endif
 
+/**
+ * Represents the type of a cell for padding accounting
+ */
+typedef enum padding_type_t {
+    /** A RELAY_DROP cell */
+    PADDING_TYPE_DROP,
+    /** A CELL_PADDING cell */
+    PADDING_TYPE_CELL,
+    /** Total counts of padding and non-padding together */
+    PADDING_TYPE_TOTAL,
+    /** Total cell counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_TOTAL,
+    /** CELL_PADDING counts for all padding-enabled channels */
+    PADDING_TYPE_ENABLED_CELL
+} padding_type_t;
+
+/** The amount of time over which the padding cell counts were counted */
+#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60)
+void rep_hist_padding_count_read(padding_type_t type);
+void rep_hist_padding_count_write(padding_type_t type);
+char *rep_hist_get_padding_count_lines(void);
+void rep_hist_reset_padding_counts(void);
+void rep_hist_prep_published_padding_counts(time_t now);
+void rep_hist_padding_count_timers(uint64_t num_timers);
+
 #endif
 
diff --git a/src/or/replaycache.c b/src/or/replaycache.c
index 8290fa6964..3d42deb90a 100644
--- a/src/or/replaycache.c
+++ b/src/or/replaycache.c
@@ -1,4 +1,4 @@
- /* Copyright (c) 2012-2016, The Tor Project, Inc. */
+ /* Copyright (c) 2012-2017, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/replaycache.h b/src/or/replaycache.h
index 64a6caf5f5..0d637939a4 100644
--- a/src/or/replaycache.h
+++ b/src/or/replaycache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/router.c b/src/or/router.c
index 31f2ff00d2..5405d31260 100644
--- a/src/or/router.c
+++ b/src/or/router.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTER_PRIVATE
@@ -131,7 +131,8 @@ get_onion_key(void)
 }
 
 /** Store a full copy of the current onion key into *<b>key</b>, and a full
- * copy of the most recent onion key into *<b>last</b>.
+ * copy of the most recent onion key into *<b>last</b>.  Store NULL into
+ * a pointer if the corresponding key does not exist.
  */
 void
 dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
@@ -139,8 +140,10 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
   tor_assert(key);
   tor_assert(last);
   tor_mutex_acquire(key_lock);
-  tor_assert(onionkey);
-  *key = crypto_pk_copy_full(onionkey);
+  if (onionkey)
+    *key = crypto_pk_copy_full(onionkey);
+  else
+    *key = NULL;
   if (lastonionkey)
     *last = crypto_pk_copy_full(lastonionkey);
   else
@@ -148,6 +151,51 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
   tor_mutex_release(key_lock);
 }
 
+/** Expire our old set of onion keys. This is done by setting
+ * last_curve25519_onion_key and lastonionkey to all zero's and NULL
+ * respectively.
+ *
+ * This function does not perform any grace period checks for the old onion
+ * keys.
+ */
+void
+expire_old_onion_keys(void)
+{
+  char *fname = NULL;
+
+  tor_mutex_acquire(key_lock);
+
+  /* Free lastonionkey and set it to NULL. */
+  if (lastonionkey) {
+    crypto_pk_free(lastonionkey);
+    lastonionkey = NULL;
+  }
+
+  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+   * construct_ntor_key_map() below. */
+  memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+  tor_mutex_release(key_lock);
+
+  fname = get_datadir_fname2("keys", "secret_onion_key.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+
+  fname = get_datadir_fname2("keys", "secret_onion_key_ntor.old");
+  if (file_status(fname) == FN_FILE) {
+    if (tor_unlink(fname) != 0) {
+      log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s",
+               fname, strerror(errno));
+    }
+  }
+  tor_free(fname);
+}
+
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
 static const curve25519_keypair_t *
@@ -162,10 +210,14 @@ construct_ntor_key_map(void)
 {
   di_digest256_map_t *m = NULL;
 
-  dimap_add_entry(&m,
-                  curve25519_onion_key.pubkey.public_key,
-                  tor_memdup(&curve25519_onion_key,
-                             sizeof(curve25519_keypair_t)));
+  if (!tor_mem_is_zero((const char*)
+                       curve25519_onion_key.pubkey.public_key,
+                       CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m,
+                    curve25519_onion_key.pubkey.public_key,
+                    tor_memdup(&curve25519_onion_key,
+                               sizeof(curve25519_keypair_t)));
+  }
   if (!tor_mem_is_zero((const char*)
                           last_curve25519_onion_key.pubkey.public_key,
                        CURVE25519_PUBKEY_LEN)) {
@@ -212,7 +264,11 @@ set_server_identity_key(crypto_pk_t *k)
 {
   crypto_pk_free(server_identitykey);
   server_identitykey = k;
-  crypto_pk_get_digest(server_identitykey, server_identitykey_digest);
+  if (crypto_pk_get_digest(server_identitykey,
+                           server_identitykey_digest) < 0) {
+    log_err(LD_BUG, "Couldn't compute our own identity key digest.");
+    tor_assert(0);
+  }
 }
 
 /** Make sure that we have set up our identity keys to match or not match as
@@ -683,6 +739,47 @@ v3_authority_check_key_expiry(void)
   last_warned = now;
 }
 
+/** Get the lifetime of an onion key in days. This value is defined by the
+ * network consesus parameter "onion-key-rotation-days". Always returns a value
+ * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
+ */
+static int
+get_onion_key_rotation_days_(void)
+{
+  return networkstatus_get_param(NULL,
+                                 "onion-key-rotation-days",
+                                 DEFAULT_ONION_KEY_LIFETIME_DAYS,
+                                 MIN_ONION_KEY_LIFETIME_DAYS,
+                                 MAX_ONION_KEY_LIFETIME_DAYS);
+}
+
+/** Get the current lifetime of an onion key in seconds. This value is defined
+ * by the network consesus parameter "onion-key-rotation-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_lifetime(void)
+{
+  return get_onion_key_rotation_days_()*24*60*60;
+}
+
+/** Get the grace period of an onion key in seconds. This value is defined by
+ * the network consesus parameter "onion-key-grace-period-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_grace_period(void)
+{
+  int grace_period;
+  grace_period = networkstatus_get_param(NULL,
+                                         "onion-key-grace-period-days",
+                                         DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         MIN_ONION_KEY_GRACE_PERIOD_DAYS,
+                                         get_onion_key_rotation_days_());
+  return grace_period*24*60*60;
+}
+
 /** Set up Tor's TLS contexts, based on our configuration and keys. Return 0
  * on success, and -1 on failure. */
 int
@@ -693,12 +790,6 @@ router_initialize_tls_context(void)
   int lifetime = options->SSLKeyLifetime;
   if (public_server_mode(options))
     flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER;
-  if (options->TLSECGroup) {
-    if (!strcasecmp(options->TLSECGroup, "P256"))
-      flags |= TOR_TLS_CTX_USE_ECDHE_P256;
-    else if (!strcasecmp(options->TLSECGroup, "P224"))
-      flags |= TOR_TLS_CTX_USE_ECDHE_P224;
-  }
   if (!lifetime) { /* we should guess a good ssl cert lifetime */
 
     /* choose between 5 and 365 days, and round to the day */
@@ -849,7 +940,12 @@ init_keys(void)
   if (init_keys_common() < 0)
     return -1;
   /* Make sure DataDirectory exists, and is private. */
-  if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) {
+  cpd_check_t cpd_opts = CPD_CREATE;
+  if (options->DataDirectoryGroupReadable)
+    cpd_opts |= CPD_GROUP_READ;
+  if (check_private_dir(options->DataDirectory, cpd_opts, options->User)) {
+    log_err(LD_OR, "Can't create/check datadirectory %s",
+            options->DataDirectory);
     return -1;
   }
   /* Check the key directory. */
@@ -871,8 +967,12 @@ init_keys(void)
     }
     cert = get_my_v3_authority_cert();
     if (cert) {
-      crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
-                           v3_digest);
+      if (crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
+                               v3_digest) < 0) {
+        log_err(LD_BUG, "Couldn't compute my v3 authority identity key "
+                "digest.");
+        return -1;
+      }
       v3_digest_set = 1;
     }
   }
@@ -901,7 +1001,8 @@ init_keys(void)
   }
 
   /* 1d. Load all ed25519 keys */
-  if (load_ed_keys(options,now) < 0)
+  const int new_signing_key = load_ed_keys(options,now);
+  if (new_signing_key < 0)
     return -1;
 
   /* 2. Read onion key.  Make it if none is found. */
@@ -923,7 +1024,7 @@ init_keys(void)
       /* We have no LastRotatedOnionKey set; either we just created the key
        * or it's a holdover from 0.1.2.4-alpha-dev or earlier.  In either case,
        * start the clock ticking now so that we will eventually rotate it even
-       * if we don't stay up for a full MIN_ONION_KEY_LIFETIME. */
+       * if we don't stay up for the full lifetime of an onion key. */
       state->LastRotatedOnionKey = onionkey_set_at = now;
       or_state_mark_dirty(state, options->AvoidDiskWrites ?
                                    time(NULL)+3600 : 0);
@@ -971,7 +1072,7 @@ init_keys(void)
 
   /* 3b. Get an ed25519 link certificate.  Note that we need to do this
    * after we set up the TLS context */
-  if (generate_ed_link_cert(options, now) < 0) {
+  if (generate_ed_link_cert(options, now, new_signing_key > 0) < 0) {
     log_err(LD_GENERAL,"Couldn't make link cert");
     return -1;
   }
@@ -1178,9 +1279,9 @@ router_should_be_directory_server(const or_options_t *options, int dir_port)
   if (accounting_is_enabled(options) &&
     get_options()->AccountingRule != ACCT_IN) {
     /* Don't spend bytes for directory traffic if we could end up hibernating,
-     * but allow DirPort otherwise. Some people set AccountingMax because
-     * they're confused or to get statistics. Directory traffic has a much
-     * larger effect on output than input so there is no reason to turn it
+     * but allow DirPort otherwise. Some relay operators set AccountingMax
+     * because they're confused or to get statistics. Directory traffic has a
+     * much larger effect on output than input so there is no reason to turn it
      * off if using AccountingRule in. */
     int interval_length = accounting_get_interval_length();
     uint32_t effective_bw = get_effective_bwrate(options);
@@ -1312,8 +1413,15 @@ extend_info_from_router(const routerinfo_t *r)
   /* Make sure we don't need to check address reachability */
   tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0));
 
+  const ed25519_public_key_t *ed_id_key;
+  if (r->cache_info.signing_key_cert)
+    ed_id_key = &r->cache_info.signing_key_cert->signing_key;
+  else
+    ed_id_key = NULL;
+
   router_get_prim_orport(r, &ap);
   return extend_info_new(r->nickname, r->cache_info.identity_digest,
+                         ed_id_key,
                          r->onion_pkey, r->onion_curve25519_pkey,
                          &ap.addr, ap.port);
 }
@@ -1372,13 +1480,23 @@ consider_testing_reachability(int test_or, int test_dir)
       !connection_get_by_type_addr_port_purpose(
                 CONN_TYPE_DIR, &addr, me->dir_port,
                 DIR_PURPOSE_FETCH_SERVERDESC)) {
+    tor_addr_port_t my_orport, my_dirport;
+    memcpy(&my_orport.addr, &addr, sizeof(addr));
+    memcpy(&my_dirport.addr, &addr, sizeof(addr));
+    my_orport.port = me->or_port;
+    my_dirport.port = me->dir_port;
     /* ask myself, via tor, for my server descriptor. */
-    directory_initiate_command(&addr, me->or_port,
-                               &addr, me->dir_port,
-                               me->cache_info.identity_digest,
-                               DIR_PURPOSE_FETCH_SERVERDESC,
-                               ROUTER_PURPOSE_GENERAL,
-                               DIRIND_ANON_DIRPORT, "authority.z", NULL, 0, 0);
+    directory_request_t *req =
+      directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+    directory_request_set_or_addr_port(req, &my_orport);
+    directory_request_set_dir_addr_port(req, &my_dirport);
+    directory_request_set_directory_id_digest(req,
+                                              me->cache_info.identity_digest);
+    // ask via an anon circuit, connecting to our dirport.
+    directory_request_set_indirection(req, DIRIND_ANON_DIRPORT);
+    directory_request_set_resource(req, "authority.z");
+    directory_initiate_request(req);
+    directory_request_free(req);
   }
 }
 
@@ -1555,8 +1673,7 @@ MOCK_IMPL(int,
 server_mode,(const or_options_t *options))
 {
   if (options->ClientOnly) return 0;
-  /* XXXX I believe we can kill off ORListenAddress here.*/
-  return (options->ORPort_set || options->ORListenAddress);
+  return (options->ORPort_set);
 }
 
 /** Return true iff we are trying to be a non-bridge server.
@@ -2180,17 +2297,15 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   }
 
   if (options->MyFamily && ! options->BridgeRelay) {
-    smartlist_t *family;
     if (!warned_nonexistent_family)
       warned_nonexistent_family = smartlist_new();
-    family = smartlist_new();
     ri->declared_family = smartlist_new();
-    smartlist_split_string(family, options->MyFamily, ",",
-      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
-    SMARTLIST_FOREACH_BEGIN(family, char *, name) {
+    config_line_t *family;
+    for (family = options->MyFamily; family; family = family->next) {
+       char *name = family->value;
        const node_t *member;
        if (!strcasecmp(name, options->Nickname))
-         goto skip; /* Don't list ourself, that's redundant */
+         continue; /* Don't list ourself, that's redundant */
        else
          member = node_get_by_nickname(name, 1);
        if (!member) {
@@ -2206,11 +2321,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
              log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
                       "declared family, but that isn't a legal nickname. "
                       "Skipping it.", escaped(name));
-           smartlist_add(warned_nonexistent_family, tor_strdup(name));
+           smartlist_add_strdup(warned_nonexistent_family, name);
          }
          if (is_legal) {
-           smartlist_add(ri->declared_family, name);
-           name = NULL;
+           smartlist_add_strdup(ri->declared_family, name);
          }
        } else if (router_digest_is_me(member->identity)) {
          /* Don't list ourself in our own family; that's redundant */
@@ -2224,15 +2338,11 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
          if (smartlist_contains_string(warned_nonexistent_family, name))
            smartlist_string_remove(warned_nonexistent_family, name);
        }
-    skip:
-       tor_free(name);
-    } SMARTLIST_FOREACH_END(name);
+    }
 
     /* remove duplicates from the list */
     smartlist_sort_strings(ri->declared_family);
     smartlist_uniq_strings(ri->declared_family);
-
-    smartlist_free(family);
   }
 
   /* Now generate the extrainfo. */
@@ -2748,7 +2858,7 @@ router_dump_router_to_string(routerinfo_t *router,
       make_ntor_onion_key_crosscert(ntor_keypair,
                          &router->cache_info.signing_key_cert->signing_key,
                          router->cache_info.published_on,
-                         MIN_ONION_KEY_LIFETIME, &sign);
+                         get_onion_key_lifetime(), &sign);
     if (!cert) {
       log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!");
       goto err;
@@ -2834,7 +2944,7 @@ router_dump_router_to_string(routerinfo_t *router,
                     "onion-key\n%s"
                     "signing-key\n%s"
                     "%s%s"
-                    "%s%s%s%s",
+                    "%s%s%s",
     router->nickname,
     address,
     router->or_port,
@@ -2857,8 +2967,7 @@ router_dump_router_to_string(routerinfo_t *router,
     ntor_cc_line ? ntor_cc_line : "",
     family_line,
     we_are_hibernating() ? "hibernating 1\n" : "",
-    "hidden-service-dir\n",
-    options->AllowSingleHopExits ? "allow-single-hop-exits\n" : "");
+    "hidden-service-dir\n");
 
   if (options->ContactInfo && strlen(options->ContactInfo)) {
     const char *ci = options->ContactInfo;
@@ -2890,7 +2999,7 @@ router_dump_router_to_string(routerinfo_t *router,
 
   /* Write the exit policy to the end of 's'. */
   if (!router->exit_policy || !smartlist_len(router->exit_policy)) {
-    smartlist_add(chunks, tor_strdup("reject *:*\n"));
+    smartlist_add_strdup(chunks, "reject *:*\n");
   } else if (router->exit_policy) {
     char *exit_policy = router_dump_exit_policy_to_string(router,1,0);
 
@@ -2912,12 +3021,12 @@ router_dump_router_to_string(routerinfo_t *router,
 
   if (decide_to_advertise_begindir(options,
                                    router->supports_tunnelled_dir_requests)) {
-    smartlist_add(chunks, tor_strdup("tunnelled-dir-server\n"));
+    smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
   }
 
   /* Sign the descriptor with Ed25519 */
   if (emit_ed_sigs)  {
-    smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
     crypto_digest_smartlist_prefix(digest, DIGEST256_LEN,
                                    ED_DESC_SIGNATURE_PREFIX,
                                    chunks, "", DIGEST_SHA256);
@@ -2933,7 +3042,7 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* Sign the descriptor with RSA */
-  smartlist_add(chunks, tor_strdup("router-signature\n"));
+  smartlist_add_strdup(chunks, "router-signature\n");
 
   crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1);
 
@@ -2948,7 +3057,7 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   /* include a last '\n' */
-  smartlist_add(chunks, tor_strdup("\n"));
+  smartlist_add_strdup(chunks, "\n");
 
   output = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3196,6 +3305,12 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     }
   }
 
+  if (options->PaddingStatistics) {
+    contents = rep_hist_get_padding_count_lines();
+    if (contents)
+      smartlist_add(chunks, contents);
+  }
+
   /* Add information about the pluggable transports we support. */
   if (options->ServerTransportPlugin) {
     char *pluggable_transports = pt_get_extra_info_descriptor_string();
@@ -3206,13 +3321,13 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
     const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
     if (bridge_stats) {
-      smartlist_add(chunks, tor_strdup(bridge_stats));
+      smartlist_add_strdup(chunks, bridge_stats);
     }
   }
 
   if (emit_ed_sigs) {
     char sha256_digest[DIGEST256_LEN];
-    smartlist_add(chunks, tor_strdup("router-sig-ed25519 "));
+    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
     crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
                                    ED_DESC_SIGNATURE_PREFIX,
                                    chunks, "", DIGEST_SHA256);
@@ -3227,7 +3342,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
-  smartlist_add(chunks, tor_strdup("router-signature\n"));
+  smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
@@ -3262,7 +3377,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
                      "descriptor.");
     goto err;
   }
-  smartlist_add(chunks, tor_strdup(sig));
+  smartlist_add_strdup(chunks, sig);
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
diff --git a/src/or/router.h b/src/or/router.h
index c30a0301b7..9c5def5218 100644
--- a/src/or/router.h
+++ b/src/or/router.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,10 +27,13 @@ crypto_pk_t *get_my_v3_authority_signing_key(void);
 authority_cert_t *get_my_v3_legacy_cert(void);
 crypto_pk_t *get_my_v3_legacy_signing_key(void);
 void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last);
+void expire_old_onion_keys(void);
 void rotate_onion_key(void);
 crypto_pk_t *init_key_from_file(const char *fname, int generate,
                                     int severity, int log_greeting);
 void v3_authority_check_key_expiry(void);
+int get_onion_key_lifetime(void);
+int get_onion_key_grace_period(void);
 
 di_digest256_map_t *construct_ntor_key_map(void);
 void ntor_key_map_free(di_digest256_map_t *map);
diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c
index ca32228fc7..fd4c6ce0dd 100644
--- a/src/or/routerkeys.c
+++ b/src/or/routerkeys.c
@@ -1,12 +1,17 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file routerkeys.c
  *
  * \brief Functions and structures to handle generating and maintaining the
- *  set of keypairs necessary to be an OR. (Some of the code in router.c
- *  belongs here.)
+ *  set of keypairs necessary to be an OR.
+ *
+ * The keys handled here now are the Ed25519 keys that Tor relays use to sign
+ * descriptors, authenticate themselves on links, and identify one another
+ * uniquely.  Other keys are maintained in router.c and rendservice.c.
+ *
+ * (TODO: The keys in router.c should go here too.)
  */
 
 #include "or.h"
@@ -19,6 +24,7 @@
 #define ENC_KEY_HEADER "Boxed Ed25519 key"
 #define ENC_KEY_TAG "master"
 
+/* DOCDOC */
 static ssize_t
 do_getpass(const char *prompt, char *buf, size_t buflen,
            int twice, const or_options_t *options)
@@ -85,6 +91,7 @@ do_getpass(const char *prompt, char *buf, size_t buflen,
   return length;
 }
 
+/* DOCDOC */
 int
 read_encrypted_secret_key(ed25519_secret_key_t *out,
                           const char *fname)
@@ -157,6 +164,7 @@ read_encrypted_secret_key(ed25519_secret_key_t *out,
   return r;
 }
 
+/* DOCDOC */
 int
 write_encrypted_secret_key(const ed25519_secret_key_t *key,
                            const char *fname)
@@ -200,6 +208,7 @@ write_encrypted_secret_key(const ed25519_secret_key_t *key,
   return r;
 }
 
+/* DOCDOC */
 static int
 write_secret_key(const ed25519_secret_key_t *key, int encrypted,
                  const char *fname,
@@ -659,10 +668,14 @@ static tor_cert_t *auth_key_cert = NULL;
 
 static uint8_t *rsa_ed_crosscert = NULL;
 static size_t rsa_ed_crosscert_len = 0;
+static time_t rsa_ed_crosscert_expiration = 0;
 
 /**
  * Running as a server: load, reload, or refresh our ed25519 keys and
  * certificates, creating and saving new ones as needed.
+ *
+ * Return -1 on failure; 0 on success if the signing key was not replaced;
+ * and 1 on success if the signing key was replaced.
  */
 int
 load_ed_keys(const or_options_t *options, time_t now)
@@ -675,6 +688,11 @@ load_ed_keys(const or_options_t *options, time_t now)
   const tor_cert_t *check_signing_cert = NULL;
   tor_cert_t *sign_cert = NULL;
   tor_cert_t *auth_cert = NULL;
+  int signing_key_changed = 0;
+
+  // It is later than 1972, since otherwise there would be no C compilers.
+  // (Try to diagnose #22466.)
+  tor_assert_nonfatal(now >= 2 * 365 * 86400);
 
 #define FAIL(msg) do {                          \
     log_warn(LD_OR, (msg));                     \
@@ -690,8 +708,10 @@ load_ed_keys(const or_options_t *options, time_t now)
       tor_cert_free(cert);                      \
     cert = (newval);                            \
   } while (0)
+#define HAPPENS_SOON(when, interval)            \
+  ((when) < now + (interval))
 #define EXPIRES_SOON(cert, interval)            \
-  (!(cert) || (cert)->valid_until < now + (interval))
+  (!(cert) || HAPPENS_SOON((cert)->valid_until, (interval)))
 
   /* XXXX support encrypted identity keys fully */
 
@@ -710,7 +730,23 @@ load_ed_keys(const or_options_t *options, time_t now)
     use_signing = sign;
   }
 
+  if (use_signing) {
+    /* We loaded a signing key with its certificate.  */
+    if (! master_signing_key) {
+      /* We didn't know one before! */
+      signing_key_changed = 1;
+    } else if (! ed25519_pubkey_eq(&use_signing->pubkey,
+                                   &master_signing_key->pubkey) ||
+               ! tor_memeq(use_signing->seckey.seckey,
+                           master_signing_key->seckey.seckey,
+                           ED25519_SECKEY_LEN)) {
+      /* We loaded a different signing key than the one we knew before. */
+      signing_key_changed = 1;
+    }
+  }
+
   if (!use_signing && master_signing_key) {
+    /* We couldn't load a signing key, but we already had one loaded */
     check_signing_cert = signing_key_cert;
     use_signing = master_signing_key;
   }
@@ -733,8 +769,12 @@ load_ed_keys(const or_options_t *options, time_t now)
 
   if (need_new_signing_key) {
     log_notice(LD_OR, "It looks like I need to generate and sign a new "
-               "medium-term signing key, because %s. To do that, I need to "
-               "load%s the permanent master identity key.",
+               "medium-term signing key, because %s. To do that, I "
+               "need to load%s the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.",
             (NULL == use_signing) ? "I don't have one" :
             EXPIRES_SOON(check_signing_cert, 0) ? "the one I have is expired" :
                "you asked me to make one with --keygen",
@@ -742,15 +782,19 @@ load_ed_keys(const or_options_t *options, time_t now)
   } else if (want_new_signing_key && !offline_master) {
     log_notice(LD_OR, "It looks like I should try to generate and sign a "
                "new medium-term signing key, because the one I have is "
-               "going to expire soon. To do that, I'm going to have to try to "
-               "load the permanent master identity key.");
+               "going to expire soon. To do that, I'm going to have to "
+               "try to load the permanent master identity key. "
+               "If the master identity key was not moved or encrypted "
+               "with a passphrase, this will be done automatically and "
+               "no further action is required. Otherwise, provide the "
+               "necessary data using 'tor --keygen' to do it manually.");
   } else if (want_new_signing_key) {
     log_notice(LD_OR, "It looks like I should try to generate and sign a "
                "new medium-term signing key, because the one I have is "
                "going to expire soon. But OfflineMasterKey is set, so I "
-               "won't try to load a permanent master identity key is set. "
-               "You will need to use 'tor --keygen' make a new signing key "
-               "and certificate.");
+               "won't try to load a permanent master identity key. You "
+               "will need to use 'tor --keygen' to make a new signing "
+               "key and certificate.");
   }
 
   {
@@ -768,8 +812,11 @@ load_ed_keys(const or_options_t *options, time_t now)
     if (options->command == CMD_KEYGEN)
       flags |= INIT_ED_KEY_TRY_ENCRYPTED;
 
-    /* Check the key directory */
-    if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) {
+    /* Check/Create the key directory */
+    cpd_check_t cpd_opts = CPD_CREATE;
+    if (options->DataDirectoryGroupReadable)
+      cpd_opts |= CPD_GROUP_READ;
+    if (check_private_dir(options->DataDirectory, cpd_opts, options->User)) {
       log_err(LD_OR, "Can't create/check datadirectory %s",
               options->DataDirectory);
       goto err;
@@ -859,6 +906,7 @@ load_ed_keys(const or_options_t *options, time_t now)
     if (!sign)
       FAIL("Missing signing key");
     use_signing = sign;
+    signing_key_changed = 1;
 
     tor_assert(sign_cert->signing_key_included);
     tor_assert(ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey));
@@ -879,17 +927,23 @@ load_ed_keys(const or_options_t *options, time_t now)
   if (options->command == CMD_KEYGEN)
     goto end;
 
-  if (!rsa_ed_crosscert && server_mode(options)) {
+  if (server_mode(options) &&
+      (!rsa_ed_crosscert ||
+       HAPPENS_SOON(rsa_ed_crosscert_expiration, 30*86400))) {
     uint8_t *crosscert;
+    time_t expiration = now+6*30*86400; /* 6 months in the future. */
     ssize_t crosscert_len = tor_make_rsa_ed25519_crosscert(&id->pubkey,
                                                    get_server_identity_key(),
-                                                   now+10*365*86400,/*XXXX*/
+                                                   expiration,
                                                    &crosscert);
+    tor_free(rsa_ed_crosscert);
     rsa_ed_crosscert_len = crosscert_len;
     rsa_ed_crosscert = crosscert;
+    rsa_ed_crosscert_expiration = expiration;
   }
 
   if (!current_auth_key ||
+      signing_key_changed ||
       EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop)) {
     auth = ed_key_new(use_signing, INIT_ED_KEY_NEEDCERT,
                       now,
@@ -917,7 +971,7 @@ load_ed_keys(const or_options_t *options, time_t now)
     SET_CERT(auth_key_cert, auth_cert);
   }
 
-  return 0;
+  return signing_key_changed;
  err:
   ed25519_keypair_free(id);
   ed25519_keypair_free(sign);
@@ -927,21 +981,39 @@ load_ed_keys(const or_options_t *options, time_t now)
   return -1;
 }
 
-/* DOCDOC */
+/**
+ * Retrieve our currently-in-use Ed25519 link certificate and id certificate,
+ * and, if they would expire soon (based on the time <b>now</b>, generate new
+ * certificates (without embedding the public part of the signing key inside).
+ * If <b>force</b> is true, always generate a new certificate.
+ *
+ * The signed_key from the current id->signing certificate will be used to
+ * sign the new key within newly generated X509 certificate.
+ *
+ * Returns -1 upon error.  Otherwise, returns 0 upon success (either when the
+ * current certificate is still valid, or when a new certificate was
+ * successfully generated, or no certificate was needed).
+ */
 int
-generate_ed_link_cert(const or_options_t *options, time_t now)
+generate_ed_link_cert(const or_options_t *options, time_t now,
+                      int force)
 {
   const tor_x509_cert_t *link_ = NULL, *id = NULL;
   tor_cert_t *link_cert = NULL;
 
   if (tor_tls_get_my_certs(1, &link_, &id) < 0 || link_ == NULL) {
+    if (!server_mode(options)) {
+        /* No need to make an Ed25519->Link cert: we are a client */
+      return 0;
+    }
     log_warn(LD_OR, "Can't get my x509 link cert.");
     return -1;
   }
 
   const common_digests_t *digests = tor_x509_cert_get_cert_digests(link_);
 
-  if (link_cert_cert &&
+  if (force == 0 &&
+      link_cert_cert &&
       ! EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop) &&
       fast_memeq(digests->d[DIGEST_SHA256], link_cert_cert->signed_key.pubkey,
                  DIGEST256_LEN)) {
@@ -967,6 +1039,17 @@ generate_ed_link_cert(const or_options_t *options, time_t now)
 #undef SET_KEY
 #undef SET_CERT
 
+/**
+ * Return 1 if any of the following are true:
+ *
+ *   - if one of our Ed25519 signing, auth, or link certificates would expire
+ *     soon w.r.t. the time <b>now</b>,
+ *   - if we do not currently have a link certificate, or
+ *   - if our cached Ed25519 link certificate is not same as the one we're
+ *     currently using.
+ *
+ * Otherwise, returns 0.
+ */
 int
 should_make_new_ed_keys(const or_options_t *options, const time_t now)
 {
@@ -996,6 +1079,62 @@ should_make_new_ed_keys(const or_options_t *options, const time_t now)
 }
 
 #undef EXPIRES_SOON
+#undef HAPPENS_SOON
+
+#ifdef TOR_UNIT_TESTS
+/* Helper for unit tests: populate the ed25519 keys without saving or
+ * loading */
+void
+init_mock_ed_keys(const crypto_pk_t *rsa_identity_key)
+{
+  routerkeys_free_all();
+
+#define MAKEKEY(k)                                      \
+  k = tor_malloc_zero(sizeof(*k));                      \
+  if (ed25519_keypair_generate(k, 0) < 0) {             \
+    log_warn(LD_BUG, "Couldn't make a keypair");        \
+    goto err;                                           \
+  }
+  MAKEKEY(master_identity_key);
+  MAKEKEY(master_signing_key);
+  MAKEKEY(current_auth_key);
+#define MAKECERT(cert, signing, signed_, type, flags)            \
+  cert = tor_cert_create(signing,                                \
+                         type,                                   \
+                         &signed_->pubkey,                       \
+                         time(NULL), 86400,                      \
+                         flags);                                 \
+  if (!cert) {                                                   \
+    log_warn(LD_BUG, "Couldn't make a %s certificate!", #cert);  \
+    goto err;                                                    \
+  }
+
+  MAKECERT(signing_key_cert,
+           master_identity_key, master_signing_key, CERT_TYPE_ID_SIGNING,
+           CERT_FLAG_INCLUDE_SIGNING_KEY);
+  MAKECERT(auth_key_cert,
+           master_signing_key, current_auth_key, CERT_TYPE_SIGNING_AUTH, 0);
+
+  if (generate_ed_link_cert(get_options(), time(NULL), 0) < 0) {
+    log_warn(LD_BUG, "Couldn't make link certificate");
+    goto err;
+  }
+
+  rsa_ed_crosscert_len = tor_make_rsa_ed25519_crosscert(
+                                     &master_identity_key->pubkey,
+                                     rsa_identity_key,
+                                     time(NULL)+86400,
+                                     &rsa_ed_crosscert);
+
+  return;
+
+ err:
+  routerkeys_free_all();
+  tor_assert_nonfatal_unreached();
+}
+#undef MAKEKEY
+#undef MAKECERT
+#endif
 
 const ed25519_public_key_t *
 get_master_identity_key(void)
@@ -1005,6 +1144,24 @@ get_master_identity_key(void)
   return &master_identity_key->pubkey;
 }
 
+/** Return true iff <b>id</b> is our Ed25519 master identity key. */
+int
+router_ed25519_id_is_me(const ed25519_public_key_t *id)
+{
+  return id && master_identity_key &&
+    ed25519_pubkey_eq(id, &master_identity_key->pubkey);
+}
+
+#ifdef TOR_UNIT_TESTS
+/* only exists for the unit tests, since otherwise the identity key
+ * should be used to sign nothing but the signing key. */
+const ed25519_keypair_t *
+get_master_identity_keypair(void)
+{
+  return master_identity_key;
+}
+#endif
+
 const ed25519_keypair_t *
 get_master_signing_keypair(void)
 {
@@ -1079,7 +1236,9 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
   uint8_t signed_data[DIGEST_LEN + ED25519_PUBKEY_LEN];
 
   *len_out = 0;
-  crypto_pk_get_digest(rsa_id_key, (char*)signed_data);
+  if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    return NULL;
+  }
   memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
 
   int r = crypto_pk_private_sign(onion_key,
@@ -1095,12 +1254,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
 
 /** Check whether an RSA-TAP cross-certification is correct. Return 0 if it
  * is, -1 if it isn't. */
-int
-check_tap_onion_key_crosscert(const uint8_t *crosscert,
-                              int crosscert_len,
-                              const crypto_pk_t *onion_pkey,
-                              const ed25519_public_key_t *master_id_pkey,
-                              const uint8_t *rsa_id_digest)
+MOCK_IMPL(int,
+check_tap_onion_key_crosscert,(const uint8_t *crosscert,
+                               int crosscert_len,
+                               const crypto_pk_t *onion_pkey,
+                               const ed25519_public_key_t *master_id_pkey,
+                               const uint8_t *rsa_id_digest))
 {
   uint8_t *cc = tor_malloc(crypto_pk_keysize(onion_pkey));
   int cc_len =
@@ -1139,9 +1298,12 @@ routerkeys_free_all(void)
   tor_cert_free(signing_key_cert);
   tor_cert_free(link_cert_cert);
   tor_cert_free(auth_key_cert);
+  tor_free(rsa_ed_crosscert);
 
   master_identity_key = master_signing_key = NULL;
   current_auth_key = NULL;
   signing_key_cert = link_cert_cert = auth_key_cert = NULL;
+  rsa_ed_crosscert = NULL; // redundant
+  rsa_ed_crosscert_len = 0;
 }
 
diff --git a/src/or/routerkeys.h b/src/or/routerkeys.h
index be9b19aea8..c10cf32a71 100644
--- a/src/or/routerkeys.h
+++ b/src/or/routerkeys.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ROUTERKEYS_H
@@ -45,6 +45,8 @@ const struct tor_cert_st *get_current_auth_key_cert(void);
 void get_master_rsa_crosscert(const uint8_t **cert_out,
                               size_t *size_out);
 
+int router_ed25519_id_is_me(const ed25519_public_key_t *id);
+
 struct tor_cert_st *make_ntor_onion_key_crosscert(
                                   const curve25519_keypair_t *onion_key,
                                   const ed25519_public_key_t *master_id_key,
@@ -55,16 +57,16 @@ uint8_t *make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
                                   const crypto_pk_t *rsa_id_key,
                                   int *len_out);
 
-int check_tap_onion_key_crosscert(const uint8_t *crosscert,
+MOCK_DECL(int, check_tap_onion_key_crosscert,(const uint8_t *crosscert,
                                   int crosscert_len,
                                   const crypto_pk_t *onion_pkey,
                                   const ed25519_public_key_t *master_id_pkey,
-                                  const uint8_t *rsa_id_digest);
+                                  const uint8_t *rsa_id_digest));
 
 int load_ed_keys(const or_options_t *options, time_t now);
 int should_make_new_ed_keys(const or_options_t *options, const time_t now);
 
-int generate_ed_link_cert(const or_options_t *options, time_t now);
+int generate_ed_link_cert(const or_options_t *options, time_t now, int force);
 
 int read_encrypted_secret_key(ed25519_secret_key_t *out,
                               const char *fname);
@@ -73,5 +75,10 @@ int write_encrypted_secret_key(const ed25519_secret_key_t *out,
 
 void routerkeys_free_all(void);
 
+#ifdef TOR_UNIT_TESTS
+const ed25519_keypair_t *get_master_identity_keypair(void);
+void init_mock_ed_keys(const crypto_pk_t *rsa_identity_key);
+#endif
+
 #endif
 
diff --git a/src/or/routerlist.c b/src/or/routerlist.c
index f73ec9baa1..2ca5199f3c 100644
--- a/src/or/routerlist.c
+++ b/src/or/routerlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -93,6 +93,7 @@
 #define ROUTERLIST_PRIVATE
 #include "or.h"
 #include "backtrace.h"
+#include "bridges.h"
 #include "crypto_ed25519.h"
 #include "circuitstats.h"
 #include "config.h"
@@ -426,8 +427,8 @@ list_sk_digests_for_authority_id, (const char *digest))
  * download_status_t or NULL if none exists. */
 
 MOCK_IMPL(download_status_t *,
-  download_status_for_authority_id_and_sk,
-  (const char *id_digest, const char *sk_digest))
+download_status_for_authority_id_and_sk,(const char *id_digest,
+                                         const char *sk_digest))
 {
   download_status_t *dl = NULL;
   cert_list_t *cl = NULL;
@@ -587,7 +588,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
                "signing key %s", from_store ? "cached" : "downloaded",
                ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
     } else {
-      int adding = directory_caches_unknown_auth_certs(get_options());
+      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
       log_info(LD_DIR, "%s %s certificate for unrecognized directory "
                "authority with signing key %s",
                adding ? "Adding" : "Not adding",
@@ -930,7 +931,8 @@ authority_certs_fetch_resource_impl(const char *resource,
                                     const routerstatus_t *rs)
 {
   const or_options_t *options = get_options();
-  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0);
+  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
+                                            resource);
 
   /* Make sure bridge clients never connect to anything but a bridge */
   if (options->UseBridges) {
@@ -946,31 +948,34 @@ authority_certs_fetch_resource_impl(const char *resource,
   const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
                                                     : DIRIND_ONEHOP;
 
+  directory_request_t *req = NULL;
   /* If we've just downloaded a consensus from a bridge, re-use that
    * bridge */
-  if (options->UseBridges && node && !get_via_tor) {
+  if (options->UseBridges && node && node->ri && !get_via_tor) {
     /* clients always make OR connections to bridges */
     tor_addr_port_t or_ap;
     /* we are willing to use a non-preferred address if we need to */
     fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
                                          &or_ap);
-    directory_initiate_command(&or_ap.addr, or_ap.port,
-                               NULL, 0, /*no dirport*/
-                               dir_hint,
-                               DIR_PURPOSE_FETCH_CERTIFICATE,
-                               0,
-                               indirection,
-                               resource, NULL, 0, 0);
-    return;
-  }
 
-  if (rs) {
-    /* If we've just downloaded a consensus from a directory, re-use that
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_or_addr_port(req, &or_ap);
+    if (dir_hint)
+      directory_request_set_directory_id_digest(req, dir_hint);
+  } else if (rs) {
+    /* And if we've just downloaded a consensus from a directory, re-use that
      * directory */
-    directory_initiate_command_routerstatus(rs,
-                                            DIR_PURPOSE_FETCH_CERTIFICATE,
-                                            0, indirection, resource, NULL,
-                                            0, 0);
+    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
+    directory_request_set_routerstatus(req, rs);
+  }
+
+  if (req) {
+    /* We've set up a request object -- fill in the other request fields, and
+     * send the request.  */
+    directory_request_set_indirection(req, indirection);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
     return;
   }
 
@@ -1012,7 +1017,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
   char *resource = NULL;
   cert_list_t *cl;
   const or_options_t *options = get_options();
-  const int cache = directory_caches_unknown_auth_certs(options);
+  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
   fp_pair_t *fp_tmp = NULL;
   char id_digest_str[2*DIGEST_LEN+1];
   char sk_digest_str[2*DIGEST_LEN+1];
@@ -1084,9 +1089,10 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
       if (!smartlist_len(voter->sigs))
         continue; /* This authority never signed this consensus, so don't
                    * go looking for a cert with key digest 0000000000. */
-      if (!cache &&
+      if (!keep_unknown &&
           !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
-        continue; /* We are not a cache, and we don't know this authority.*/
+        continue; /* We don't want unknown certs, and we don't know this
+                   * authority.*/
 
       /*
        * If we don't know *any* cert for this authority, and a download by ID
@@ -1203,7 +1209,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
     int need_plus = 0;
     smartlist_t *fps = smartlist_new();
 
-    smartlist_add(fps, tor_strdup("fp/"));
+    smartlist_add_strdup(fps, "fp/");
 
     SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
       char *fp = NULL;
@@ -1243,7 +1249,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now,
     int need_plus = 0;
     smartlist_t *fp_pairs = smartlist_new();
 
-    smartlist_add(fp_pairs, tor_strdup("fp-sk/"));
+    smartlist_add_strdup(fp_pairs, "fp-sk/");
 
     SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
       char *fp_pair = NULL;
@@ -1824,43 +1830,24 @@ router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc,
   return 0;
 }
 
-/* Check if we already have a directory fetch from ds, for serverdesc
- * (including extrainfo) or microdesc documents.
+/* Check if we already have a directory fetch from the ipv4 or ipv6
+ * router, for serverdesc (including extrainfo) or microdesc documents.
  * If so, return 1, if not, return 0.
  */
 static int
-router_is_already_dir_fetching_ds(const dir_server_t *ds,
-                                  int serverdesc,
-                                  int microdesc)
+router_is_already_dir_fetching_(uint32_t ipv4_addr,
+                                const tor_addr_t *ipv6_addr,
+                                uint16_t dir_port,
+                                int serverdesc,
+                                int microdesc)
 {
   tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
 
   /* Assume IPv6 DirPort is the same as IPv4 DirPort */
-  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ds->addr);
-  ipv4_dir_ap.port = ds->dir_port;
-  tor_addr_copy(&ipv6_dir_ap.addr, &ds->ipv6_addr);
-  ipv6_dir_ap.port = ds->dir_port;
-
-  return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
-       || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
-}
-
-/* Check if we already have a directory fetch from rs, for serverdesc
- * (including extrainfo) or microdesc documents.
- * If so, return 1, if not, return 0.
- */
-static int
-router_is_already_dir_fetching_rs(const routerstatus_t *rs,
-                                  int serverdesc,
-                                  int microdesc)
-{
-  tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap;
-
-  /* Assume IPv6 DirPort is the same as IPv4 DirPort */
-  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, rs->addr);
-  ipv4_dir_ap.port = rs->dir_port;
-  tor_addr_copy(&ipv6_dir_ap.addr, &rs->ipv6_addr);
-  ipv6_dir_ap.port = rs->dir_port;
+  tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ipv4_addr);
+  ipv4_dir_ap.port = dir_port;
+  tor_addr_copy(&ipv6_dir_ap.addr, ipv6_addr);
+  ipv6_dir_ap.port = dir_port;
 
   return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc)
        || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc));
@@ -1952,6 +1939,21 @@ router_picked_poor_directory_log(const routerstatus_t *rs)
     }                                                                         \
   STMT_END
 
+/* Common code used in the loop within router_pick_directory_server_impl and
+ * router_pick_trusteddirserver_impl.
+ *
+ * Check if the given <b>identity</b> supports extrainfo. If not, skip further
+ * checks.
+ */
+#define SKIP_MISSING_TRUSTED_EXTRAINFO(type, identity)                        \
+  STMT_BEGIN                                                                  \
+    int is_trusted_extrainfo = router_digest_is_trusted_dir_type(             \
+                               (identity), EXTRAINFO_DIRINFO);                \
+    if (((type) & EXTRAINFO_DIRINFO) &&                                       \
+        !router_supports_extrainfo((identity), is_trusted_extrainfo))         \
+      continue;                                                               \
+  STMT_END
+
 /* When iterating through the routerlist, can OR address/port preference
  * and reachability checks be skipped?
  */
@@ -1998,7 +2000,6 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
   const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL);
   const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH);
   const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH);
-  const int for_guard = (flags & PDS_FOR_GUARD);
   int try_excluding = 1, n_excluded = 0, n_busy = 0;
   int try_ip_pref = 1;
 
@@ -2020,7 +2021,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
 
   /* Find all the running dirservers we know about. */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
-    int is_trusted, is_trusted_extrainfo;
+    int is_trusted;
     int is_overloaded;
     const routerstatus_t *status = node->rs;
     const country_t country = node->country;
@@ -2031,20 +2032,9 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
       continue;
     if (requireother && router_digest_is_me(node->identity))
       continue;
-    is_trusted = router_digest_is_trusted_dir(node->identity);
-    is_trusted_extrainfo = router_digest_is_trusted_dir_type(
-                           node->identity, EXTRAINFO_DIRINFO);
-    if ((type & EXTRAINFO_DIRINFO) &&
-        !router_supports_extrainfo(node->identity, is_trusted_extrainfo))
-      continue;
-    /* Don't make the same node a guard twice */
-    if (for_guard && node->using_as_guard) {
-      continue;
-    }
-    /* Ensure that a directory guard is actually a guard node. */
-    if (for_guard && !node->is_possible_guard) {
-      continue;
-    }
+
+    SKIP_MISSING_TRUSTED_EXTRAINFO(type, node->identity);
+
     if (try_excluding &&
         routerset_contains_routerstatus(options->ExcludeNodes, status,
                                         country)) {
@@ -2052,14 +2042,17 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
       continue;
     }
 
-    if (router_is_already_dir_fetching_rs(status,
-                                          no_serverdesc_fetching,
-                                          no_microdesc_fetching)) {
+    if (router_is_already_dir_fetching_(status->addr,
+                                        &status->ipv6_addr,
+                                        status->dir_port,
+                                        no_serverdesc_fetching,
+                                        no_microdesc_fetching)) {
       ++n_busy;
       continue;
     }
 
     is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now;
+    is_trusted = router_digest_is_trusted_dir(node->identity);
 
     /* Clients use IPv6 addresses if the server has one and the client
      * prefers IPv6.
@@ -2191,11 +2184,9 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
       if (!d->is_running) continue;
       if ((type & d->type) == 0)
         continue;
-      int is_trusted_extrainfo = router_digest_is_trusted_dir_type(
-                                 d->digest, EXTRAINFO_DIRINFO);
-      if ((type & EXTRAINFO_DIRINFO) &&
-          !router_supports_extrainfo(d->digest, is_trusted_extrainfo))
-        continue;
+
+      SKIP_MISSING_TRUSTED_EXTRAINFO(type, d->digest);
+
       if (requireother && me && router_digest_is_me(d->digest))
         continue;
       if (try_excluding &&
@@ -2205,8 +2196,11 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
         continue;
       }
 
-      if (router_is_already_dir_fetching_ds(d, no_serverdesc_fetching,
-                                            no_microdesc_fetching)) {
+      if (router_is_already_dir_fetching_(d->addr,
+                                          &d->ipv6_addr,
+                                          d->dir_port,
+                                          no_serverdesc_fetching,
+                                          no_microdesc_fetching)) {
         ++n_busy;
         continue;
       }
@@ -2327,17 +2321,16 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
  * we can pick a node for a circuit.
  */
 void
-router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid,
-                                      int need_uptime, int need_capacity,
-                                      int need_guard, int need_desc,
-                                      int pref_addr, int direct_conn)
+router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                      int need_capacity, int need_guard,
+                                      int need_desc, int pref_addr,
+                                      int direct_conn)
 {
   const int check_reach = !router_skip_or_reachability(get_options(),
                                                        pref_addr);
   /* XXXX MOVE */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
-    if (!node->is_running ||
-        (!node->is_valid && !allow_invalid))
+    if (!node->is_running || !node->is_valid)
       continue;
     if (need_desc && !(node->ri || (node->rs && node->md)))
       continue;
@@ -2798,8 +2791,6 @@ node_sl_choose_by_bandwidth(const smartlist_t *sl,
  * a minimum uptime, return one of those.
  * If <b>CRN_NEED_CAPACITY</b> is set in flags, weight your choice by the
  * advertised capacity of each router.
- * If <b>CRN_ALLOW_INVALID</b> is not set in flags, consider only Valid
- * routers.
  * If <b>CRN_NEED_GUARD</b> is set in flags, consider only Guard routers.
  * If <b>CRN_WEIGHT_AS_EXIT</b> is set in flags, we weight bandwidths as if
  * picking an exit node, otherwise we weight bandwidths for picking a relay
@@ -2820,7 +2811,6 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   const int need_uptime = (flags & CRN_NEED_UPTIME) != 0;
   const int need_capacity = (flags & CRN_NEED_CAPACITY) != 0;
   const int need_guard = (flags & CRN_NEED_GUARD) != 0;
-  const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0;
   const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0;
   const int need_desc = (flags & CRN_NEED_DESC) != 0;
   const int pref_addr = (flags & CRN_PREF_ADDR) != 0;
@@ -2836,14 +2826,12 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   rule = weight_for_exit ? WEIGHT_FOR_EXIT :
     (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
 
-  /* Exclude relays that allow single hop exit circuits, if the user
-   * wants to (such relays might be risky) */
-  if (get_options()->ExcludeSingleHopRelays) {
-    SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node,
-      if (node_allows_single_hop_exits(node)) {
-        smartlist_add(excludednodes, node);
-      });
-  }
+  /* Exclude relays that allow single hop exit circuits. This is an obsolete
+   * option since 0.2.9.2-alpha and done by default in 0.3.1.0-alpha. */
+  SMARTLIST_FOREACH(nodelist_get_list(), node_t *, node,
+    if (node_allows_single_hop_exits(node)) {
+      smartlist_add(excludednodes, node);
+    });
 
   /* If the node_t is not found we won't be to exclude ourself but we
    * won't be able to pick ourself in router_choose_random_node() so
@@ -2851,8 +2839,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   if ((r = router_get_my_routerinfo()))
     routerlist_add_node_and_family(excludednodes, r);
 
-  router_add_running_nodes_to_smartlist(sl, allow_invalid,
-                                        need_uptime, need_capacity,
+  router_add_running_nodes_to_smartlist(sl, need_uptime, need_capacity,
                                         need_guard, need_desc, pref_addr,
                                         direct_conn);
   log_debug(LD_CIRC,
@@ -3008,20 +2995,6 @@ router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type)
   return 0;
 }
 
-/** Return true iff <b>addr</b> is the address of one of our trusted
- * directory authorities. */
-int
-router_addr_is_trusted_dir(uint32_t addr)
-{
-  if (!trusted_dir_servers)
-    return 0;
-  SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ent,
-    if (ent->addr == addr)
-      return 1;
-    );
-  return 0;
-}
-
 /** If hexdigest is correctly formed, base16_decode it into
  * digest, which must have DIGEST_LEN space in it.
  * Return 0 on success, -1 on failure.
@@ -3087,8 +3060,8 @@ router_get_by_extrainfo_digest,(const char *digest))
 /** Return the signed descriptor for the extrainfo_t in our routerlist whose
  * extra-info-digest is <b>digest</b>. Return NULL if no such extra-info
  * document is known. */
-signed_descriptor_t *
-extrainfo_get_by_descriptor_digest(const char *digest)
+MOCK_IMPL(signed_descriptor_t *,
+extrainfo_get_by_descriptor_digest,(const char *digest))
 {
   extrainfo_t *ei;
   tor_assert(digest);
@@ -3926,7 +3899,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg,
                router_describe(router));
       *msg = "Router descriptor is not referenced by any network-status.";
 
-      /* Only journal this desc if we'll be serving it. */
+      /* Only journal this desc if we want to keep old descriptors */
       if (!from_cache && should_cache_old_descriptors())
         signed_desc_append_to_journal(&router->cache_info,
                                       &routerlist->desc_store);
@@ -4513,7 +4486,7 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
                         ei->cache_info.identity_digest,
                       DIGEST_LEN);
         smartlist_string_remove(requested_fingerprints, fp);
-        /* We silently let people stuff us with extrainfos we didn't ask for,
+        /* We silently let relays stuff us with extrainfos we didn't ask for,
          * so long as we would have wanted them anyway.  Since we always fetch
          * all the extrainfos we want, and we never actually act on them
          * inside Tor, this should be harmless. */
@@ -4556,13 +4529,14 @@ router_load_extrainfo_from_string(const char *s, const char *eos,
   smartlist_free(extrainfo_list);
 }
 
-/** Return true iff any networkstatus includes a descriptor whose digest
- * is that of <b>desc</b>. */
+/** Return true iff the latest ns-flavored consensus includes a descriptor
+ * whose digest is that of <b>desc</b>. */
 static int
 signed_desc_digest_is_recognized(signed_descriptor_t *desc)
 {
   const routerstatus_t *rs;
-  networkstatus_t *consensus = networkstatus_get_latest_consensus();
+  networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(
+                                                                      FLAV_NS);
 
   if (consensus) {
     rs = networkstatus_vote_find_entry(consensus, desc->identity_digest);
@@ -4608,7 +4582,7 @@ router_exit_policy_rejects_all(const routerinfo_t *router)
   return router->policy_is_reject_star;
 }
 
-/** Create an directory server at <b>address</b>:<b>port</b>, with OR identity
+/** Create a directory server at <b>address</b>:<b>port</b>, with OR identity
  * key <b>digest</b> which has DIGEST_LEN bytes.  If <b>address</b> is NULL,
  * add ourself.  If <b>is_authority</b>, this is a directory authority.  Return
  * the new directory server entry on success or NULL on failure. */
@@ -4919,9 +4893,10 @@ list_pending_fpsk_downloads(fp_pair_map_t *result)
  * range.)  If <b>source</b> is given, download from <b>source</b>;
  * otherwise, download from an appropriate random directory server.
  */
-MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
-          (const routerstatus_t *source, int purpose, smartlist_t *digests,
-           int lo, int hi, int pds_flags))
+MOCK_IMPL(STATIC void,
+initiate_descriptor_downloads,(const routerstatus_t *source,
+                               int purpose, smartlist_t *digests,
+                               int lo, int hi, int pds_flags))
 {
   char *resource, *cp;
   int digest_len, enc_digest_len;
@@ -4973,10 +4948,11 @@ MOCK_IMPL(STATIC void, initiate_descriptor_downloads,
 
   if (source) {
     /* We know which authority or directory mirror we want. */
-    directory_initiate_command_routerstatus(source, purpose,
-                                            ROUTER_PURPOSE_GENERAL,
-                                            DIRIND_ONEHOP,
-                                            resource, NULL, 0, 0);
+    directory_request_t *req = directory_request_new(purpose);
+    directory_request_set_routerstatus(req, source);
+    directory_request_set_resource(req, resource);
+    directory_initiate_request(req);
+    directory_request_free(req);
   } else {
     directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource,
                                  pds_flags, DL_WANT_ANY_DIRSERVER);
@@ -5190,8 +5166,8 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_would_reject;
         continue; /* We would throw it out immediately. */
       }
-      if (!directory_caches_dir_info(options) &&
-          !client_would_use_router(rs, now, options)) {
+      if (!we_want_to_fetch_flavor(options, consensus->flavor) &&
+          !client_would_use_router(rs, now)) {
         ++n_wouldnt_use;
         continue; /* We would never use it ourself. */
       }
diff --git a/src/or/routerlist.h b/src/or/routerlist.h
index 606e9085ce..e0ed4e623a 100644
--- a/src/or/routerlist.h
+++ b/src/or/routerlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -62,10 +62,10 @@ int router_skip_or_reachability(const or_options_t *options, int try_ip_pref);
 int router_get_my_share_of_directory_requests(double *v3_share_out);
 void router_reset_status_download_failures(void);
 int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2);
-void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid,
-                                           int need_uptime, int need_capacity,
-                                           int need_guard, int need_desc,
-                                           int pref_addr, int direct_conn);
+void router_add_running_nodes_to_smartlist(smartlist_t *sl, int need_uptime,
+                                           int need_capacity, int need_guard,
+                                           int need_desc, int pref_addr,
+                                           int direct_conn);
 
 const routerinfo_t *routerlist_find_my_routerinfo(void);
 uint32_t router_get_advertised_bandwidth(const routerinfo_t *router);
@@ -86,14 +86,14 @@ int router_digest_is_trusted_dir_type(const char *digest,
 #define router_digest_is_trusted_dir(d) \
   router_digest_is_trusted_dir_type((d), NO_DIRINFO)
 
-int router_addr_is_trusted_dir(uint32_t addr);
 int hexdigest_to_digest(const char *hexdigest, char *digest);
 const routerinfo_t *router_get_by_id_digest(const char *digest);
 routerinfo_t *router_get_mutable_by_digest(const char *digest);
 signed_descriptor_t *router_get_by_descriptor_digest(const char *digest);
 MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest,
           (const char *digest));
-signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest);
+MOCK_DECL(signed_descriptor_t *,extrainfo_get_by_descriptor_digest,
+          (const char *digest));
 const char *signed_descriptor_get_body(const signed_descriptor_t *desc);
 const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc);
 routerlist_t *router_get_routerlist(void);
@@ -124,7 +124,7 @@ static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s);
  */
 static inline int
 WRA_WAS_ADDED(was_router_added_t s) {
-  return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR;
+  return s == ROUTER_ADDED_SUCCESSFULLY;
 }
 /** Return true iff the outcome code in <b>s</b> indicates that the descriptor
  * was not added because it was either:
diff --git a/src/or/routerparse.c b/src/or/routerparse.c
index 521e237be2..22521a3069 100644
--- a/src/or/routerparse.c
+++ b/src/or/routerparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -60,6 +60,7 @@
 #include "circuitstats.h"
 #include "dirserv.h"
 #include "dirvote.h"
+#include "parsecommon.h"
 #include "policies.h"
 #include "protover.h"
 #include "rendcommon.h"
@@ -81,267 +82,6 @@
 
 /****************************************************************************/
 
-/** Enumeration of possible token types.  The ones starting with K_ correspond
- * to directory 'keywords'. A_ is for an annotation, R or C is related to
- * hidden services, ERR_ is an error in the tokenizing process, EOF_ is an
- * end-of-file marker, and NIL_ is used to encode not-a-token.
- */
-typedef enum {
-  K_ACCEPT = 0,
-  K_ACCEPT6,
-  K_DIRECTORY_SIGNATURE,
-  K_RECOMMENDED_SOFTWARE,
-  K_REJECT,
-  K_REJECT6,
-  K_ROUTER,
-  K_SIGNED_DIRECTORY,
-  K_SIGNING_KEY,
-  K_ONION_KEY,
-  K_ONION_KEY_NTOR,
-  K_ROUTER_SIGNATURE,
-  K_PUBLISHED,
-  K_RUNNING_ROUTERS,
-  K_ROUTER_STATUS,
-  K_PLATFORM,
-  K_PROTO,
-  K_OPT,
-  K_BANDWIDTH,
-  K_CONTACT,
-  K_NETWORK_STATUS,
-  K_UPTIME,
-  K_DIR_SIGNING_KEY,
-  K_FAMILY,
-  K_FINGERPRINT,
-  K_HIBERNATING,
-  K_READ_HISTORY,
-  K_WRITE_HISTORY,
-  K_NETWORK_STATUS_VERSION,
-  K_DIR_SOURCE,
-  K_DIR_OPTIONS,
-  K_CLIENT_VERSIONS,
-  K_SERVER_VERSIONS,
-  K_RECOMMENDED_CLIENT_PROTOCOLS,
-  K_RECOMMENDED_RELAY_PROTOCOLS,
-  K_REQUIRED_CLIENT_PROTOCOLS,
-  K_REQUIRED_RELAY_PROTOCOLS,
-  K_OR_ADDRESS,
-  K_ID,
-  K_P,
-  K_P6,
-  K_R,
-  K_A,
-  K_S,
-  K_V,
-  K_W,
-  K_M,
-  K_EXTRA_INFO,
-  K_EXTRA_INFO_DIGEST,
-  K_CACHES_EXTRA_INFO,
-  K_HIDDEN_SERVICE_DIR,
-  K_ALLOW_SINGLE_HOP_EXITS,
-  K_IPV6_POLICY,
-  K_ROUTER_SIG_ED25519,
-  K_IDENTITY_ED25519,
-  K_MASTER_KEY_ED25519,
-  K_ONION_KEY_CROSSCERT,
-  K_NTOR_ONION_KEY_CROSSCERT,
-
-  K_DIRREQ_END,
-  K_DIRREQ_V2_IPS,
-  K_DIRREQ_V3_IPS,
-  K_DIRREQ_V2_REQS,
-  K_DIRREQ_V3_REQS,
-  K_DIRREQ_V2_SHARE,
-  K_DIRREQ_V3_SHARE,
-  K_DIRREQ_V2_RESP,
-  K_DIRREQ_V3_RESP,
-  K_DIRREQ_V2_DIR,
-  K_DIRREQ_V3_DIR,
-  K_DIRREQ_V2_TUN,
-  K_DIRREQ_V3_TUN,
-  K_ENTRY_END,
-  K_ENTRY_IPS,
-  K_CELL_END,
-  K_CELL_PROCESSED,
-  K_CELL_QUEUED,
-  K_CELL_TIME,
-  K_CELL_CIRCS,
-  K_EXIT_END,
-  K_EXIT_WRITTEN,
-  K_EXIT_READ,
-  K_EXIT_OPENED,
-
-  K_DIR_KEY_CERTIFICATE_VERSION,
-  K_DIR_IDENTITY_KEY,
-  K_DIR_KEY_PUBLISHED,
-  K_DIR_KEY_EXPIRES,
-  K_DIR_KEY_CERTIFICATION,
-  K_DIR_KEY_CROSSCERT,
-  K_DIR_ADDRESS,
-  K_DIR_TUNNELLED,
-
-  K_VOTE_STATUS,
-  K_VALID_AFTER,
-  K_FRESH_UNTIL,
-  K_VALID_UNTIL,
-  K_VOTING_DELAY,
-
-  K_KNOWN_FLAGS,
-  K_PARAMS,
-  K_BW_WEIGHTS,
-  K_VOTE_DIGEST,
-  K_CONSENSUS_DIGEST,
-  K_ADDITIONAL_DIGEST,
-  K_ADDITIONAL_SIGNATURE,
-  K_CONSENSUS_METHODS,
-  K_CONSENSUS_METHOD,
-  K_LEGACY_DIR_KEY,
-  K_DIRECTORY_FOOTER,
-  K_SIGNING_CERT_ED,
-  K_SR_FLAG,
-  K_COMMIT,
-  K_PREVIOUS_SRV,
-  K_CURRENT_SRV,
-  K_PACKAGE,
-
-  A_PURPOSE,
-  A_LAST_LISTED,
-  A_UNKNOWN_,
-
-  R_RENDEZVOUS_SERVICE_DESCRIPTOR,
-  R_VERSION,
-  R_PERMANENT_KEY,
-  R_SECRET_ID_PART,
-  R_PUBLICATION_TIME,
-  R_PROTOCOL_VERSIONS,
-  R_INTRODUCTION_POINTS,
-  R_SIGNATURE,
-
-  R_IPO_IDENTIFIER,
-  R_IPO_IP_ADDRESS,
-  R_IPO_ONION_PORT,
-  R_IPO_ONION_KEY,
-  R_IPO_SERVICE_KEY,
-
-  C_CLIENT_NAME,
-  C_DESCRIPTOR_COOKIE,
-  C_CLIENT_KEY,
-
-  ERR_,
-  EOF_,
-  NIL_
-} directory_keyword;
-
-#define MIN_ANNOTATION A_PURPOSE
-#define MAX_ANNOTATION A_UNKNOWN_
-
-/** Structure to hold a single directory token.
- *
- * We parse a directory by breaking it into "tokens", each consisting
- * of a keyword, a line full of arguments, and a binary object.  The
- * arguments and object are both optional, depending on the keyword
- * type.
- *
- * This structure is only allocated in memareas; do not allocate it on
- * the heap, or token_clear() won't work.
- */
-typedef struct directory_token_t {
-  directory_keyword tp;        /**< Type of the token. */
-  int n_args:30;               /**< Number of elements in args */
-  char **args;                 /**< Array of arguments from keyword line. */
-
-  char *object_type;           /**< -----BEGIN [object_type]-----*/
-  size_t object_size;          /**< Bytes in object_body */
-  char *object_body;           /**< Contents of object, base64-decoded. */
-
-  crypto_pk_t *key;        /**< For public keys only.  Heap-allocated. */
-
-  char *error;                 /**< For ERR_ tokens only. */
-} directory_token_t;
-
-/* ********************************************************************** */
-
-/** We use a table of rules to decide how to parse each token type. */
-
-/** Rules for whether the keyword needs an object. */
-typedef enum {
-  NO_OBJ,        /**< No object, ever. */
-  NEED_OBJ,      /**< Object is required. */
-  NEED_SKEY_1024,/**< Object is required, and must be a 1024 bit private key */
-  NEED_KEY_1024, /**< Object is required, and must be a 1024 bit public key */
-  NEED_KEY,      /**< Object is required, and must be a public key. */
-  OBJ_OK,        /**< Object is optional. */
-} obj_syntax;
-
-#define AT_START 1
-#define AT_END 2
-
-/** Determines the parsing rules for a single token type. */
-typedef struct token_rule_t {
-  /** The string value of the keyword identifying the type of item. */
-  const char *t;
-  /** The corresponding directory_keyword enum. */
-  directory_keyword v;
-  /** Minimum number of arguments for this item */
-  int min_args;
-  /** Maximum number of arguments for this item */
-  int max_args;
-  /** If true, we concatenate all arguments for this item into a single
-   * string. */
-  int concat_args;
-  /** Requirements on object syntax for this item. */
-  obj_syntax os;
-  /** Lowest number of times this item may appear in a document. */
-  int min_cnt;
-  /** Highest number of times this item may appear in a document. */
-  int max_cnt;
-  /** One or more of AT_START/AT_END to limit where the item may appear in a
-   * document. */
-  int pos;
-  /** True iff this token is an annotation. */
-  int is_annotation;
-} token_rule_t;
-
-/**
- * @name macros for defining token rules
- *
- * Helper macros to define token tables.  's' is a string, 't' is a
- * directory_keyword, 'a' is a trio of argument multiplicities, and 'o' is an
- * object syntax.
- */
-/**@{*/
-
-/** Appears to indicate the end of a table. */
-#define END_OF_TABLE { NULL, NIL_, 0,0,0, NO_OBJ, 0, INT_MAX, 0, 0 }
-/** An item with no restrictions: used for obsolete document types */
-#define T(s,t,a,o)    { s, t, a, o, 0, INT_MAX, 0, 0 }
-/** An item with no restrictions on multiplicity or location. */
-#define T0N(s,t,a,o)  { s, t, a, o, 0, INT_MAX, 0, 0 }
-/** An item that must appear exactly once */
-#define T1(s,t,a,o)   { s, t, a, o, 1, 1, 0, 0 }
-/** An item that must appear exactly once, at the start of the document */
-#define T1_START(s,t,a,o)   { s, t, a, o, 1, 1, AT_START, 0 }
-/** An item that must appear exactly once, at the end of the document */
-#define T1_END(s,t,a,o)   { s, t, a, o, 1, 1, AT_END, 0 }
-/** An item that must appear one or more times */
-#define T1N(s,t,a,o)  { s, t, a, o, 1, INT_MAX, 0, 0 }
-/** An item that must appear no more than once */
-#define T01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 0 }
-/** An annotation that must appear no more than once */
-#define A01(s,t,a,o)  { s, t, a, o, 0, 1, 0, 1 }
-
-/** Argument multiplicity: any number of arguments. */
-#define ARGS        0,INT_MAX,0
-/** Argument multiplicity: no arguments. */
-#define NO_ARGS     0,0,0
-/** Argument multiplicity: concatenate all arguments. */
-#define CONCAT_ARGS 1,1,1
-/** Argument multiplicity: at least <b>n</b> arguments. */
-#define GE(n)       n,INT_MAX,0
-/** Argument multiplicity: exactly <b>n</b> arguments. */
-#define EQ(n)       n,n,0
-/**@}*/
-
 /** List of tokens recognized in router descriptors */
 static token_rule_t routerdesc_token_table[] = {
   T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
@@ -619,6 +359,7 @@ static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok);
 static int router_get_hash_impl_helper(const char *s, size_t s_len,
                             const char *start_str,
                             const char *end_str, char end_c,
+                            int log_severity,
                             const char **start_out, const char **end_out);
 static int router_get_hash_impl(const char *s, size_t s_len, char *digest,
                                 const char *start_str, const char *end_str,
@@ -628,30 +369,9 @@ static int router_get_hashes_impl(const char *s, size_t s_len,
                                   common_digests_t *digests,
                                   const char *start_str, const char *end_str,
                                   char end_char);
-static void token_clear(directory_token_t *tok);
-static smartlist_t *find_all_by_keyword(smartlist_t *s, directory_keyword k);
 static smartlist_t *find_all_exitpolicy(smartlist_t *s);
-static directory_token_t *find_by_keyword_(smartlist_t *s,
-                                           directory_keyword keyword,
-                                           const char *keyword_str);
-#define find_by_keyword(s, keyword) find_by_keyword_((s), (keyword), #keyword)
-static directory_token_t *find_opt_by_keyword(smartlist_t *s,
-                                              directory_keyword keyword);
-
-#define TS_ANNOTATIONS_OK 1
-#define TS_NOCHECK 2
-#define TS_NO_NEW_ANNOTATIONS 4
-static int tokenize_string(memarea_t *area,
-                           const char *start, const char *end,
-                           smartlist_t *out,
-                           token_rule_t *table,
-                           int flags);
-static directory_token_t *get_next_token(memarea_t *area,
-                                         const char **s,
-                                         const char *eos,
-                                         token_rule_t *table);
-#define CST_CHECK_AUTHORITY   (1<<0)
-#define CST_NO_CHECK_OBJTYPE  (1<<1)
+
+#define CST_NO_CHECK_OBJTYPE  (1<<0)
 static int check_signature_token(const char *digest,
                                  ssize_t digest_len,
                                  directory_token_t *tok,
@@ -995,7 +715,7 @@ dump_desc_populate_one_file, (const char *dirname, const char *f))
    * filename.
    */
   if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
-                       DIGEST_SHA256) != 0) {
+                       DIGEST_SHA256) < 0) {
     /* Weird, but okay */
     log_info(LD_DIR,
              "Unable to hash content of %s from unparseable descriptors "
@@ -1144,8 +864,8 @@ dump_desc_populate_fifo_from_directory(const char *dirname)
  * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
  * than one descriptor to disk per minute. If there is already such a
  * file in the data directory, overwrite it. */
-STATIC void
-dump_desc(const char *desc, const char *type)
+MOCK_IMPL(STATIC void,
+dump_desc,(const char *desc, const char *type))
 {
   tor_assert(desc);
   tor_assert(type);
@@ -1159,7 +879,7 @@ dump_desc(const char *desc, const char *type)
   /* Get the hash for logging purposes anyway */
   len = strlen(desc);
   if (crypto_digest256((char *)digest_sha256, desc, len,
-                       DIGEST_SHA256) != 0) {
+                       DIGEST_SHA256) < 0) {
     log_info(LD_DIR,
              "Unable to parse descriptor of type %s, and unable to even hash"
              " it!", type);
@@ -1269,6 +989,41 @@ router_get_router_hash(const char *s, size_t s_len, char *digest)
                               DIGEST_SHA1);
 }
 
+/** Try to find the start and end of the signed portion of a networkstatus
+ * document in <b>s</b>. On success, set <b>start_out</b> to the first
+ * character of the document, and <b>end_out</b> to a position one after the
+ * final character of the signed document, and return 0.  On failure, return
+ * -1. */
+int
+router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              const char **start_out,
+                                              const char **end_out)
+{
+  return router_get_hash_impl_helper(s, strlen(s),
+                                     "network-status-version",
+                                     "\ndirectory-signature",
+                                     ' ', LOG_INFO,
+                                     start_out, end_out);
+}
+
+/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the
+ * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no
+ * signed portion can be identified.  Return 0 on success, -1 on failure. */
+int
+router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                           const char *s)
+{
+  const char *start, *end;
+  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+    start = s;
+    end = s + strlen(s);
+  }
+  tor_assert(start);
+  tor_assert(end);
+  return crypto_digest256((char*)digest_out, start, end-start,
+                          DIGEST_SHA3_256);
+}
+
 /** Set <b>digests</b> to all the digests of the consensus document in
  * <b>s</b> */
 int
@@ -1453,28 +1208,15 @@ tor_version_is_obsolete(const char *myversion, const char *versionlist)
   return ret;
 }
 
-/** Return true iff <b>key</b> is allowed to sign directories.
- */
-static int
-dir_signing_key_is_trusted(crypto_pk_t *key)
+MOCK_IMPL(STATIC int,
+signed_digest_equals, (const uint8_t *d1, const uint8_t *d2, size_t len))
 {
-  char digest[DIGEST_LEN];
-  if (!key) return 0;
-  if (crypto_pk_get_digest(key, digest) < 0) {
-    log_warn(LD_DIR, "Error computing dir-signing-key digest");
-    return 0;
-  }
-  if (!router_digest_is_trusted_dir(digest)) {
-    log_warn(LD_DIR, "Listed dir-signing-key is not trusted");
-    return 0;
-  }
-  return 1;
+  return tor_memeq(d1, d2, len);
 }
 
 /** Check whether the object body of the token in <b>tok</b> has a good
- * signature for <b>digest</b> using key <b>pkey</b>.  If
- * <b>CST_CHECK_AUTHORITY</b> is set, make sure that <b>pkey</b> is the key of
- * a directory authority.  If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
+ * signature for <b>digest</b> using key <b>pkey</b>.
+ * If <b>CST_NO_CHECK_OBJTYPE</b> is set, do not check
  * the object type of the signature object. Use <b>doctype</b> as the type of
  * the document when generating log messages.  Return 0 on success, negative
  * on failure.
@@ -1489,7 +1231,6 @@ check_signature_token(const char *digest,
 {
   char *signed_digest;
   size_t keysize;
-  const int check_authority = (flags & CST_CHECK_AUTHORITY);
   const int check_objtype = ! (flags & CST_NO_CHECK_OBJTYPE);
 
   tor_assert(pkey);
@@ -1497,12 +1238,6 @@ check_signature_token(const char *digest,
   tor_assert(digest);
   tor_assert(doctype);
 
-  if (check_authority && !dir_signing_key_is_trusted(pkey)) {
-    log_warn(LD_DIR, "Key on %s did not come from an authority; rejecting",
-             doctype);
-    return -1;
-  }
-
   if (check_objtype) {
     if (strcmp(tok->object_type, "SIGNATURE")) {
       log_warn(LD_DIR, "Bad object type on %s signature", doctype);
@@ -1521,7 +1256,8 @@ check_signature_token(const char *digest,
   }
   //  log_debug(LD_DIR,"Signed %s hash starts %s", doctype,
   //            hex_str(signed_digest,4));
-  if (tor_memneq(digest, signed_digest, digest_len)) {
+  if (! signed_digest_equals((const uint8_t *)digest,
+                             (const uint8_t *)signed_digest, digest_len)) {
     log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype);
     tor_free(signed_digest);
     return -1;
@@ -2087,7 +1823,8 @@ router_parse_entry_from_string(const char *s, const char *end,
 
       if (router_get_hash_impl_helper(s, end-s, "router ",
                                       "\nrouter-sig-ed25519",
-                                      ' ', &signed_start, &signed_end) < 0) {
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
         log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor");
         goto err;
       }
@@ -2100,12 +1837,13 @@ router_parse_entry_from_string(const char *s, const char *end,
 
       ed25519_checkable_t check[3];
       int check_ok[3];
-      if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) {
+      time_t expires = TIME_MAX;
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, &expires) < 0) {
         log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
         goto err;
       }
       if (tor_cert_get_checkable_sig(&check[1],
-                                     ntor_cc_cert, &ntor_cc_pk) < 0) {
+                               ntor_cc_cert, &ntor_cc_pk, &expires) < 0) {
         log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert.");
         goto err;
       }
@@ -2135,10 +1873,7 @@ router_parse_entry_from_string(const char *s, const char *end,
       }
 
       /* We check this before adding it to the routerlist. */
-      if (cert->valid_until < ntor_cc_cert->valid_until)
-        router->cert_expiration_time = cert->valid_until;
-      else
-        router->cert_expiration_time = ntor_cc_cert->valid_until;
+      router->cert_expiration_time = expires;
     }
   }
 
@@ -2220,7 +1955,7 @@ router_parse_entry_from_string(const char *s, const char *end,
                  escaped(tok->args[i]));
         goto err;
       }
-      smartlist_add(router->declared_family, tor_strdup(tok->args[i]));
+      smartlist_add_strdup(router->declared_family, tok->args[i]);
     }
   }
 
@@ -2332,6 +2067,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
    * parse that's covered by the hash. */
   int can_dl_again = 0;
 
+  if (BUG(s == NULL))
+    return NULL;
+
   if (!end) {
     end = s + strlen(s);
   }
@@ -2439,7 +2177,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
 
       if (router_get_hash_impl_helper(s, end-s, "extra-info ",
                                       "\nrouter-sig-ed25519",
-                                      ' ', &signed_start, &signed_end) < 0) {
+                                      ' ', LOG_WARN,
+                                      &signed_start, &signed_end) < 0) {
         log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo");
         goto err;
       }
@@ -2452,7 +2191,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end,
 
       ed25519_checkable_t check[2];
       int check_ok[2];
-      if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) {
+      if (tor_cert_get_checkable_sig(&check[0], cert, NULL, NULL) < 0) {
         log_err(LD_BUG, "Couldn't create 'checkable' for cert.");
         goto err;
       }
@@ -2846,7 +2585,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
       goto err;
     }
   } else if (flav == FLAV_MICRODESC) {
-    offset = -1; /* There is no identity digest */
+    offset = -1; /* There is no descriptor digest in an md consensus r line */
   }
 
   if (vote_rs) {
@@ -2964,6 +2703,12 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     rs->protocols_known = 1;
     rs->supports_extend2_cells =
       protocol_list_supports_protocol(tok->args[0], PRT_RELAY, 2);
+    rs->supports_ed25519_link_handshake =
+      protocol_list_supports_protocol(tok->args[0], PRT_LINKAUTH, 3);
+    rs->supports_ed25519_hs_intro =
+      protocol_list_supports_protocol(tok->args[0], PRT_HSINTRO, 4);
+    rs->supports_v3_hsdir =
+      protocol_list_supports_protocol(tok->args[0], PRT_HSDIR, 2);
   }
   if ((tok = find_opt_by_keyword(tokens, K_V))) {
     tor_assert(tok->n_args == 1);
@@ -3639,6 +3384,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   networkstatus_voter_info_t *voter = NULL;
   networkstatus_t *ns = NULL;
   common_digests_t ns_digests;
+  uint8_t sha3_as_signed[DIGEST256_LEN];
   const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
   directory_token_t *tok;
   struct in_addr in;
@@ -3652,7 +3398,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests)) {
+  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
@@ -3669,6 +3416,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   ns = tor_malloc_zero(sizeof(networkstatus_t));
   memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
+  memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed));
 
   tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
   tor_assert(tok);
@@ -3723,9 +3471,9 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
     if (tok) {
       for (i=0; i < tok->n_args; ++i)
-        smartlist_add(ns->supported_methods, tor_strdup(tok->args[i]));
+        smartlist_add_strdup(ns->supported_methods, tok->args[i]);
     } else {
-      smartlist_add(ns->supported_methods, tor_strdup("1"));
+      smartlist_add_strdup(ns->supported_methods, "1");
     }
   } else {
     tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
@@ -3807,7 +3555,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     ns->package_lines = smartlist_new();
     if (package_lst) {
       SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
-                    smartlist_add(ns->package_lines, tor_strdup(t->args[0])));
+                    smartlist_add_strdup(ns->package_lines, t->args[0]));
     }
     smartlist_free(package_lst);
   }
@@ -3816,7 +3564,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   ns->known_flags = smartlist_new();
   inorder = 1;
   for (i = 0; i < tok->n_args; ++i) {
-    smartlist_add(ns->known_flags, tor_strdup(tok->args[i]));
+    smartlist_add_strdup(ns->known_flags, tok->args[i]);
     if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
       log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
       inorder = 0;
@@ -3868,7 +3616,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
       tor_free(last_kwd);
       last_kwd = tor_strndup(tok->args[i], eq_pos);
-      smartlist_add(ns->net_params, tor_strdup(tok->args[i]));
+      smartlist_add_strdup(ns->net_params, tok->args[i]);
     }
     if (!inorder) {
       log_warn(LD_DIR, "params not in order");
@@ -4011,11 +3759,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     if (ns->type != NS_TYPE_CONSENSUS) {
       vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
-                                               rs, 0, 0))
+                                               rs, 0, 0)) {
         smartlist_add(ns->routerstatus_list, rs);
-      else {
-        tor_free(rs->version);
-        tor_free(rs);
+      } else {
+        vote_routerstatus_free(rs);
       }
     } else {
       routerstatus_t *rs;
@@ -4111,7 +3858,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
         log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
         goto err;
       }
-      smartlist_add(ns->weight_params, tor_strdup(tok->args[i]));
+      smartlist_add_strdup(ns->weight_params, tok->args[i]);
     }
   }
 
@@ -4740,445 +4487,6 @@ assert_addr_policy_ok(smartlist_t *lst)
   });
 }
 
-/*
- * Low-level tokenizer for router descriptors and directories.
- */
-
-/** Free all resources allocated for <b>tok</b> */
-static void
-token_clear(directory_token_t *tok)
-{
-  if (tok->key)
-    crypto_pk_free(tok->key);
-}
-
-#define ALLOC_ZERO(sz) memarea_alloc_zero(area,sz)
-#define ALLOC(sz) memarea_alloc(area,sz)
-#define STRDUP(str) memarea_strdup(area,str)
-#define STRNDUP(str,n) memarea_strndup(area,(str),(n))
-
-#define RET_ERR(msg)                                               \
-  STMT_BEGIN                                                       \
-    if (tok) token_clear(tok);                                      \
-    tok = ALLOC_ZERO(sizeof(directory_token_t));                   \
-    tok->tp = ERR_;                                                \
-    tok->error = STRDUP(msg);                                      \
-    goto done_tokenizing;                                          \
-  STMT_END
-
-/** Helper: make sure that the token <b>tok</b> with keyword <b>kwd</b> obeys
- * the object syntax of <b>o_syn</b>.  Allocate all storage in <b>area</b>.
- * Return <b>tok</b> on success, or a new ERR_ token if the token didn't
- * conform to the syntax we wanted.
- **/
-static inline directory_token_t *
-token_check_object(memarea_t *area, const char *kwd,
-                   directory_token_t *tok, obj_syntax o_syn)
-{
-  char ebuf[128];
-  switch (o_syn) {
-    case NO_OBJ:
-      /* No object is allowed for this token. */
-      if (tok->object_body) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected object for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      if (tok->key) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Unexpected public key for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    case NEED_OBJ:
-      /* There must be a (non-key) object. */
-      if (!tok->object_body) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Missing object for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    case NEED_KEY_1024: /* There must be a 1024-bit public key. */
-    case NEED_SKEY_1024: /* There must be a 1024-bit private key. */
-      if (tok->key && crypto_pk_num_bits(tok->key) != PK_BYTES*8) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Wrong size on key for %s: %d bits",
-                     kwd, crypto_pk_num_bits(tok->key));
-        RET_ERR(ebuf);
-      }
-      /* fall through */
-    case NEED_KEY: /* There must be some kind of key. */
-      if (!tok->key) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Missing public key for %s", kwd);
-        RET_ERR(ebuf);
-      }
-      if (o_syn != NEED_SKEY_1024) {
-        if (crypto_pk_key_is_private(tok->key)) {
-          tor_snprintf(ebuf, sizeof(ebuf),
-               "Private key given for %s, which wants a public key", kwd);
-          RET_ERR(ebuf);
-        }
-      } else { /* o_syn == NEED_SKEY_1024 */
-        if (!crypto_pk_key_is_private(tok->key)) {
-          tor_snprintf(ebuf, sizeof(ebuf),
-               "Public key given for %s, which wants a private key", kwd);
-          RET_ERR(ebuf);
-        }
-      }
-      break;
-    case OBJ_OK:
-      /* Anything goes with this token. */
-      break;
-  }
-
- done_tokenizing:
-  return tok;
-}
-
-/** Helper: parse space-separated arguments from the string <b>s</b> ending at
- * <b>eol</b>, and store them in the args field of <b>tok</b>.  Store the
- * number of parsed elements into the n_args field of <b>tok</b>.  Allocate
- * all storage in <b>area</b>.  Return the number of arguments parsed, or
- * return -1 if there was an insanely high number of arguments. */
-static inline int
-get_token_arguments(memarea_t *area, directory_token_t *tok,
-                    const char *s, const char *eol)
-{
-/** Largest number of arguments we'll accept to any token, ever. */
-#define MAX_ARGS 512
-  char *mem = memarea_strndup(area, s, eol-s);
-  char *cp = mem;
-  int j = 0;
-  char *args[MAX_ARGS];
-  while (*cp) {
-    if (j == MAX_ARGS)
-      return -1;
-    args[j++] = cp;
-    cp = (char*)find_whitespace(cp);
-    if (!cp || !*cp)
-      break; /* End of the line. */
-    *cp++ = '\0';
-    cp = (char*)eat_whitespace(cp);
-  }
-  tok->n_args = j;
-  tok->args = memarea_memdup(area, args, j*sizeof(char*));
-  return j;
-#undef MAX_ARGS
-}
-
-/** Helper function: read the next token from *s, advance *s to the end of the
- * token, and return the parsed token.  Parse *<b>s</b> according to the list
- * of tokens in <b>table</b>.
- */
-static directory_token_t *
-get_next_token(memarea_t *area,
-               const char **s, const char *eos, token_rule_t *table)
-{
-  /** Reject any object at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_UNPARSED_OBJECT_SIZE (128*1024)
-  /** Reject any line at least this big; it is probably an overflow, an
-   * attack, a bug, or some other nonsense. */
-#define MAX_LINE_LENGTH (128*1024)
-
-  const char *next, *eol, *obstart;
-  size_t obname_len;
-  int i;
-  directory_token_t *tok;
-  obj_syntax o_syn = NO_OBJ;
-  char ebuf[128];
-  const char *kwd = "";
-
-  tor_assert(area);
-  tok = ALLOC_ZERO(sizeof(directory_token_t));
-  tok->tp = ERR_;
-
-  /* Set *s to first token, eol to end-of-line, next to after first token */
-  *s = eat_whitespace_eos(*s, eos); /* eat multi-line whitespace */
-  tor_assert(eos >= *s);
-  eol = memchr(*s, '\n', eos-*s);
-  if (!eol)
-    eol = eos;
-  if (eol - *s > MAX_LINE_LENGTH) {
-    RET_ERR("Line far too long");
-  }
-
-  next = find_whitespace_eos(*s, eol);
-
-  if (!strcmp_len(*s, "opt", next-*s)) {
-    /* Skip past an "opt" at the start of the line. */
-    *s = eat_whitespace_eos_no_nl(next, eol);
-    next = find_whitespace_eos(*s, eol);
-  } else if (*s == eos) {  /* If no "opt", and end-of-line, line is invalid */
-    RET_ERR("Unexpected EOF");
-  }
-
-  /* Search the table for the appropriate entry.  (I tried a binary search
-   * instead, but it wasn't any faster.) */
-  for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
-      /* We've found the keyword. */
-      kwd = table[i].t;
-      tok->tp = table[i].v;
-      o_syn = table[i].os;
-      *s = eat_whitespace_eos_no_nl(next, eol);
-      /* We go ahead whether there are arguments or not, so that tok->args is
-       * always set if we want arguments. */
-      if (table[i].concat_args) {
-        /* The keyword takes the line as a single argument */
-        tok->args = ALLOC(sizeof(char*));
-        tok->args[0] = STRNDUP(*s,eol-*s); /* Grab everything on line */
-        tok->n_args = 1;
-      } else {
-        /* This keyword takes multiple arguments. */
-        if (get_token_arguments(area, tok, *s, eol)<0) {
-          tor_snprintf(ebuf, sizeof(ebuf),"Far too many arguments to %s", kwd);
-          RET_ERR(ebuf);
-        }
-        *s = eol;
-      }
-      if (tok->n_args < table[i].min_args) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Too few arguments to %s", kwd);
-        RET_ERR(ebuf);
-      } else if (tok->n_args > table[i].max_args) {
-        tor_snprintf(ebuf, sizeof(ebuf), "Too many arguments to %s", kwd);
-        RET_ERR(ebuf);
-      }
-      break;
-    }
-  }
-
-  if (tok->tp == ERR_) {
-    /* No keyword matched; call it an "K_opt" or "A_unrecognized" */
-    if (*s < eol && **s == '@')
-      tok->tp = A_UNKNOWN_;
-    else
-      tok->tp = K_OPT;
-    tok->args = ALLOC(sizeof(char*));
-    tok->args[0] = STRNDUP(*s, eol-*s);
-    tok->n_args = 1;
-    o_syn = OBJ_OK;
-  }
-
-  /* Check whether there's an object present */
-  *s = eat_whitespace_eos(eol, eos);  /* Scan from end of first line */
-  tor_assert(eos >= *s);
-  eol = memchr(*s, '\n', eos-*s);
-  if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
-    goto check_object;
-
-  obstart = *s; /* Set obstart to start of object spec */
-  if (*s+16 >= eol || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
-      (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
-    RET_ERR("Malformed object: bad begin line");
-  }
-  tok->object_type = STRNDUP(*s+11, eol-*s-16);
-  obname_len = eol-*s-16; /* store objname length here to avoid a strlen() */
-  *s = eol+1;    /* Set *s to possible start of object data (could be eos) */
-
-  /* Go to the end of the object */
-  next = tor_memstr(*s, eos-*s, "-----END ");
-  if (!next) {
-    RET_ERR("Malformed object: missing object end line");
-  }
-  tor_assert(eos >= next);
-  eol = memchr(next, '\n', eos-next);
-  if (!eol)  /* end-of-line marker, or eos if there's no '\n' */
-    eol = eos;
-  /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
-  if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
-    tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
-             tok->object_type);
-    ebuf[sizeof(ebuf)-1] = '\0';
-    RET_ERR(ebuf);
-  }
-  if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
-    RET_ERR("Couldn't parse object: missing footer or object much too big.");
-
-  if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
-      RET_ERR("Couldn't parse public key.");
-  } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart))
-      RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
-  }
-  *s = eol;
-
- check_object:
-  tok = token_check_object(area, kwd, tok, o_syn);
-
- done_tokenizing:
-  return tok;
-
-#undef RET_ERR
-#undef ALLOC
-#undef ALLOC_ZERO
-#undef STRDUP
-#undef STRNDUP
-}
-
-/** Read all tokens from a string between <b>start</b> and <b>end</b>, and add
- * them to <b>out</b>.  Parse according to the token rules in <b>table</b>.
- * Caller must free tokens in <b>out</b>.  If <b>end</b> is NULL, use the
- * entire string.
- */
-static int
-tokenize_string(memarea_t *area,
-                const char *start, const char *end, smartlist_t *out,
-                token_rule_t *table, int flags)
-{
-  const char **s;
-  directory_token_t *tok = NULL;
-  int counts[NIL_];
-  int i;
-  int first_nonannotation;
-  int prev_len = smartlist_len(out);
-  tor_assert(area);
-
-  s = &start;
-  if (!end) {
-    end = start+strlen(start);
-  } else {
-    /* it's only meaningful to check for nuls if we got an end-of-string ptr */
-    if (memchr(start, '\0', end-start)) {
-      log_warn(LD_DIR, "parse error: internal NUL character.");
-      return -1;
-    }
-  }
-  for (i = 0; i < NIL_; ++i)
-    counts[i] = 0;
-
-  SMARTLIST_FOREACH(out, const directory_token_t *, t, ++counts[t->tp]);
-
-  while (*s < end && (!tok || tok->tp != EOF_)) {
-    tok = get_next_token(area, s, end, table);
-    if (tok->tp == ERR_) {
-      log_warn(LD_DIR, "parse error: %s", tok->error);
-      token_clear(tok);
-      return -1;
-    }
-    ++counts[tok->tp];
-    smartlist_add(out, tok);
-    *s = eat_whitespace_eos(*s, end);
-  }
-
-  if (flags & TS_NOCHECK)
-    return 0;
-
-  if ((flags & TS_ANNOTATIONS_OK)) {
-    first_nonannotation = -1;
-    for (i = 0; i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp < MIN_ANNOTATION || tok->tp > MAX_ANNOTATION) {
-        first_nonannotation = i;
-        break;
-      }
-    }
-    if (first_nonannotation < 0) {
-      log_warn(LD_DIR, "parse error: item contains only annotations");
-      return -1;
-    }
-    for (i=first_nonannotation;  i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
-        log_warn(LD_DIR, "parse error: Annotations mixed with keywords");
-        return -1;
-      }
-    }
-    if ((flags & TS_NO_NEW_ANNOTATIONS)) {
-      if (first_nonannotation != prev_len) {
-        log_warn(LD_DIR, "parse error: Unexpected annotations.");
-        return -1;
-      }
-    }
-  } else {
-    for (i=0;  i < smartlist_len(out); ++i) {
-      tok = smartlist_get(out, i);
-      if (tok->tp >= MIN_ANNOTATION && tok->tp <= MAX_ANNOTATION) {
-        log_warn(LD_DIR, "parse error: no annotations allowed.");
-        return -1;
-      }
-    }
-    first_nonannotation = 0;
-  }
-  for (i = 0; table[i].t; ++i) {
-    if (counts[table[i].v] < table[i].min_cnt) {
-      log_warn(LD_DIR, "Parse error: missing %s element.", table[i].t);
-      return -1;
-    }
-    if (counts[table[i].v] > table[i].max_cnt) {
-      log_warn(LD_DIR, "Parse error: too many %s elements.", table[i].t);
-      return -1;
-    }
-    if (table[i].pos & AT_START) {
-      if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, first_nonannotation))->tp != table[i].v) {
-        log_warn(LD_DIR, "Parse error: first item is not %s.", table[i].t);
-        return -1;
-      }
-    }
-    if (table[i].pos & AT_END) {
-      if (smartlist_len(out) < 1 ||
-          (tok = smartlist_get(out, smartlist_len(out)-1))->tp != table[i].v) {
-        log_warn(LD_DIR, "Parse error: last item is not %s.", table[i].t);
-        return -1;
-      }
-    }
-  }
-  return 0;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; return
- * NULL if no such keyword is found.
- */
-static directory_token_t *
-find_opt_by_keyword(smartlist_t *s, directory_keyword keyword)
-{
-  SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t);
-  return NULL;
-}
-
-/** Find the first token in <b>s</b> whose keyword is <b>keyword</b>; fail
- * with an assert if no such keyword is found.
- */
-static directory_token_t *
-find_by_keyword_(smartlist_t *s, directory_keyword keyword,
-                 const char *keyword_as_string)
-{
-  directory_token_t *tok = find_opt_by_keyword(s, keyword);
-  if (PREDICT_UNLIKELY(!tok)) {
-    log_err(LD_BUG, "Missing %s [%d] in directory object that should have "
-         "been validated. Internal error.", keyword_as_string, (int)keyword);
-    tor_assert(tok);
-  }
-  return tok;
-}
-
-/** If there are any directory_token_t entries in <b>s</b> whose keyword is
- * <b>k</b>, return a newly allocated smartlist_t containing all such entries,
- * in the same order in which they occur in <b>s</b>.  Otherwise return
- * NULL. */
-static smartlist_t *
-find_all_by_keyword(smartlist_t *s, directory_keyword k)
-{
-  smartlist_t *out = NULL;
-  SMARTLIST_FOREACH(s, directory_token_t *, t,
-                    if (t->tp == k) {
-                      if (!out)
-                        out = smartlist_new();
-                      smartlist_add(out, t);
-                    });
-  return out;
-}
-
 /** Return a newly allocated smartlist of all accept or reject tokens in
  * <b>s</b>.
  */
@@ -5204,16 +4512,18 @@ static int
 router_get_hash_impl_helper(const char *s, size_t s_len,
                             const char *start_str,
                             const char *end_str, char end_c,
+                            int log_severity,
                             const char **start_out, const char **end_out)
 {
   const char *start, *end;
   start = tor_memstr(s, s_len, start_str);
   if (!start) {
-    log_warn(LD_DIR,"couldn't find start of hashed material \"%s\"",start_str);
+    log_fn(log_severity,LD_DIR,
+           "couldn't find start of hashed material \"%s\"",start_str);
     return -1;
   }
   if (start != s && *(start-1) != '\n') {
-    log_warn(LD_DIR,
+    log_fn(log_severity,LD_DIR,
              "first occurrence of \"%s\" is not at the start of a line",
              start_str);
     return -1;
@@ -5221,12 +4531,14 @@ router_get_hash_impl_helper(const char *s, size_t s_len,
   end = tor_memstr(start+strlen(start_str),
                    s_len - (start-s) - strlen(start_str), end_str);
   if (!end) {
-    log_warn(LD_DIR,"couldn't find end of hashed material \"%s\"",end_str);
+    log_fn(log_severity,LD_DIR,
+           "couldn't find end of hashed material \"%s\"",end_str);
     return -1;
   }
   end = memchr(end+strlen(end_str), end_c, s_len - (end-s) - strlen(end_str));
   if (!end) {
-    log_warn(LD_DIR,"couldn't find EOL");
+    log_fn(log_severity,LD_DIR,
+           "couldn't find EOL");
     return -1;
   }
   ++end;
@@ -5250,17 +4562,28 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest,
                      digest_algorithm_t alg)
 {
   const char *start=NULL, *end=NULL;
-  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
                                   &start,&end)<0)
     return -1;
 
+  return router_compute_hash_final(digest, start, end-start, alg);
+}
+
+/** Compute the digest of the <b>len</b>-byte directory object at
+ * <b>start</b>, using <b>alg</b>. Store the result in <b>digest</b>, which
+ * must be long enough to hold it. */
+MOCK_IMPL(STATIC int,
+router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg))
+{
   if (alg == DIGEST_SHA1) {
-    if (crypto_digest(digest, start, end-start)) {
+    if (crypto_digest(digest, start, len) < 0) {
       log_warn(LD_BUG,"couldn't compute digest");
       return -1;
     }
   } else {
-    if (crypto_digest256(digest, start, end-start, alg)) {
+    if (crypto_digest256(digest, start, len, alg) < 0) {
       log_warn(LD_BUG,"couldn't compute digest");
       return -1;
     }
@@ -5276,7 +4599,7 @@ router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests,
                        const char *end_str, char end_c)
 {
   const char *start=NULL, *end=NULL;
-  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,
+  if (router_get_hash_impl_helper(s,s_len,start_str,end_str,end_c,LOG_WARN,
                                   &start,&end)<0)
     return -1;
 
@@ -5446,11 +4769,13 @@ microdescs_parse_from_string(const char *s, const char *eos,
         if (!strcmp(t->args[0], "ed25519")) {
           if (md->ed25519_identity_pkey) {
             log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+            smartlist_free(id_lines);
             goto next;
           }
           ed25519_public_key_t k;
           if (ed25519_public_from_base64(&k, t->args[1])<0) {
             log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+            smartlist_free(id_lines);
             goto next;
           }
           md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
@@ -5476,7 +4801,7 @@ microdescs_parse_from_string(const char *s, const char *eos,
                    escaped(tok->args[i]));
           goto next;
         }
-        smartlist_add(md->family, tor_strdup(tok->args[i]));
+        smartlist_add_strdup(md->family, tok->args[i]);
       }
     }
 
@@ -5923,7 +5248,8 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
    * descriptor. */
   tok = find_by_keyword(tokens, R_PUBLICATION_TIME);
   tor_assert(tok->n_args == 1);
-  if (parse_iso_time_(tok->args[0], &result->timestamp, strict_time_fmt) < 0) {
+  if (parse_iso_time_(tok->args[0], &result->timestamp,
+                      strict_time_fmt, 0) < 0) {
     log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]);
     goto err;
   }
@@ -5967,7 +5293,10 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
                             "v2 rendezvous service descriptor") < 0)
     goto err;
   /* Verify that descriptor ID belongs to public key and secret ID part. */
-  crypto_pk_get_digest(result->pk, public_key_hash);
+  if (crypto_pk_get_digest(result->pk, public_key_hash) < 0) {
+    log_warn(LD_REND, "Unable to compute rend descriptor public key digest");
+    goto err;
+  }
   rend_get_descriptor_id_bytes(test_desc_id, public_key_hash,
                                secret_id_part);
   if (tor_memneq(desc_id_out, test_desc_id, DIGEST_LEN)) {
diff --git a/src/or/routerparse.h b/src/or/routerparse.h
index 01a5de88e8..088f773c5e 100644
--- a/src/or/routerparse.h
+++ b/src/or/routerparse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,11 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest);
 int router_get_dir_hash(const char *s, char *digest);
 int router_get_networkstatus_v3_hashes(const char *s,
                                        common_digests_t *digests);
+int router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                                  const char **start_out,
+                                                  const char **end_out);
+int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
+                                               const char *s);
 int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest);
 #define DIROBJ_MAX_SIG_LEN 256
 char *router_get_dirobj_signature(const char *digest,
@@ -113,7 +118,6 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
     (const char *dirname, const char *f));
 STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-STATIC void dump_desc(const char *desc, const char *type);
 STATIC void dump_desc_fifo_cleanup(void);
 struct memarea_t;
 STATIC routerstatus_t *routerstatus_parse_entry_from_string(
@@ -123,6 +127,12 @@ STATIC routerstatus_t *routerstatus_parse_entry_from_string(
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
                                      consensus_flavor_t flav);
+MOCK_DECL(STATIC void,dump_desc,(const char *desc, const char *type));
+MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
+                           const char *start, size_t len,
+                           digest_algorithm_t alg));
+MOCK_DECL(STATIC int, signed_digest_equals,
+          (const uint8_t *d1, const uint8_t *d2, size_t len));
 #endif
 
 #define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1"
diff --git a/src/or/routerset.c b/src/or/routerset.c
index 58b66ea777..4906c6a51d 100644
--- a/src/or/routerset.c
+++ b/src/or/routerset.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,6 +28,7 @@
 #define ROUTERSET_PRIVATE
 
 #include "or.h"
+#include "bridges.h"
 #include "geoip.h"
 #include "nodelist.h"
 #include "policies.h"
@@ -262,12 +263,12 @@ routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set)
     geoip_get_country("A1") >= 0;
 
   if (add_unknown) {
-    smartlist_add(set->country_names, tor_strdup("??"));
-    smartlist_add(set->list, tor_strdup("{??}"));
+    smartlist_add_strdup(set->country_names, "??");
+    smartlist_add_strdup(set->list, "{??}");
   }
   if (add_a1) {
-    smartlist_add(set->country_names, tor_strdup("a1"));
-    smartlist_add(set->list, tor_strdup("{a1}"));
+    smartlist_add_strdup(set->country_names, "a1");
+    smartlist_add_strdup(set->list, "{a1}");
   }
 
   if (add_unknown || add_a1) {
@@ -334,6 +335,18 @@ routerset_contains_node(const routerset_t *set, const node_t *node)
     return 0;
 }
 
+/** Return true iff <b>routerset</b> contains the bridge <b>bridge</b>. */
+int
+routerset_contains_bridge(const routerset_t *set, const bridge_info_t *bridge)
+{
+  const char *id = (const char*)bridge_get_rsa_id_digest(bridge);
+  const tor_addr_port_t *addrport = bridge_get_addr_port(bridge);
+
+  tor_assert(addrport);
+  return routerset_contains(set, &addrport->addr, addrport->port,
+                            NULL, id, -1);
+}
+
 /** Add every known node_t that is a member of <b>routerset</b> to
  * <b>out</b>, but never add any that are part of <b>excludeset</b>.
  * If <b>running_only</b>, only add the running ones. */
diff --git a/src/or/routerset.h b/src/or/routerset.h
index c2f7205c3e..a63677b471 100644
--- a/src/or/routerset.h
+++ b/src/or/routerset.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,8 +26,11 @@ int routerset_contains_routerstatus(const routerset_t *set,
                                     country_t country);
 int routerset_contains_extendinfo(const routerset_t *set,
                                   const extend_info_t *ei);
-
+struct bridge_info_t;
+int routerset_contains_bridge(const routerset_t *set,
+                              const struct bridge_info_t *bridge);
 int routerset_contains_node(const routerset_t *set, const node_t *node);
+
 void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
                              const routerset_t *excludeset,
                              int running_only);
diff --git a/src/or/scheduler.c b/src/or/scheduler.c
index 49ac1b939a..fac545fba7 100644
--- a/src/or/scheduler.c
+++ b/src/or/scheduler.c
@@ -1,11 +1,6 @@
-/* * Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-/**
- * \file scheduler.c
- * \brief Relay scheduling system
- **/
-
 #include "or.h"
 
 #define TOR_CHANNEL_INTERNAL_ /* For channel_flush_some_cells() */
@@ -32,66 +27,102 @@ static uint32_t sched_q_high_water = 32768;
 
 static uint32_t sched_max_flush_cells = 16;
 
-/*
- * Write scheduling works by keeping track of which channels can
+/**
+ * \file scheduler.c
+ * \brief Channel scheduling system: decides which channels should send and
+ * receive when.
+ *
+ * This module implements a scheduler algorithm, to decide
+ * which channels should send/receive when.
+ *
+ * The earliest versions of Tor approximated a kind of round-robin system
+ * among active connections, but only approximated it.
+ *
+ * Now, write scheduling works by keeping track of which channels can
  * accept cells, and have cells to write.  From the scheduler's perspective,
  * a channel can be in four possible states:
  *
- * 1.) Not open for writes, no cells to send
- *     - Not much to do here, and the channel will have scheduler_state ==
- *       SCHED_CHAN_IDLE
- *     - Transitions from:
- *       - Open for writes/has cells by simultaneously draining all circuit
+ * <ol>
+ * <li>
+ *   Not open for writes, no cells to send.
+ *     <ul><li> Not much to do here, and the channel will have scheduler_state
+ *       == SCHED_CHAN_IDLE
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Open for writes/has cells by simultaneously draining all circuit
  *         queues and filling the output buffer.
- *     - Transitions to:
- *       - Not open for writes/has cells by arrival of cells on an attached
+ *       </ul>
+ *     <li> Transitions to:
+ *      <ul>
+ *       <li> Not open for writes/has cells by arrival of cells on an attached
  *         circuit (this would be driven from append_cell_to_circuit_queue())
- *       - Open for writes/no cells by a channel type specific path;
+ *       <li> Open for writes/no cells by a channel type specific path;
  *         driven from connection_or_flushed_some() for channel_tls_t.
+ *      </ul>
+ *    </ul>
  *
- * 2.) Open for writes, no cells to send
- *     - Not much here either; this will be the state an idle but open channel
- *       can be expected to settle in.  It will have scheduler_state ==
- *       SCHED_CHAN_WAITING_FOR_CELLS
- *     - Transitions from:
- *       - Not open for writes/no cells by flushing some of the output
+ * <li> Open for writes, no cells to send
+ *   <ul>
+ *     <li>Not much here either; this will be the state an idle but open
+ *       channel can be expected to settle in.  It will have scheduler_state
+ *       == SCHED_CHAN_WAITING_FOR_CELLS
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/no cells by flushing some of the output
  *         buffer.
- *       - Open for writes/has cells by the scheduler moving cells from
+ *       <li>Open for writes/has cells by the scheduler moving cells from
  *         circuit queues to channel output queue, but not having enough
  *         to fill the output queue.
- *     - Transitions to:
- *       - Open for writes/has cells by arrival of new cells on an attached
+ *       </ul>
+ *     <li> Transitions to:
+ *       <ul>
+ *        <li>Open for writes/has cells by arrival of new cells on an attached
  *         circuit, in append_cell_to_circuit_queue()
+ *       </ul>
+ *     </ul>
  *
- * 3.) Not open for writes, cells to send
- *     - This is the state of a busy circuit limited by output bandwidth;
+ * <li>Not open for writes, cells to send
+ *     <ul>
+ *     <li>This is the state of a busy circuit limited by output bandwidth;
  *       cells have piled up in the circuit queues waiting to be relayed.
  *       The channel will have scheduler_state == SCHED_CHAN_WAITING_TO_WRITE.
- *     - Transitions from:
- *       - Not open for writes/no cells by arrival of cells on an attached
+ *     <li> Transitions from:
+ *       <ul>
+ *       <li>Not open for writes/no cells by arrival of cells on an attached
  *         circuit
- *       - Open for writes/has cells by filling an output buffer without
+ *       <li> Open for writes/has cells by filling an output buffer without
  *         draining all cells from attached circuits
- *    - Transitions to:
- *       - Opens for writes/has cells by draining some of the output buffer
+ *       </ul>
+ *    <li> Transitions to:
+ *       <ul>
+ *       <li>Opens for writes/has cells by draining some of the output buffer
  *         via the connection_or_flushed_some() path (for channel_tls_t).
+ *       </ul>
+ *    </ul>
  *
- * 4.) Open for writes, cells to send
- *     - This connection is ready to relay some cells and waiting for
+ * <li>Open for writes, cells to send
+ *     <ul>
+ *     <li>This connection is ready to relay some cells and waiting for
  *       the scheduler to choose it.  The channel will have scheduler_state ==
  *       SCHED_CHAN_PENDING.
- *     - Transitions from:
- *       - Not open for writes/has cells by the connection_or_flushed_some()
+ *     <li>Transitions from:
+ *       <ul>
+ *       <li> Not open for writes/has cells by the connection_or_flushed_some()
  *         path
- *       - Open for writes/no cells by the append_cell_to_circuit_queue()
+ *       <li> Open for writes/no cells by the append_cell_to_circuit_queue()
  *         path
- *     - Transitions to:
- *       - Not open for writes/no cells by draining all circuit queues and
- *         simultaneously filling the output buffer.
- *       - Not open for writes/has cells by writing enough cells to fill the
+ *       </ul>
+ *     <li> Transitions to:
+ *       <ul>
+ *        <li>Not open for writes/no cells by draining all circuit queues and
+ *          simultaneously filling the output buffer.
+ *        <li>Not open for writes/has cells by writing enough cells to fill the
  *         output buffer
- *       - Open for writes/no cells by draining all attached circuit queues
+ *        <li>Open for writes/no cells by draining all attached circuit queues
  *         without also filling the output buffer
+ *       </ul>
+ *    </ul>
+ * </ol>
  *
  * Other event-driven parts of the code move channels between these scheduling
  * states by calling scheduler functions; the scheduler only runs on open-for-
diff --git a/src/or/scheduler.h b/src/or/scheduler.h
index 3dcfd2faca..e29c13de7e 100644
--- a/src/or/scheduler.h
+++ b/src/or/scheduler.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* * Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/shared_random.c b/src/or/shared_random.c
index 5f6b03f1ba..25ca0611cd 100644
--- a/src/or/shared_random.c
+++ b/src/or/shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -192,7 +192,7 @@ verify_commit_and_reveal(const sr_commit_t *commit)
     /* Use the invariant length since the encoded reveal variable has an
      * extra byte for the NUL terminated byte. */
     if (crypto_digest256(received_hashed_reveal, commit->encoded_reveal,
-                         SR_REVEAL_BASE64_LEN, commit->alg)) {
+                         SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
       /* Unable to digest the reveal blob, this is unlikely. */
       goto invalid;
     }
@@ -230,9 +230,7 @@ commit_decode(const char *encoded, sr_commit_t *commit)
 {
   int decoded_len = 0;
   size_t offset = 0;
-  /* XXX: Needs two extra bytes for the base64 decode calculation matches
-   * the binary length once decoded. #17868. */
-  char b64_decoded[SR_COMMIT_LEN + 2];
+  char b64_decoded[SR_COMMIT_LEN];
 
   tor_assert(encoded);
   tor_assert(commit);
@@ -284,9 +282,7 @@ STATIC int
 reveal_decode(const char *encoded, sr_commit_t *commit)
 {
   int decoded_len = 0;
-  /* XXX: Needs two extra bytes for the base64 decode calculation matches
-   * the binary length once decoded. #17868. */
-  char b64_decoded[SR_REVEAL_LEN + 2];
+  char b64_decoded[SR_REVEAL_LEN];
 
   tor_assert(encoded);
   tor_assert(commit);
@@ -502,6 +498,20 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   return vote_line;
 }
 
+/* Convert a given srv object to a string for the control port. This doesn't
+ * fail and the srv object MUST be valid. */
+static char *
+srv_to_control_string(const sr_srv_t *srv)
+{
+  char *srv_str;
+  char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1];
+  tor_assert(srv);
+
+  sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv);
+  tor_asprintf(&srv_str, "%s", srv_hash_encoded);
+  return srv_str;
+}
+
 /* Return a heap allocated string that contains the given <b>srv</b> string
  * representation formatted for a networkstatus document using the
  * <b>key</b> as the start of the line. This doesn't return NULL. */
@@ -932,7 +942,7 @@ sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
   /* The invariant length is used here since the encoded reveal variable
    * has an extra byte added for the NULL terminated byte. */
   if (crypto_digest256(commit->hashed_reveal, commit->encoded_reveal,
-                       SR_REVEAL_BASE64_LEN, commit->alg)) {
+                       SR_REVEAL_BASE64_LEN, commit->alg) < 0) {
     goto error;
   }
 
@@ -1012,7 +1022,7 @@ sr_compute_srv(void)
     SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
     smartlist_free(chunks);
     if (crypto_digest256(hashed_reveals, reveals, strlen(reveals),
-                         SR_DIGEST_ALG)) {
+                         SR_DIGEST_ALG) < 0) {
       goto end;
     }
     current_srv = generate_srv(hashed_reveals, reveal_num,
@@ -1348,6 +1358,38 @@ sr_save_and_cleanup(void)
   sr_cleanup();
 }
 
+/* Return the current SRV string representation for the control port. Return a
+ * newly allocated string on success containing the value else "" if not found
+ * or if we don't have a valid consensus yet. */
+char *
+sr_get_current_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.current_srv) {
+    srv_str = srv_to_control_string(c->sr_info.current_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
+/* Return the previous SRV string representation for the control port. Return
+ * a newly allocated string on success containing the value else "" if not
+ * found or if we don't have a valid consensus yet. */
+char *
+sr_get_previous_for_control(void)
+{
+  char *srv_str;
+  const networkstatus_t *c = networkstatus_get_latest_consensus();
+  if (c && c->sr_info.previous_srv) {
+    srv_str = srv_to_control_string(c->sr_info.previous_srv);
+  } else {
+    srv_str = tor_strdup("");
+  }
+  return srv_str;
+}
+
 #ifdef TOR_UNIT_TESTS
 
 /* Set the global value of number of SRV agreements so the test can play
diff --git a/src/or/shared_random.h b/src/or/shared_random.h
index 9885934cc7..1f027c70e0 100644
--- a/src/or/shared_random.h
+++ b/src/or/shared_random.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_H
@@ -36,17 +36,14 @@
 
 /* Length of base64 encoded commit NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
-#define SR_COMMIT_BASE64_LEN \
-  (((SR_COMMIT_LEN - 1) / 3) * 4 + 4)
+#define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN))
 /* Length of base64 encoded reveal NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
-#define SR_REVEAL_BASE64_LEN \
-  (((SR_REVEAL_LEN - 1) / 3) * 4 + 4)
+#define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN))
 /* Length of base64 encoded shared random value. It's 32 bytes long so 44
  * bytes from the base64_encode_size formula. That includes the '='
  * character at the end. */
-#define SR_SRV_VALUE_BASE64_LEN \
-  (((DIGEST256_LEN - 1) / 3) * 4 + 4)
+#define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN))
 
 /* Assert if commit valid flag is not set. */
 #define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid)
@@ -129,6 +126,10 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+
+char *sr_get_current_for_control(void);
+char *sr_get_previous_for_control(void);
+
 #ifdef SHARED_RANDOM_PRIVATE
 
 /* Encode */
diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c
index 8438d46404..a147f5a347 100644
--- a/src/or/shared_random_state.c
+++ b/src/or/shared_random_state.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/shared_random_state.h b/src/or/shared_random_state.h
index 43a7f1d284..3526ad47d3 100644
--- a/src/or/shared_random_state.h
+++ b/src/or/shared_random_state.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_STATE_H
diff --git a/src/or/statefile.c b/src/or/statefile.c
index 8fa4324b25..d0606b3012 100644
--- a/src/or/statefile.c
+++ b/src/or/statefile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -102,6 +102,8 @@ static config_var_t state_vars_[] = {
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
+  V(Guard,                            LINELIST, NULL),
+
   V(TorVersion,                       STRING,   NULL),
 
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
diff --git a/src/or/statefile.h b/src/or/statefile.h
index b13743481d..10c09324bc 100644
--- a/src/or/statefile.h
+++ b/src/or/statefile.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_STATEFILE_H
diff --git a/src/or/status.c b/src/or/status.c
index fa2238b9f9..52763a7042 100644
--- a/src/or/status.c
+++ b/src/or/status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/or/status.h b/src/or/status.h
index b97e835037..c1a0033ce0 100644
--- a/src/or/status.h
+++ b/src/or/status.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_STATUS_H
diff --git a/src/or/tor_main.c b/src/or/tor_main.c
index d67eda2ac9..a3a8838602 100644
--- a/src/or/tor_main.c
+++ b/src/or/tor_main.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 extern const char tor_git_revision[];
diff --git a/src/or/torcert.c b/src/or/torcert.c
index a6a33c675a..658e620ca5 100644
--- a/src/or/torcert.c
+++ b/src/or/torcert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,8 +6,27 @@
  *
  * \brief Implementation for ed25519-signed certificates as used in the Tor
  * protocol.
+ *
+ * This certificate format is designed to be simple and compact; it's
+ * documented in tor-spec.txt in the torspec.git repository.  All of the
+ * certificates in this format are signed with an Ed25519 key; the
+ * contents themselves may be another Ed25519 key, a digest of a
+ * RSA key, or some other material.
+ *
+ * In this module there is also support for a crooss-certification of
+ * Ed25519 identities using (older) RSA1024 identities.
+ *
+ * Tor uses other types of certificate too, beyond those described in this
+ * module. Notably, our use of TLS requires us to touch X.509 certificates,
+ * even though sensible people would stay away from those. Our X.509
+ * certificates are represented with tor_x509_cert_t, and implemented in
+ * tortls.c.  We also have a separate certificate type that authorities
+ * use to authenticate their RSA signing keys with their RSA identity keys:
+ * that one is authority_cert_t, and it's mostly handled in routerlist.c.
  */
 
+#include "or.h"
+#include "config.h"
 #include "crypto.h"
 #include "torcert.h"
 #include "ed25519_cert.h"
@@ -137,7 +156,12 @@ tor_cert_parse(const uint8_t *encoded, const size_t len)
   cert->encoded_len = len;
 
   memcpy(cert->signed_key.pubkey, parsed->certified_key, 32);
-  cert->valid_until = parsed->exp_field * 3600;
+  int64_t valid_until_64 = ((int64_t)parsed->exp_field) * 3600;
+#if SIZEOF_TIME_T < SIZEOF_INT64_T
+  if (valid_until_64 > TIME_MAX)
+    valid_until_64 = TIME_MAX - 1;
+#endif
+  cert->valid_until = (time_t) valid_until_64;
   cert->cert_type = parsed->cert_type;
 
   for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) {
@@ -164,11 +188,17 @@ tor_cert_parse(const uint8_t *encoded, const size_t len)
 }
 
 /** Fill in <b>checkable_out</b> with the information needed to check
- * the signature on <b>cert</b> with <b>pubkey</b>. */
+ * the signature on <b>cert</b> with <b>pubkey</b>.
+ *
+ * On success, if <b>expiration_out</b> is provided, and it is some time
+ * _after_ the expiration time of this certificate, set it to the
+ * expiration time of this certificate.
+ */
 int
 tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
                            const tor_cert_t *cert,
-                           const ed25519_public_key_t *pubkey)
+                           const ed25519_public_key_t *pubkey,
+                           time_t *expiration_out)
 {
   if (! pubkey) {
     if (cert->signing_key_included)
@@ -185,6 +215,10 @@ tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
   memcpy(checkable_out->signature.sig,
          cert->encoded + signed_len, ED25519_SIG_LEN);
 
+  if (expiration_out) {
+    *expiration_out = MIN(*expiration_out, cert->valid_until);
+  }
+
   return 0;
 }
 
@@ -199,14 +233,15 @@ tor_cert_checksig(tor_cert_t *cert,
 {
   ed25519_checkable_t checkable;
   int okay;
+  time_t expires = TIME_MAX;
 
-  if (now && now > cert->valid_until) {
-    cert->cert_expired = 1;
+  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey, &expires) < 0)
     return -1;
-  }
 
-  if (tor_cert_get_checkable_sig(&checkable, cert, pubkey) < 0)
+  if (now && now > expires) {
+    cert->cert_expired = 1;
     return -1;
+  }
 
   if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) {
     cert->sig_bad = 1;
@@ -255,6 +290,8 @@ tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2)
   return tor_cert_eq(cert1, cert2);
 }
 
+#define RSA_ED_CROSSCERT_PREFIX "Tor TLS RSA/Ed25519 cross-certificate"
+
 /** Create new cross-certification object to certify <b>ed_key</b> as the
  * master ed25519 identity key for the RSA identity key <b>rsa_key</b>.
  * Allocates and stores the encoded certificate in *<b>cert</b>, and returns
@@ -265,6 +302,10 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
                                time_t expires,
                                uint8_t **cert)
 {
+  // It is later than 1985, since otherwise there would be no C89
+  // compilers. (Try to diagnose #22466.)
+  tor_assert_nonfatal(expires >= 15 * 365 * 86400);
+
   uint8_t *res;
 
   rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new();
@@ -279,11 +320,21 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
   ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc);
   tor_assert(sz > 0 && sz <= alloc_sz);
 
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+
   const int signed_part_len = 32 + 4;
+  crypto_digest_add_bytes(d, (char*)res, signed_part_len);
+
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
   int siglen = crypto_pk_private_sign(rsa_key,
                                       (char*)rsa_ed_crosscert_getarray_sig(cc),
                                       rsa_ed_crosscert_getlen_sig(cc),
-                                      (char*)res, signed_part_len);
+                                      (char*)digest, sizeof(digest));
   tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key));
   tor_assert(siglen <= UINT8_MAX);
   cc->sig_len = siglen;
@@ -295,3 +346,350 @@ tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
   return sz;
 }
 
+/**
+ * Check whether the <b>crosscert_len</b> byte certificate in <b>crosscert</b>
+ * is in fact a correct cross-certification of <b>master_key</b> using
+ * the RSA key <b>rsa_id_key</b>.
+ *
+ * Also reject the certificate if it expired before
+ * <b>reject_if_expired_before</b>.
+ *
+ * Return 0 on success, negative on failure.
+ */
+int
+rsa_ed25519_crosscert_check(const uint8_t *crosscert,
+                            const size_t crosscert_len,
+                            const crypto_pk_t *rsa_id_key,
+                            const ed25519_public_key_t *master_key,
+                            const time_t reject_if_expired_before)
+{
+  rsa_ed_crosscert_t *cc = NULL;
+  int rv;
+
+#define ERR(code, s)                                            \
+  do {                                                          \
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,                      \
+           "Received a bad RSA->Ed25519 crosscert: %s",         \
+           (s));                                                \
+    rv = (code);                                                \
+    goto err;                                                   \
+  } while (0)
+
+  if (BUG(crypto_pk_keysize(rsa_id_key) > PK_BYTES))
+    return -1;
+
+  if (BUG(!crosscert))
+    return -1;
+
+  ssize_t parsed_len = rsa_ed_crosscert_parse(&cc, crosscert, crosscert_len);
+  if (parsed_len < 0 || crosscert_len != (size_t)parsed_len) {
+    ERR(-2, "Unparseable or overlong crosscert");
+  }
+
+  if (tor_memneq(rsa_ed_crosscert_getarray_ed_key(cc),
+                 master_key->pubkey,
+                 ED25519_PUBKEY_LEN)) {
+    ERR(-3, "Crosscert did not match Ed25519 key");
+  }
+
+  const uint32_t expiration_date = rsa_ed_crosscert_get_expiration(cc);
+  const uint64_t expiration_time = expiration_date * 3600;
+
+  if (reject_if_expired_before < 0 ||
+      expiration_time < (uint64_t)reject_if_expired_before) {
+    ERR(-4, "Crosscert is expired");
+  }
+
+  const uint8_t *eos = rsa_ed_crosscert_get_end_of_signed(cc);
+  const uint8_t *sig = rsa_ed_crosscert_getarray_sig(cc);
+  const uint8_t siglen = rsa_ed_crosscert_get_sig_len(cc);
+  tor_assert(eos >= crosscert);
+  tor_assert((size_t)(eos - crosscert) <= crosscert_len);
+  tor_assert(siglen == rsa_ed_crosscert_getlen_sig(cc));
+
+  /* Compute the digest */
+  uint8_t digest[DIGEST256_LEN];
+  crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256);
+  crypto_digest_add_bytes(d, RSA_ED_CROSSCERT_PREFIX,
+                          strlen(RSA_ED_CROSSCERT_PREFIX));
+  crypto_digest_add_bytes(d, (char*)crosscert, eos-crosscert);
+  crypto_digest_get_digest(d, (char*)digest, sizeof(digest));
+  crypto_digest_free(d);
+
+  /* Now check the signature */
+  uint8_t signed_[PK_BYTES];
+  int signed_len = crypto_pk_public_checksig(rsa_id_key,
+                                          (char*)signed_, sizeof(signed_),
+                                          (char*)sig, siglen);
+  if (signed_len < DIGEST256_LEN) {
+    ERR(-5, "Bad signature, or length of signed data not as expected");
+  }
+
+  if (tor_memneq(digest, signed_, DIGEST256_LEN)) {
+    ERR(-6, "The signature was good, but it didn't match the data");
+  }
+
+  rv = 0;
+ err:
+  rsa_ed_crosscert_free(cc);
+  return rv;
+}
+
+/** Construct and return a new empty or_handshake_certs object */
+or_handshake_certs_t *
+or_handshake_certs_new(void)
+{
+  return tor_malloc_zero(sizeof(or_handshake_certs_t));
+}
+
+/** Release all storage held in <b>certs</b> */
+void
+or_handshake_certs_free(or_handshake_certs_t *certs)
+{
+  if (!certs)
+    return;
+
+  tor_x509_cert_free(certs->auth_cert);
+  tor_x509_cert_free(certs->link_cert);
+  tor_x509_cert_free(certs->id_cert);
+
+  tor_cert_free(certs->ed_id_sign);
+  tor_cert_free(certs->ed_sign_link);
+  tor_cert_free(certs->ed_sign_auth);
+  tor_free(certs->ed_rsa_crosscert);
+
+  memwipe(certs, 0xBD, sizeof(*certs));
+  tor_free(certs);
+}
+
+#undef ERR
+#define ERR(s)                                                  \
+  do {                                                          \
+    log_fn(severity, LD_PROTOCOL,                               \
+           "Received a bad CERTS cell: %s",                     \
+           (s));                                                \
+    return 0;                                                   \
+  } while (0)
+
+int
+or_handshake_certs_rsa_ok(int severity,
+                          or_handshake_certs_t *certs,
+                          tor_tls_t *tls,
+                          time_t now)
+{
+  tor_x509_cert_t *link_cert = certs->link_cert;
+  tor_x509_cert_t *auth_cert = certs->auth_cert;
+  tor_x509_cert_t *id_cert = certs->id_cert;
+
+  if (certs->started_here) {
+    if (! (id_cert && link_cert))
+      ERR("The certs we wanted (ID, Link) were missing");
+    if (! tor_tls_cert_matches_key(tls, link_cert))
+      ERR("The link certificate didn't match the TLS public key");
+    if (! tor_tls_cert_is_valid(severity, link_cert, id_cert, now, 0))
+      ERR("The link certificate was not valid");
+    if (! tor_tls_cert_is_valid(severity, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  } else {
+    if (! (id_cert && auth_cert))
+      ERR("The certs we wanted (ID, Auth) were missing");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, auth_cert, id_cert, now, 1))
+      ERR("The authentication certificate was not valid");
+    if (! tor_tls_cert_is_valid(LOG_PROTOCOL_WARN, id_cert, id_cert, now, 1))
+      ERR("The ID certificate was not valid");
+  }
+
+  return 1;
+}
+
+/** Check all the ed25519 certificates in <b>certs</b> against each other, and
+ * against the peer certificate in <b>tls</b> if appropriate.  On success,
+ * return 0; on failure, return a negative value and warn at level
+ * <b>severity</b> */
+int
+or_handshake_certs_ed25519_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now)
+{
+  ed25519_checkable_t check[10];
+  unsigned n_checkable = 0;
+  time_t expiration = TIME_MAX;
+
+#define ADDCERT(cert, pk)                                               \
+  do {                                                                  \
+    tor_assert(n_checkable < ARRAY_LENGTH(check));                      \
+    if (tor_cert_get_checkable_sig(&check[n_checkable++], cert, pk,     \
+                                   &expiration) < 0)                    \
+      ERR("Could not get checkable cert.");                             \
+  } while (0)
+
+  if (! certs->ed_id_sign || !certs->ed_id_sign->signing_key_included) {
+    ERR("No Ed25519 signing key");
+  }
+  ADDCERT(certs->ed_id_sign, NULL);
+
+  if (certs->started_here) {
+    if (! certs->ed_sign_link)
+      ERR("No Ed25519 link key");
+    {
+      /* check for a match with the TLS cert. */
+      tor_x509_cert_t *peer_cert = tor_tls_get_peer_cert(tls);
+      if (BUG(!peer_cert)) {
+        /* This is a bug, because if we got to this point, we are a connection
+         * that was initiated here, and we completed a TLS handshake. The
+         * other side *must* have given us a certificate! */
+        ERR("No x509 peer cert"); // LCOV_EXCL_LINE
+      }
+      const common_digests_t *peer_cert_digests =
+        tor_x509_cert_get_cert_digests(peer_cert);
+      int okay = tor_memeq(peer_cert_digests->d[DIGEST_SHA256],
+                           certs->ed_sign_link->signed_key.pubkey,
+                           DIGEST256_LEN);
+      tor_x509_cert_free(peer_cert);
+      if (!okay)
+        ERR("Link certificate does not match TLS certificate");
+    }
+
+    ADDCERT(certs->ed_sign_link, &certs->ed_id_sign->signed_key);
+
+  } else {
+    if (! certs->ed_sign_auth)
+      ERR("No Ed25519 link authentication key");
+    ADDCERT(certs->ed_sign_auth, &certs->ed_id_sign->signed_key);
+  }
+
+  if (expiration < now) {
+    ERR("At least one certificate expired.");
+  }
+
+  /* Okay, we've gotten ready to check all the Ed25519 certificates.
+   * Now, we are going to check the RSA certificate's cross-certification
+   * with the ED certificates.
+   *
+   * FFFF In the future, we might want to make this optional.
+   */
+
+  tor_x509_cert_t *rsa_id_cert = certs->id_cert;
+  if (!rsa_id_cert) {
+    ERR("Missing legacy RSA ID certificate");
+  }
+  if (! tor_tls_cert_is_valid(severity, rsa_id_cert, rsa_id_cert, now, 1)) {
+    ERR("The legacy RSA ID certificate was not valid");
+  }
+  if (! certs->ed_rsa_crosscert) {
+    ERR("Missing RSA->Ed25519 crosscert");
+  }
+  crypto_pk_t *rsa_id_key = tor_tls_cert_get_key(rsa_id_cert);
+  if (!rsa_id_key) {
+    ERR("RSA ID cert had no RSA key");
+  }
+
+  if (rsa_ed25519_crosscert_check(certs->ed_rsa_crosscert,
+                                  certs->ed_rsa_crosscert_len,
+                                  rsa_id_key,
+                                  &certs->ed_id_sign->signing_key,
+                                  now) < 0) {
+    crypto_pk_free(rsa_id_key);
+    ERR("Invalid RSA->Ed25519 crosscert");
+  }
+  crypto_pk_free(rsa_id_key);
+  rsa_id_key = NULL;
+
+  /* FFFF We could save a little time in the client case by queueing
+   * this batch to check it later, along with the signature from the
+   * AUTHENTICATE cell. That will change our data flow a bit, though,
+   * so I say "postpone". */
+
+  if (ed25519_checksig_batch(NULL, check, n_checkable) < 0) {
+    ERR("At least one Ed25519 certificate was badly signed");
+  }
+
+  return 1;
+}
+
+/**
+ * Check the Ed certificates and/or the RSA certificates, as appropriate.  If
+ * we obtained an Ed25519 identity, set *ed_id_out. If we obtained an RSA
+ * identity, set *rs_id_out. Otherwise, set them both to NULL.
+ */
+void
+or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out)
+{
+  tor_assert(ed_id_out);
+  tor_assert(rsa_id_out);
+
+  *ed_id_out = NULL;
+  *rsa_id_out = NULL;
+
+  if (certs->ed_id_sign) {
+    if (or_handshake_certs_ed25519_ok(severity, certs, tls, now)) {
+      tor_assert(certs->ed_id_sign);
+      tor_assert(certs->id_cert);
+
+      *ed_id_out = &certs->ed_id_sign->signing_key;
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+
+      /* If we reached this point, we did not look at any of the
+       * subsidiary RSA certificates, so we'd better just remove them.
+       */
+      tor_x509_cert_free(certs->link_cert);
+      tor_x509_cert_free(certs->auth_cert);
+      certs->link_cert = certs->auth_cert = NULL;
+    }
+    /* We do _not_ fall through here.  If you provided us Ed25519
+     * certificates, we expect to verify them! */
+  } else {
+    /* No ed25519 keys given in the CERTS cell */
+    if (or_handshake_certs_rsa_ok(severity, certs, tls, now)) {
+      *rsa_id_out = tor_x509_cert_get_id_digests(certs->id_cert);
+    }
+  }
+}
+
+/* === ENCODING === */
+
+/* Encode the ed25519 certificate <b>cert</b> and put the newly allocated
+ * string in <b>cert_str_out</b>. Return 0 on success else a negative value. */
+int
+tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out)
+{
+  int ret = -1;
+  char *ed_cert_b64 = NULL;
+  size_t ed_cert_b64_len;
+
+  tor_assert(cert);
+  tor_assert(cert_str_out);
+
+  /* Get the encoded size and add the NUL byte. */
+  ed_cert_b64_len = base64_encode_size(cert->encoded_len,
+                                       BASE64_ENCODE_MULTILINE) + 1;
+  ed_cert_b64 = tor_malloc_zero(ed_cert_b64_len);
+
+  /* Base64 encode the encoded certificate. */
+  if (base64_encode(ed_cert_b64, ed_cert_b64_len,
+                    (const char *) cert->encoded, cert->encoded_len,
+                    BASE64_ENCODE_MULTILINE) < 0) {
+    log_err(LD_BUG, "Couldn't base64-encode ed22519 cert!");
+    goto err;
+  }
+
+  /* Put everything together in a NUL terminated string. */
+  tor_asprintf(cert_str_out,
+               "-----BEGIN ED25519 CERT-----\n"
+               "%s"
+               "-----END ED25519 CERT-----",
+               ed_cert_b64);
+  /* Success! */
+  ret = 0;
+
+ err:
+  tor_free(ed_cert_b64);
+  return ret;
+}
+
diff --git a/src/or/torcert.h b/src/or/torcert.h
index 9c819c0abb..51f7665f1e 100644
--- a/src/or/torcert.h
+++ b/src/or/torcert.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TORCERT_H_INCLUDED
@@ -6,12 +6,15 @@
 
 #include "crypto_ed25519.h"
 
-#define SIGNED_KEY_TYPE_ED25519 0x01
+#define SIGNED_KEY_TYPE_ED25519     0x01
 
-#define CERT_TYPE_ID_SIGNING    0x04
-#define CERT_TYPE_SIGNING_LINK  0x05
-#define CERT_TYPE_SIGNING_AUTH  0x06
-#define CERT_TYPE_ONION_ID      0x0A
+#define CERT_TYPE_ID_SIGNING        0x04
+#define CERT_TYPE_SIGNING_LINK      0x05
+#define CERT_TYPE_SIGNING_AUTH      0x06
+#define CERT_TYPE_SIGNING_HS_DESC   0x08
+#define CERT_TYPE_AUTH_HS_IP_KEY    0x09
+#define CERT_TYPE_ONION_ID          0x0A
+#define CERT_TYPE_CROSS_HS_IP_KEYS  0x0B
 
 #define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1
 
@@ -57,8 +60,9 @@ tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen);
 void tor_cert_free(tor_cert_t *cert);
 
 int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out,
-                                 const tor_cert_t *out,
-                                 const ed25519_public_key_t *pubkey);
+                               const tor_cert_t *out,
+                               const ed25519_public_key_t *pubkey,
+                               time_t *expiration_out);
 
 int tor_cert_checksig(tor_cert_t *cert,
                       const ed25519_public_key_t *pubkey, time_t now);
@@ -71,6 +75,30 @@ ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key,
                                        const crypto_pk_t *rsa_key,
                                        time_t expires,
                                        uint8_t **cert);
+int rsa_ed25519_crosscert_check(const uint8_t *crosscert,
+                                const size_t crosscert_len,
+                                const crypto_pk_t *rsa_id_key,
+                                const ed25519_public_key_t *master_key,
+                                const time_t reject_if_expired_before);
+
+or_handshake_certs_t *or_handshake_certs_new(void);
+void or_handshake_certs_free(or_handshake_certs_t *certs);
+int or_handshake_certs_rsa_ok(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now);
+int or_handshake_certs_ed25519_ok(int severity,
+                                  or_handshake_certs_t *certs,
+                                  tor_tls_t *tls,
+                                  time_t now);
+void or_handshake_certs_check_both(int severity,
+                              or_handshake_certs_t *certs,
+                              tor_tls_t *tls,
+                              time_t now,
+                              const ed25519_public_key_t **ed_id_out,
+                              const common_digests_t **rsa_id_out);
+
+int tor_cert_encode_ed22519(const tor_cert_t *cert, char **cert_str_out);
 
 #endif
 
diff --git a/src/or/transports.c b/src/or/transports.c
index 7a52b737e4..31849a8d15 100644
--- a/src/or/transports.c
+++ b/src/or/transports.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -91,13 +91,13 @@
 
 #define PT_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "config.h"
 #include "circuitbuild.h"
 #include "transports.h"
 #include "util.h"
 #include "router.h"
 #include "statefile.h"
-#include "entrynodes.h"
 #include "connection_or.h"
 #include "ext_orport.h"
 #include "control.h"
@@ -430,7 +430,7 @@ add_transport_to_proxy(const char *transport, managed_proxy_t *mp)
 {
   tor_assert(mp->transports_to_launch);
   if (!smartlist_contains_string(mp->transports_to_launch, transport))
-    smartlist_add(mp->transports_to_launch, tor_strdup(transport));
+    smartlist_add_strdup(mp->transports_to_launch, transport);
 }
 
 /** Called when a SIGHUP occurs. Returns true if managed proxy
@@ -480,7 +480,6 @@ proxy_needs_restart(const managed_proxy_t *mp)
  *  preparations and then flag its state so that it will be relaunched
  *  in the next tick. */
 static void
-
 proxy_prepare_for_restart(managed_proxy_t *mp)
 {
   transport_t *t_tmp = NULL;
@@ -1322,7 +1321,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     tor_free(state_tmp);
   }
 
-  smartlist_add(envs, tor_strdup("TOR_PT_MANAGED_TRANSPORT_VER=1"));
+  smartlist_add_strdup(envs, "TOR_PT_MANAGED_TRANSPORT_VER=1");
 
   {
     char *transports_to_launch =
diff --git a/src/or/transports.h b/src/or/transports.h
index 7de90dcbec..44a9626e50 100644
--- a/src/or/transports.h
+++ b/src/or/transports.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in
new file mode 100644
index 0000000000..414b253a57
--- /dev/null
+++ b/src/rust/.cargo/config.in
@@ -0,0 +1,8 @@
+[source]
+
+@RUST_DL@ [source.crates-io]
+@RUST_DL@ registry = 'https://github.com/rust-lang/crates.io-index'
+@RUST_DL@ replace-with = 'vendored-sources'
+
+@RUST_DL@ [source.vendored-sources]
+@RUST_DL@ directory = '@RUST_DEPENDENCIES@'
diff --git a/src/rust/.rustfmt.toml b/src/rust/.rustfmt.toml
new file mode 100644
index 0000000000..f25bd51883
--- /dev/null
+++ b/src/rust/.rustfmt.toml
@@ -0,0 +1,2 @@
+max_width = 80
+comment_width = 80
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
new file mode 100644
index 0000000000..4ac9606ce8
--- /dev/null
+++ b/src/rust/Cargo.lock
@@ -0,0 +1,14 @@
+[root]
+name = "tor_util"
+version = "0.0.1"
+dependencies = [
+ "libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)" = "babb8281da88cba992fa1f4ddec7d63ed96280a1a53ec9b919fd37b53d71e502"
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
new file mode 100644
index 0000000000..fc4377e8b4
--- /dev/null
+++ b/src/rust/Cargo.toml
@@ -0,0 +1,7 @@
+[workspace]
+members = ["tor_util"]
+
+[profile.release]
+debug = true
+panic = "abort"
+
diff --git a/src/rust/include.am b/src/rust/include.am
new file mode 100644
index 0000000000..20afc6c4db
--- /dev/null
+++ b/src/rust/include.am
@@ -0,0 +1,6 @@
+include src/rust/tor_util/include.am
+
+EXTRA_DIST +=\
+	src/rust/Cargo.toml \
+	src/rust/Cargo.lock \
+	src/rust/.cargo/config.in
diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml
new file mode 100644
index 0000000000..f175fbdfb0
--- /dev/null
+++ b/src/rust/tor_util/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+authors = ["The Tor Project"]
+name = "tor_util"
+version = "0.0.1"
+
+[lib]
+name = "tor_util"
+path = "lib.rs"
+crate_type = ["rlib", "staticlib"]
+
+[dependencies]
+libc = "*"
+
diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs
new file mode 100644
index 0000000000..af4bfc41af
--- /dev/null
+++ b/src/rust/tor_util/ffi.rs
@@ -0,0 +1,56 @@
+//! FFI functions, only to be called from C.
+//!
+//! Equivalent C versions of these live in `src/common/compat_rust.c`
+
+use std::mem::forget;
+use std::ffi::CString;
+
+use libc;
+use rust_string::RustString;
+
+/// Free the passed `RustString` (`rust_str_t` in C), to be used in place of
+/// `tor_free`().
+///
+/// # Examples
+/// ```c
+/// rust_str_t r_s = rust_welcome_string();
+/// rust_str_free(r_s);
+/// ```
+#[no_mangle]
+#[cfg_attr(feature = "cargo-clippy", allow(needless_pass_by_value))]
+pub unsafe extern "C" fn rust_str_free(_str: RustString) {
+    // Empty body: Just drop _str and we're done (Drop takes care of it).
+}
+
+/// Lends an immutable, NUL-terminated C String.
+///
+/// # Examples
+/// ```c
+/// rust_str_t r_s = rust_welcome_string();
+/// const char *s = rust_str_get(r_s);
+/// printf("%s", s);
+/// rust_str_free(r_s);
+/// ```
+#[no_mangle]
+pub unsafe extern "C" fn rust_str_get(str: RustString) -> *const libc::c_char {
+    let res = str.as_ptr();
+    forget(str);
+    res
+}
+
+/// Returns a short string to announce Rust support during startup.
+///
+/// # Examples
+/// ```c
+/// rust_str_t r_s = rust_welcome_string();
+/// const char *s = rust_str_get(r_s);
+/// printf("%s", s);
+/// rust_str_free(r_s);
+/// ```
+#[no_mangle]
+pub extern "C" fn rust_welcome_string() -> RustString {
+    let s = CString::new("Tor is running with Rust integration. Please report \
+                          any bugs you encouter.")
+            .unwrap();
+    RustString::from(s)
+}
diff --git a/src/rust/tor_util/include.am b/src/rust/tor_util/include.am
new file mode 100644
index 0000000000..f0cd63920c
--- /dev/null
+++ b/src/rust/tor_util/include.am
@@ -0,0 +1,13 @@
+EXTRA_DIST +=\
+	src/rust/tor_util/Cargo.toml \
+	src/rust/tor_util/lib.rs \
+	src/rust/tor_util/ffi.rs \
+	src/rust/tor_util/rust_string.rs
+
+src/rust/target/release/libtor_util.a: FORCE
+	( cd "$(abs_top_srcdir)/src/rust/tor_util" ; \
+		CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \
+		CARGO_HOME="$(abs_top_builddir)/src/rust" \
+		$(CARGO) build --release --quiet $(CARGO_ONLINE) )
+
+FORCE:
diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs
new file mode 100644
index 0000000000..79d583d1ae
--- /dev/null
+++ b/src/rust/tor_util/lib.rs
@@ -0,0 +1,13 @@
+//! C <-> Rust compatibility helpers and types.
+//!
+//! Generically useful, small scale helpers should go here. This goes for both
+//! the C side (in the form of the ffi module) as well as the Rust side
+//! (individual modules per functionality). The corresponding C stuff lives in
+//! `src/common/compat_rust.{c,h}`.
+
+extern crate libc;
+
+mod rust_string;
+pub mod ffi;
+
+pub use rust_string::*;
diff --git a/src/rust/tor_util/rust_string.rs b/src/rust/tor_util/rust_string.rs
new file mode 100644
index 0000000000..46ec3fd7a8
--- /dev/null
+++ b/src/rust/tor_util/rust_string.rs
@@ -0,0 +1,101 @@
+use std::ffi::CString;
+use std::mem::forget;
+use libc;
+
+/// Compatibility wrapper for strings allocated in Rust and passed to C.
+///
+/// Rust doesn't ensure the safety of freeing memory across an FFI boundary, so
+/// we need to take special care to ensure we're not accidentally calling
+/// `tor_free`() on any string allocated in Rust. To more easily differentiate
+/// between strings that possibly (if Rust support is enabled) were allocated
+/// in Rust, C has the `rust_str_t` helper type. The equivalent on the Rust
+/// side is `RustString`.
+///
+/// Note: This type must not be used for strings allocated in C.
+#[repr(C)]
+#[derive(Debug)]
+pub struct RustString(*mut libc::c_char);
+
+impl RustString {
+    /// Returns a pointer to the underlying NUL-terminated byte array.
+    ///
+    /// Note that this function is not typically useful for Rust callers,
+    /// except in a direct FFI context.
+    ///
+    /// # Examples
+    /// ```
+    /// # use tor_util::RustString;
+    /// use std::ffi::CString;
+    ///
+    /// let r = RustString::from(CString::new("asdf").unwrap());
+    /// let c_str = r.as_ptr();
+    /// assert_eq!(b'a', unsafe { *c_str as u8});
+    /// ```
+    pub fn as_ptr(&self) -> *const libc::c_char {
+        self.0 as *const libc::c_char
+    }
+}
+
+impl From<CString> for RustString {
+    /// Constructs a new `RustString`
+    ///
+    /// # Examples
+    /// ```
+    /// # use tor_util::RustString;
+    /// use std::ffi::CString;
+    ///
+    /// let r = RustString::from(CString::new("asdf").unwrap());
+    /// ```
+    fn from(str: CString) -> RustString {
+        RustString(str.into_raw())
+    }
+}
+
+impl Into<CString> for RustString {
+    /// Reconstructs a `CString` from this `RustString`.
+    ///
+    /// Useful to take ownership back from a `RustString` that was given to C
+    /// code.
+    ///
+    /// # Examples
+    /// ```
+    /// # use tor_util::RustString;
+    /// use std::ffi::CString;
+    ///
+    /// let cs = CString::new("asdf").unwrap();
+    /// let r = RustString::from(cs.clone());
+    /// let cs2 = r.into();
+    /// assert_eq!(cs, cs2);
+    /// ```
+    fn into(self) -> CString {
+        // Calling from_raw is always OK here: We only construct self using
+        // valid CStrings and don't expose anything that could mutate it
+        let ret = unsafe { CString::from_raw(self.0) };
+        forget(self);
+        ret
+    }
+}
+
+impl Drop for RustString {
+    fn drop(&mut self) {
+        // Don't use into() here, because we would need to move out of
+        // self. Same safety consideration. Immediately drop the created
+        // CString, which takes care of freeing the wrapped string.
+        unsafe { CString::from_raw(self.0) };
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::mem;
+    use super::*;
+
+    use libc;
+
+    /// Ensures we're not adding overhead by using RustString.
+    #[test]
+    fn size_of() {
+        assert_eq!(mem::size_of::<*mut libc::c_char>(),
+                   mem::size_of::<RustString>())
+    }
+}
diff --git a/src/rust/tor_util/tests/rust_string.rs b/src/rust/tor_util/tests/rust_string.rs
new file mode 100644
index 0000000000..1ff605a43c
--- /dev/null
+++ b/src/rust/tor_util/tests/rust_string.rs
@@ -0,0 +1,37 @@
+extern crate tor_util;
+extern crate libc;
+
+use std::ffi::CString;
+use tor_util::RustString;
+
+#[test]
+fn rust_string_conversions_preserve_c_string() {
+    let s = CString::new("asdf foo").unwrap();
+    let r = RustString::from(s.clone());
+    let r2 = RustString::from(s.clone());
+    let c = r2.as_ptr();
+    assert_eq!(unsafe { libc::strlen(c) }, 8);
+    let c_str = r.into();
+    assert_eq!(s, c_str);
+}
+
+#[test]
+fn empty_string() {
+    let s = CString::new("").unwrap();
+    let r = RustString::from(s.clone());
+    let c = r.as_ptr();
+    assert_eq!(unsafe { libc::strlen(c) }, 0);
+    let c_str = r.into();
+    assert_eq!(s, c_str);
+}
+
+#[test]
+fn c_string_with_unicode() {
+    // The euro sign is three bytes
+    let s = CString::new("asd€asd").unwrap();
+    let r = RustString::from(s.clone());
+    let c = r.as_ptr();
+    assert_eq!(unsafe { libc::strlen(c) }, 9);
+    let c_str = r.into();
+    assert_eq!(s, c_str);
+}
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index 0ba56d7036..605f1a92c3 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -12,11 +12,12 @@ LIBS = ..\..\..\build-alpha\lib\libevent.lib \
  crypt32.lib gdi32.lib user32.lib
 
 TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
-        test_containers.obj \
+        test_consdiff.obj test_containers.obj \
 	test_controller_events.obj test_crypto.obj test_data.obj test_dir.obj \
 	test_checkdir.obj test_microdesc.obj test_pt.obj test_util.obj \
         test_config.obj test_connection.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
+	test_channelpadding.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
diff --git a/src/test/bench.c b/src/test/bench.c
index 30984fda70..a44dc94a61 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 extern const char tor_git_revision[];
@@ -28,6 +28,7 @@ const char tor_git_revision[] = "";
 #include "crypto_curve25519.h"
 #include "onion_ntor.h"
 #include "crypto_ed25519.h"
+#include "consdiff.h"
 
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
@@ -120,7 +121,7 @@ bench_onion_TAP(void)
   uint64_t start, end;
   char os[TAP_ONIONSKIN_CHALLENGE_LEN];
   char or[TAP_ONIONSKIN_REPLY_LEN];
-  crypto_dh_t *dh_out;
+  crypto_dh_t *dh_out = NULL;
 
   key = crypto_pk_new();
   key2 = crypto_pk_new();
@@ -175,6 +176,7 @@ bench_onion_TAP(void)
          NANOCOUNT(start, end, iters)/1e3);
 
  done:
+  crypto_dh_free(dh_out);
   crypto_pk_free(key);
   crypto_pk_free(key2);
 }
@@ -672,6 +674,28 @@ main(int argc, const char **argv)
   or_options_t *options;
 
   tor_threads_init();
+  tor_compress_init();
+
+  if (argc == 4 && !strcmp(argv[1], "diff")) {
+    init_logging(1);
+    const int N = 200;
+    char *f1 = read_file_to_str(argv[2], RFTS_BIN, NULL);
+    char *f2 = read_file_to_str(argv[3], RFTS_BIN, NULL);
+    if (! f1 || ! f2) {
+      perror("X");
+      return 1;
+    }
+    for (i = 0; i < N; ++i) {
+      char *diff = consensus_diff_generate(f1, f2);
+      tor_free(diff);
+    }
+    char *diff = consensus_diff_generate(f1, f2);
+    printf("%s", diff);
+    tor_free(f1);
+    tor_free(f2);
+    tor_free(diff);
+    return 0;
+  }
 
   for (i = 1; i < argc; ++i) {
     if (!strcmp(argv[i], "--list")) {
diff --git a/src/test/bt_test.py b/src/test/bt_test.py
index 30591453b9..4cb3326042 100755
--- a/src/test/bt_test.py
+++ b/src/test/bt_test.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2015, The Tor Project, Inc
+# Copyright 2013-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py
index d5a3a79910..af5010415e 100644
--- a/src/test/ed25519_exts_ref.py
+++ b/src/test/ed25519_exts_ref.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2014-2015, The Tor Project, Inc
+# Copyright 2014-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/fakechans.h b/src/test/fakechans.h
index fa0e37dbe6..c0de430e3d 100644
--- a/src/test/fakechans.h
+++ b/src/test/fakechans.h
@@ -1,4 +1,4 @@
- /* Copyright (c) 2014-2016, The Tor Project, Inc. */
+ /* Copyright (c) 2014-2017, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 #ifndef TOR_FAKECHANS_H
diff --git a/src/test/fuzz/dict/consensus b/src/test/fuzz/dict/consensus
new file mode 100644
index 0000000000..3fcd9ee7ff
--- /dev/null
+++ b/src/test/fuzz/dict/consensus
@@ -0,0 +1,52 @@
+"a"
+"additional-digest"
+"additional-signature"
+"bandwidth-weights"
+"client-versions"
+"consensus-digest"
+"consensus-method"
+"consensus-methods"
+"contact"
+"dir-address"
+"directory-footer"
+"directory-signature"
+"dir-identity-key"
+"dir-key-certificate-version"
+"dir-key-certification"
+"dir-key-crosscert"
+"dir-key-expires"
+"dir-key-published"
+"dir-signing-key"
+"dir-source"
+"fingerprint"
+"fresh-until"
+"id"
+"known-flags"
+"legacy-dir-key"
+"m"
+"network-status-version"
+"opt"
+"p"
+"package"
+"params"
+"pr"
+"published"
+"r"
+"recommended-client-protocols"
+"recommended-relay-protocols"
+"required-client-protocols"
+"required-relay-protocols"
+"s"
+"server-versions"
+"shared-rand-commit"
+"shared-rand-current-value"
+"shared-rand-participate"
+"shared-rand-previous-value"
+"signing-ed25519"
+"v"
+"valid-after"
+"valid-until"
+"vote-digest"
+"vote-status"
+"voting-delay"
+"w"
diff --git a/src/test/fuzz/dict/descriptor b/src/test/fuzz/dict/descriptor
new file mode 100644
index 0000000000..110ee3e820
--- /dev/null
+++ b/src/test/fuzz/dict/descriptor
@@ -0,0 +1,41 @@
+"reject"
+"accept"
+"reject6"
+"accept6"
+"router"
+"ipv6-policy"
+"signing-key"
+"onion-key"
+"ntor-onion-key"
+"router-signature"
+"published"
+"uptime"
+"fingerprint"
+"hibernating"
+"platform"
+"proto"
+"contact"
+"read-history"
+"write-history"
+"extra-info-digest"
+"hidden-service-dir"
+"identity-ed25519"
+"master-key-ed25519"
+"router-sig-ed25519"
+"onion-key-crosscert"
+"ntor-onion-key-crosscert"
+"allow-single-hop-exits"
+"family"
+"caches-extra-info"
+"or-address"
+"opt"
+ "bandwidth"
+"@purpose"
+"tunnelled-dir-server"
+"-----BEGIN"
+"-----END"
+"-----"
+"ED25519 CERT"
+"RSA PUBLIC KEY"
+"CROSSCERT"
+"SIGNATURE"
diff --git a/src/test/fuzz/dict/extrainfo b/src/test/fuzz/dict/extrainfo
new file mode 100644
index 0000000000..eba7a1e4ce
--- /dev/null
+++ b/src/test/fuzz/dict/extrainfo
@@ -0,0 +1,32 @@
+"cell-circuits-per-decile"
+"cell-processed-cells"
+"cell-queued-cells"
+"cell-stats-end"
+"cell-time-in-queue"
+"dirreq-stats-end"
+"dirreq-v2-direct-dl"
+"dirreq-v2-ips"
+"dirreq-v2-reqs"
+"dirreq-v2-resp"
+"dirreq-v2-share"
+"dirreq-v2-tunneled-dl"
+"dirreq-v3-direct-dl"
+"dirreq-v3-ips"
+"dirreq-v3-reqs"
+"dirreq-v3-resp"
+"dirreq-v3-share"
+"dirreq-v3-tunneled-dl"
+"entry-ips"
+"entry-stats-end"
+"exit-kibibytes-read"
+"exit-kibibytes-written"
+"exit-stats-end"
+"exit-streams-opened"
+"extra-info"
+"identity-ed25519"
+"opt"
+"published"
+"read-history"
+"router-sig-ed25519"
+"router-signature"
+"write-history"
diff --git a/src/test/fuzz/dict/hsdescv2 b/src/test/fuzz/dict/hsdescv2
new file mode 100644
index 0000000000..48788301dc
--- /dev/null
+++ b/src/test/fuzz/dict/hsdescv2
@@ -0,0 +1,8 @@
+"introduction-points"
+"permanent-key"
+"protocol-versions"
+"publication-time"
+"rendezvous-service-descriptor"
+"secret-id-part"
+"signature"
+"version"
diff --git a/src/test/fuzz/dict/http b/src/test/fuzz/dict/http
new file mode 100644
index 0000000000..3b0531579d
--- /dev/null
+++ b/src/test/fuzz/dict/http
@@ -0,0 +1,24 @@
+#
+# AFL dictionary for the Tor Directory protocol's HTTP headers
+# ------------------------------------------------------------
+#
+# Extracted from directory_handle_command() in the tor source code
+#
+# Copyright (c) 2016-2017, The Tor Project, Inc.
+# See LICENSE for licensing information
+#
+# Usage:
+# Select the dictionaries relevant to the part of the directory protocol you
+# are fuzzing, and feed them to your fuzzer (if it supports dictionaries).
+
+http_header_body_delimiter = "\x0d\x0a\x0d\x0a"
+http_header_header_delimiter = "\x0d\x0a"
+# multi-character tokens only
+#http_header_value_delimiter = " "
+
+content_length_header = "Content-Length:"
+forwarded_for_header = "Forwarded-For:"
+x_forwarded_for_header = "X-Forwarded-For:"
+
+get_command = "GET"
+post_command = "POST"
diff --git a/src/test/fuzz/dict/iptsv2 b/src/test/fuzz/dict/iptsv2
new file mode 100644
index 0000000000..57791c5e3c
--- /dev/null
+++ b/src/test/fuzz/dict/iptsv2
@@ -0,0 +1,6 @@
+"introduction-point"
+"ip-address"
+"onion-port"
+"onion-key"
+"service-key"
+
diff --git a/src/test/fuzz/dict/microdesc b/src/test/fuzz/dict/microdesc
new file mode 100644
index 0000000000..fdd0567b65
--- /dev/null
+++ b/src/test/fuzz/dict/microdesc
@@ -0,0 +1,7 @@
+"onion-key"
+"ntor-onion-key"
+"id"
+"a"
+"family"
+"p"
+"p6"
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
new file mode 100755
index 0000000000..68efc1abc5
--- /dev/null
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+if [ ! -d "$1" ] ; then
+    echo "I need a directory"
+    exit 1
+fi
+
+for fn in "$1"/* ; do
+    prev=`basename "$fn"`
+    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
+    if [ "$prev" == "$post" ] ; then
+      echo "OK $prev"
+    else
+      echo "mv $prev $post"
+      mv "$fn" "$1/$post"
+    fi
+done
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
new file mode 100644
index 0000000000..6610ade7ad
--- /dev/null
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -0,0 +1,78 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "networkstatus.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+static int
+mock_signed_digest_equals__yes(const uint8_t *d1, const uint8_t *d2,
+                               size_t len)
+{
+  (void) tor_memeq(d1, d2, len);
+  return 1;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  MOCK(signed_digest_equals, mock_signed_digest_equals__yes);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  networkstatus_t *ns;
+  char *str = tor_memdup_nulterm(data, sz);
+  const char *eos = NULL;
+  networkstatus_type_t tp = NS_TYPE_CONSENSUS;
+  if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
+    tp = NS_TYPE_VOTE;
+  const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
+  ns = networkstatus_parse_vote_from_string(str,
+                                            &eos,
+                                            tp);
+  if (ns) {
+    log_debug(LD_GENERAL, "Parsing as %s okay", what);
+    networkstatus_vote_free(ns);
+  } else {
+    log_debug(LD_GENERAL, "Parsing as %s failed", what);
+  }
+  tor_free(str);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c
new file mode 100644
index 0000000000..1a50beae17
--- /dev/null
+++ b/src/test/fuzz/fuzz_descriptor.c
@@ -0,0 +1,79 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "routerlist.h"
+#include "routerkeys.h"
+#include "fuzzing.h"
+
+static int
+mock_check_tap_onion_key_crosscert__nocheck(const uint8_t *crosscert,
+                                   int crosscert_len,
+                                   const crypto_pk_t *onion_pkey,
+                                   const ed25519_public_key_t *master_id_pkey,
+                                   const uint8_t *rsa_id_digest)
+{
+  tor_assert(crosscert && onion_pkey && master_id_pkey && rsa_id_digest);
+  /* we could look at crosscert[..] */
+  (void) crosscert_len;
+  return 0;
+}
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(check_tap_onion_key_crosscert,
+       mock_check_tap_onion_key_crosscert__nocheck);
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  routerinfo_t *ri;
+  const char *str = (const char*) data;
+  ri = router_parse_entry_from_string((const char *)str,
+                                      str+sz,
+                                      0, 0, 0, NULL);
+  if (ri) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    routerinfo_free(ri);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
new file mode 100644
index 0000000000..642380b512
--- /dev/null
+++ b/src/test/fuzz/fuzz_diff.c
@@ -0,0 +1,69 @@
+/* Copyright (c) 2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "orconfig.h"
+#include "or.h"
+#include "consdiff.h"
+
+#include "fuzzing.h"
+
+static int
+mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+{
+  (void)c;
+  memset(d->sha3_256, 3, sizeof(d->sha3_256));
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  MOCK(consensus_compute_digest, mock_consensus_compute_digest_);
+  MOCK(consensus_compute_digest_as_signed, mock_consensus_compute_digest_);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(consensus_compute_digest);
+  UNMOCK(consensus_compute_digest_as_signed);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+#define SEP "=====\n"
+#define SEPLEN strlen(SEP)
+  const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN);
+  if (! separator)
+    return 0;
+  size_t c1_len = separator - stdin_buf;
+  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  size_t c2_len = data_size - c1_len - SEPLEN;
+  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+
+  char *c3 = consensus_diff_generate(c1, c2);
+
+  if (c3) {
+    char *c4 = consensus_diff_apply(c1, c3);
+    tor_assert(c4);
+    if (strcmp(c2, c4)) {
+      printf("%s\n", escaped(c1));
+      printf("%s\n", escaped(c2));
+      printf("%s\n", escaped(c3));
+      printf("%s\n", escaped(c4));
+    }
+    tor_assert(! strcmp(c2, c4));
+    tor_free(c3);
+    tor_free(c4);
+  }
+  tor_free(c1);
+  tor_free(c2);
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
new file mode 100644
index 0000000000..8d7bf751bf
--- /dev/null
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "orconfig.h"
+#include "or.h"
+#include "consdiff.h"
+
+#include "fuzzing.h"
+
+static int
+mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+{
+  (void)c;
+  memset(d->sha3_256, 3, sizeof(d->sha3_256));
+  return 0;
+}
+
+static int
+mock_consensus_digest_eq_(const uint8_t *a, const uint8_t *b)
+{
+  (void)a;
+  (void)b;
+  return 1;
+}
+
+int
+fuzz_init(void)
+{
+  MOCK(consensus_compute_digest, mock_consensus_compute_digest_);
+  MOCK(consensus_digest_eq, mock_consensus_digest_eq_);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(consensus_compute_digest);
+  UNMOCK(consensus_digest_eq);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+#define SEP "=====\n"
+#define SEPLEN strlen(SEP)
+  const uint8_t *separator = tor_memmem(stdin_buf, data_size, SEP, SEPLEN);
+  if (! separator)
+    return 0;
+  size_t c1_len = separator - stdin_buf;
+  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  size_t c2_len = data_size - c1_len - SEPLEN;
+  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+
+  char *c3 = consensus_diff_apply(c1, c2);
+
+  tor_free(c1);
+  tor_free(c2);
+  tor_free(c3);
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c
new file mode 100644
index 0000000000..2a3de7ecf7
--- /dev/null
+++ b/src/test/fuzz/fuzz_extrainfo.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "routerlist.h"
+#include "routerkeys.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static int
+mock_router_produce_hash_final__nohash(char *digest,
+                                       const char *start, size_t len,
+                                       digest_algorithm_t alg)
+{
+  (void)start;
+  (void)len;
+  /* we could look at start[..] */
+  if (alg == DIGEST_SHA1)
+    memset(digest, 0x01, 20);
+  else
+    memset(digest, 0x02, 32);
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  MOCK(router_compute_hash_final, mock_router_produce_hash_final__nohash);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  extrainfo_t *ei;
+  const char *str = (const char*) data;
+  int again = 0;
+  ei = extrainfo_parse_entry_from_string((const char *)str,
+                                         str+sz,
+                                         0, NULL, &again);
+  if (ei) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    extrainfo_free(ei);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c
new file mode 100644
index 0000000000..19db265716
--- /dev/null
+++ b/src/test/fuzz/fuzz_hsdescv2.c
@@ -0,0 +1,52 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "rendcommon.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  rend_service_descriptor_t *desc = NULL;
+  char desc_id[64];
+  char *ipts = NULL;
+  size_t ipts_size, esize;
+  const char *next;
+  char *str = tor_memdup_nulterm(data, sz);
+  (void) rend_parse_v2_service_descriptor(&desc, desc_id, &ipts, &ipts_size,
+                                          &esize, &next, str, 1);
+  if (desc) {
+    log_debug(LD_GENERAL, "Parsing okay");
+    rend_service_descriptor_free(desc);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  tor_free(ipts);
+  tor_free(str);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
new file mode 100644
index 0000000000..2ffeb60244
--- /dev/null
+++ b/src/test/fuzz/fuzz_http.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define BUFFERS_PRIVATE
+#define DIRECTORY_PRIVATE
+
+#include "or.h"
+#include "backtrace.h"
+#include "buffers.h"
+#include "config.h"
+#include "connection.h"
+#include "directory.h"
+#include "torlog.h"
+
+#include "fuzzing.h"
+
+static void
+mock_connection_write_to_buf_impl_(const char *string, size_t len,
+                                   connection_t *conn, int compressed)
+{
+  log_debug(LD_GENERAL, "%sResponse:\n%u\nConnection: %p\n%s\n",
+            compressed ? "Compressed " : "", (unsigned)len, conn, string);
+}
+
+static int
+mock_directory_handle_command_get(dir_connection_t *conn,
+                                      const char *headers,
+                                      const char *body,
+                                      size_t body_len)
+{
+  (void)conn;
+
+  log_debug(LD_GENERAL, "Method:\nGET\n");
+
+  if (headers) {
+    log_debug(LD_GENERAL, "Header-Length:\n%u\n", (unsigned)strlen(headers));
+    log_debug(LD_GENERAL, "Headers:\n%s\n", headers);
+  }
+
+  log_debug(LD_GENERAL, "Body-Length:\n%u\n", (unsigned)body_len);
+  if (body) {
+    log_debug(LD_GENERAL, "Body:\n%s\n", body);
+  }
+
+  /* Always tell the caller we succeeded */
+  return 0;
+}
+
+static int
+mock_directory_handle_command_post(dir_connection_t *conn,
+                                       const char *headers,
+                                       const char *body,
+                                       size_t body_len)
+{
+  (void)conn;
+
+  log_debug(LD_GENERAL, "Method:\nPOST\n");
+
+  if (headers) {
+    log_debug(LD_GENERAL, "Header-Length:\n%u\n", (unsigned)strlen(headers));
+    log_debug(LD_GENERAL, "Headers:\n%s\n", headers);
+  }
+
+  log_debug(LD_GENERAL, "Body-Length:\n%u\n", (unsigned)body_len);
+  if (body) {
+    log_debug(LD_GENERAL, "Body:\n%s\n", body);
+  }
+
+  /* Always tell the caller we succeeded */
+  return 0;
+}
+
+int
+fuzz_init(void)
+{
+  /* Set up fake response handler */
+  MOCK(connection_write_to_buf_impl_, mock_connection_write_to_buf_impl_);
+  /* Set up the fake handler functions */
+  MOCK(directory_handle_command_get, mock_directory_handle_command_get);
+  MOCK(directory_handle_command_post, mock_directory_handle_command_post);
+
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(directory_handle_command_get);
+  UNMOCK(directory_handle_command_post);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  dir_connection_t dir_conn;
+
+  /* Set up the fake connection */
+  memset(&dir_conn, 0, sizeof(dir_connection_t));
+  dir_conn.base_.type = CONN_TYPE_DIR;
+  /* Apparently tor sets this before directory_handle_command() is called. */
+  dir_conn.base_.address = tor_strdup("replace-this-address.example.com");
+
+  dir_conn.base_.inbuf = buf_new_with_data((char*)stdin_buf, data_size);
+  if (!dir_conn.base_.inbuf) {
+    log_debug(LD_GENERAL, "Zero-Length-Input\n");
+    goto done;
+  }
+
+  /* Parse the headers */
+  int rv = directory_handle_command(&dir_conn);
+
+  /* TODO: check the output is correctly parsed based on the input */
+
+  /* Report the parsed origin address */
+  if (dir_conn.base_.address) {
+    log_debug(LD_GENERAL, "Address:\n%s\n", dir_conn.base_.address);
+  }
+
+  log_debug(LD_GENERAL, "Result:\n%d\n", rv);
+
+ done:
+  /* Reset. */
+  tor_free(dir_conn.base_.address);
+  buf_free(dir_conn.base_.inbuf);
+  dir_conn.base_.inbuf = NULL;
+
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c
new file mode 100644
index 0000000000..4abde0c16d
--- /dev/null
+++ b/src/test/fuzz/fuzz_iptsv2.c
@@ -0,0 +1,46 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "rendcommon.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  rend_service_descriptor_t *desc =
+    tor_malloc_zero(sizeof(rend_service_descriptor_t));
+  const char *str = (const char*) data;
+  int r = rend_parse_introduction_points(desc, str, sz);
+  if (r >= 0) {
+    log_debug(LD_GENERAL, "Parsing okay: %d", r);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  rend_service_descriptor_free(desc);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c
new file mode 100644
index 0000000000..396115026e
--- /dev/null
+++ b/src/test/fuzz/fuzz_microdesc.c
@@ -0,0 +1,47 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "microdesc.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  const char *str = (const char*) data;
+  smartlist_t *result = microdescs_parse_from_string((const char *)str,
+                                                     str+sz,
+                                                     0, SAVED_NOWHERE, NULL);
+  if (result) {
+    log_debug(LD_GENERAL, "Parsing okay: %d", smartlist_len(result));
+    SMARTLIST_FOREACH(result, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(result);
+  } else {
+    log_debug(LD_GENERAL, "Parsing failed");
+  }
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
new file mode 100755
index 0000000000..b4a17ed8cb
--- /dev/null
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -0,0 +1,34 @@
+MEMLIMIT_BYTES=21990500990976
+
+N_CPUS=1
+if [ $# -ge 1 ]; then
+    N_CPUS="$1"
+    shift
+fi
+
+FILTER=echo
+
+for i in `seq -w "$N_CPUS"`; do
+    if [ "$i" -eq 1 ]; then
+        if [ "$N_CPUS" -eq 1 ]; then
+            INSTANCE=""
+            NUMBER=""
+        else
+            INSTANCE="-M"
+            NUMBER="$i"
+        fi
+    else
+        INSTANCE="-S"
+        NUMBER="$i"
+    fi
+    # use whatever remains on the command-line to prefix the fuzzer command
+    # you have to copy and paste and run these commands yourself
+    "$FILTER" "$@" \
+        ../afl/afl-fuzz \
+        -i src/test/fuzz/fuzz_dir_testcase \
+        -o src/test/fuzz/fuzz_dir_findings \
+        -x src/test/fuzz/fuzz_dir_dictionary/fuzz_dir_http_header.dct \
+        -m "$MEMLIMIT_BYTES" \
+        "$INSTANCE" "$NUMBER" \
+        -- src/test/fuzz_dir
+done
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
new file mode 100644
index 0000000000..baf0610a0b
--- /dev/null
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -0,0 +1,82 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define ROUTERPARSE_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#include "or.h"
+#include "routerparse.h"
+#include "memarea.h"
+#include "microdesc.h"
+#include "networkstatus.h"
+#include "fuzzing.h"
+
+static void
+mock_dump_desc__nodump(const char *desc, const char *type)
+{
+  (void)desc;
+  (void)type;
+}
+
+static networkstatus_t *dummy_vote = NULL;
+static memarea_t *area = NULL;
+
+int
+fuzz_init(void)
+{
+  disable_signature_checking();
+  MOCK(dump_desc, mock_dump_desc__nodump);
+  ed25519_init();
+  area = memarea_new();
+  dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
+  dummy_vote->known_flags = smartlist_new();
+  smartlist_split_string(dummy_vote->known_flags,
+                         "Authority BadExit Exit Fast Guard HSDir "
+                         "NoEdConsensus Running Stable V2Dir Valid",
+                         " ", 0, 0);
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  SMARTLIST_FOREACH(dummy_vote->known_flags, char *, cp, tor_free(cp));
+  smartlist_free(dummy_vote->known_flags);
+  tor_free(dummy_vote);
+  return 0;
+}
+
+int
+fuzz_main(const uint8_t *data, size_t sz)
+{
+  char *str = tor_memdup_nulterm(data, sz);
+  const char *s;
+  routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
+  vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
+  smartlist_t *tokens = smartlist_new();
+
+  s = str;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                               NULL, NULL, 26, FLAV_NS);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  s = str;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                               NULL, NULL, 26, FLAV_MICRODESC);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  s = str;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+                                              dummy_vote, vrs, 26, FLAV_NS);
+  tor_assert(smartlist_len(tokens) == 0);
+
+  log_debug(LD_GENERAL,
+            "ns=%p, md=%p, vote=%p", rs_ns, rs_md, rs_vote);
+
+  routerstatus_free(rs_md);
+  routerstatus_free(rs_ns);
+  vote_routerstatus_free(vrs);
+  memarea_clear(area);
+  smartlist_free(tokens);
+  tor_free(str);
+  return 0;
+}
+
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
new file mode 100644
index 0000000000..aecdbb4e52
--- /dev/null
+++ b/src/test/fuzz/fuzzing.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#ifndef FUZZING_H
+#define FUZZING_H
+
+int fuzz_init(void);
+int fuzz_cleanup(void);
+int fuzz_main(const uint8_t *data, size_t sz);
+
+void disable_signature_checking(void);
+
+#endif /* FUZZING_H */
+
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
new file mode 100644
index 0000000000..7aee92df63
--- /dev/null
+++ b/src/test/fuzz/fuzzing_common.c
@@ -0,0 +1,191 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+#define CRYPTO_ED25519_PRIVATE
+#include "orconfig.h"
+#include "or.h"
+#include "backtrace.h"
+#include "config.h"
+#include "fuzzing.h"
+#include "crypto.h"
+#include "crypto_ed25519.h"
+
+extern const char tor_git_revision[];
+const char tor_git_revision[] = "";
+
+static or_options_t *mock_options = NULL;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mock_options;
+}
+
+static int
+mock_crypto_pk_public_checksig__nocheck(const crypto_pk_t *env, char *to,
+                                        size_t tolen,
+                                        const char *from, size_t fromlen)
+{
+  tor_assert(env && to && from);
+  (void)fromlen;
+  /* We could look at from[0..fromlen-1] ... */
+  tor_assert(tolen >= crypto_pk_keysize(env));
+  memset(to, 0x01, 20);
+  return 20;
+}
+
+static int
+mock_crypto_pk_public_checksig_digest__nocheck(crypto_pk_t *env,
+                                               const char *data,
+                                               size_t datalen,
+                                               const char *sig,
+                                               size_t siglen)
+{
+  tor_assert(env && data && sig);
+  (void)datalen;
+  (void)siglen;
+  /* We could look at data[..] and sig[..] */
+  return 0;
+}
+
+static int
+mock_ed25519_checksig__nocheck(const ed25519_signature_t *signature,
+                      const uint8_t *msg, size_t len,
+                      const ed25519_public_key_t *pubkey)
+{
+  tor_assert(signature && msg && pubkey);
+  /* We could look at msg[0..len-1] ... */
+  (void)len;
+  return 0;
+}
+
+static int
+mock_ed25519_checksig_batch__nocheck(int *okay_out,
+                                     const ed25519_checkable_t *checkable,
+                                     int n_checkable)
+{
+  tor_assert(checkable);
+  int i;
+  for (i = 0; i < n_checkable; ++i) {
+    /* We could look at messages and signatures XXX */
+    tor_assert(checkable[i].pubkey);
+    tor_assert(checkable[i].msg);
+    if (okay_out)
+      okay_out[i] = 1;
+  }
+  return 0;
+}
+
+static int
+mock_ed25519_impl_spot_check__nocheck(void)
+{
+  return 0;
+}
+
+void
+disable_signature_checking(void)
+{
+  MOCK(crypto_pk_public_checksig,
+       mock_crypto_pk_public_checksig__nocheck);
+  MOCK(crypto_pk_public_checksig_digest,
+       mock_crypto_pk_public_checksig_digest__nocheck);
+  MOCK(ed25519_checksig, mock_ed25519_checksig__nocheck);
+  MOCK(ed25519_checksig_batch, mock_ed25519_checksig_batch__nocheck);
+  MOCK(ed25519_impl_spot_check, mock_ed25519_impl_spot_check__nocheck);
+}
+
+static void
+global_init(void)
+{
+  tor_threads_init();
+  tor_compress_init();
+  {
+    struct sipkey sipkey = { 1337, 7331 };
+    siphash_set_global_key(&sipkey);
+  }
+
+  /* Initialise logging first */
+  init_logging(1);
+  configure_backtrace_handler(get_version());
+
+  /* set up the options. */
+  mock_options = tor_malloc(sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  /* Make BUG() and nonfatal asserts crash */
+  tor_set_failed_assertion_callback(abort);
+}
+
+#ifdef LLVM_FUZZ
+int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+int
+LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
+{
+  static int initialized = 0;
+  if (!initialized) {
+    global_init();
+    if (fuzz_init() < 0)
+      abort();
+    initialized = 1;
+  }
+
+  return fuzz_main(Data, Size);
+}
+
+#else /* Not LLVM_FUZZ, so AFL. */
+
+int
+main(int argc, char **argv)
+{
+  size_t size;
+
+  global_init();
+
+  /* Disable logging by default to speed up fuzzing. */
+  int loglevel = LOG_ERR;
+
+  for (int i = 1; i < argc; ++i) {
+    if (!strcmp(argv[i], "--warn")) {
+      loglevel = LOG_WARN;
+    } else if (!strcmp(argv[i], "--notice")) {
+      loglevel = LOG_NOTICE;
+    } else if (!strcmp(argv[i], "--info")) {
+      loglevel = LOG_INFO;
+    } else if (!strcmp(argv[i], "--debug")) {
+      loglevel = LOG_DEBUG;
+    }
+  }
+
+  {
+    log_severity_list_t s;
+    memset(&s, 0, sizeof(s));
+    set_log_severity_config(loglevel, LOG_ERR, &s);
+    /* ALWAYS log bug warnings. */
+    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    add_stream_log(&s, "", fileno(stdout));
+  }
+
+  if (fuzz_init() < 0)
+    abort();
+
+#ifdef __AFL_HAVE_MANUAL_CONTROL
+  /* Tell AFL to pause and fork here - ignored if not using AFL */
+  __AFL_INIT();
+#endif
+
+#define MAX_FUZZ_SIZE (128*1024)
+  char *input = read_file_to_str_until_eof(0, MAX_FUZZ_SIZE, &size);
+  tor_assert(input);
+  char *raw = tor_memdup(input, size); /* Because input is nul-terminated */
+  tor_free(input);
+  fuzz_main((const uint8_t*)raw, size);
+  tor_free(raw);
+
+  if (fuzz_cleanup() < 0)
+    abort();
+
+  tor_free(mock_options);
+  UNMOCK(get_options);
+  return 0;
+}
+
+#endif
+
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
new file mode 100644
index 0000000000..2961dab56f
--- /dev/null
+++ b/src/test/fuzz/include.am
@@ -0,0 +1,305 @@
+# This file was generated by fuzzing_include_am.py; do not hand-edit unless
+# you enjoy having your changes erased.
+FUZZING_CPPFLAGS = \
+	$(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
+FUZZING_CFLAGS = \
+	$(AM_CFLAGS) $(TEST_CFLAGS)
+FUZZING_LDFLAG = \
+	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+FUZZING_LIBS = \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LIBEVENT_LIBS@ \
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
+	@TOR_SYSTEMD_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	@TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
+
+oss-fuzz-prereqs: \
+	src/or/libtor-testing.a \
+	src/common/libor-crypto-testing.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	src/common/libor-testing.a \
+	src/common/libor-ctime-testing.a \
+	src/common/libor-event-testing.a \
+	src/trunnel/libor-trunnel-testing.a
+
+noinst_HEADERS += \
+	src/test/fuzz/fuzzing.h
+
+LIBFUZZER = -lFuzzer
+LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS)
+LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG)
+LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++
+
+LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ
+LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS)
+
+# ===== AFL fuzzers
+src_test_fuzz_fuzz_consensus_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_consensus.c
+src_test_fuzz_fuzz_consensus_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_consensus_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_consensus_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_consensus_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_descriptor_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_descriptor.c
+src_test_fuzz_fuzz_descriptor_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_descriptor_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_descriptor_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_descriptor_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_diff_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_diff.c
+src_test_fuzz_fuzz_diff_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_diff_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_diff_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_diff_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_diff_apply_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_diff_apply.c
+src_test_fuzz_fuzz_diff_apply_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_diff_apply_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_diff_apply_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_diff_apply_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_extrainfo_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_extrainfo.c
+src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_hsdescv2_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_hsdescv2.c
+src_test_fuzz_fuzz_hsdescv2_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_hsdescv2_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_hsdescv2_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_hsdescv2_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_http_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_http.c
+src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_iptsv2_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_iptsv2.c
+src_test_fuzz_fuzz_iptsv2_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_iptsv2_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_iptsv2_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_iptsv2_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_microdesc_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_microdesc.c
+src_test_fuzz_fuzz_microdesc_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_microdesc_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_microdesc_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_microdesc_LDADD = $(FUZZING_LIBS)
+
+src_test_fuzz_fuzz_vrs_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_vrs.c
+src_test_fuzz_fuzz_vrs_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_vrs_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_vrs_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_vrs_LDADD = $(FUZZING_LIBS)
+
+FUZZERS = \
+	src/test/fuzz/fuzz-consensus \
+	src/test/fuzz/fuzz-descriptor \
+	src/test/fuzz/fuzz-diff \
+	src/test/fuzz/fuzz-diff-apply \
+	src/test/fuzz/fuzz-extrainfo \
+	src/test/fuzz/fuzz-hsdescv2 \
+	src/test/fuzz/fuzz-http \
+	src/test/fuzz/fuzz-iptsv2 \
+	src/test/fuzz/fuzz-microdesc \
+	src/test/fuzz/fuzz-vrs
+
+# ===== libfuzzer
+
+if LIBFUZZER_ENABLED
+src_test_fuzz_lf_fuzz_consensus_SOURCES = \
+	$(src_test_fuzz_fuzz_consensus_SOURCES)
+src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_consensus_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_consensus_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_consensus_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_descriptor_SOURCES = \
+	$(src_test_fuzz_fuzz_descriptor_SOURCES)
+src_test_fuzz_lf_fuzz_descriptor_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_descriptor_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_descriptor_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_descriptor_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_diff_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_SOURCES)
+src_test_fuzz_lf_fuzz_diff_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_diff_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_diff_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_diff_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_diff_apply_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_apply_SOURCES)
+src_test_fuzz_lf_fuzz_diff_apply_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_diff_apply_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_diff_apply_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_diff_apply_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_extrainfo_SOURCES = \
+	$(src_test_fuzz_fuzz_extrainfo_SOURCES)
+src_test_fuzz_lf_fuzz_extrainfo_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_extrainfo_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_extrainfo_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_extrainfo_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_hsdescv2_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv2_SOURCES)
+src_test_fuzz_lf_fuzz_hsdescv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv2_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_hsdescv2_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_hsdescv2_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_http_SOURCES = \
+	$(src_test_fuzz_fuzz_http_SOURCES)
+src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \
+	$(src_test_fuzz_fuzz_iptsv2_SOURCES)
+src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_iptsv2_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_iptsv2_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_iptsv2_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_microdesc_SOURCES = \
+	$(src_test_fuzz_fuzz_microdesc_SOURCES)
+src_test_fuzz_lf_fuzz_microdesc_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_microdesc_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_microdesc_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_microdesc_LDADD = $(LIBFUZZER_LIBS)
+
+src_test_fuzz_lf_fuzz_vrs_SOURCES = \
+	$(src_test_fuzz_fuzz_vrs_SOURCES)
+src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_vrs_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_vrs_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS)
+
+LIBFUZZER_FUZZERS = \
+	src/test/fuzz/lf-fuzz-consensus \
+	src/test/fuzz/lf-fuzz-descriptor \
+	src/test/fuzz/lf-fuzz-diff \
+	src/test/fuzz/lf-fuzz-diff-apply \
+	src/test/fuzz/lf-fuzz-extrainfo \
+	src/test/fuzz/lf-fuzz-hsdescv2 \
+	src/test/fuzz/lf-fuzz-http \
+	src/test/fuzz/lf-fuzz-iptsv2 \
+	src/test/fuzz/lf-fuzz-microdesc \
+	src/test/fuzz/lf-fuzz-vrs
+
+else
+LIBFUZZER_FUZZERS =
+endif
+
+# ===== oss-fuzz
+
+if OSS_FUZZ_ENABLED
+src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \
+	$(src_test_fuzz_fuzz_consensus_SOURCES)
+src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_consensus_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_descriptor_a_SOURCES = \
+	$(src_test_fuzz_fuzz_descriptor_SOURCES)
+src_test_fuzz_liboss_fuzz_descriptor_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_descriptor_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_diff_a_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_SOURCES)
+src_test_fuzz_liboss_fuzz_diff_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_diff_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_diff_apply_a_SOURCES = \
+	$(src_test_fuzz_fuzz_diff_apply_SOURCES)
+src_test_fuzz_liboss_fuzz_diff_apply_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_diff_apply_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_extrainfo_a_SOURCES = \
+	$(src_test_fuzz_fuzz_extrainfo_SOURCES)
+src_test_fuzz_liboss_fuzz_extrainfo_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_extrainfo_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_hsdescv2_a_SOURCES = \
+	$(src_test_fuzz_fuzz_hsdescv2_SOURCES)
+src_test_fuzz_liboss_fuzz_hsdescv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_hsdescv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_http_a_SOURCES = \
+	$(src_test_fuzz_fuzz_http_SOURCES)
+src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \
+	$(src_test_fuzz_fuzz_iptsv2_SOURCES)
+src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_iptsv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_microdesc_a_SOURCES = \
+	$(src_test_fuzz_fuzz_microdesc_SOURCES)
+src_test_fuzz_liboss_fuzz_microdesc_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_microdesc_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
+	$(src_test_fuzz_fuzz_vrs_SOURCES)
+src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+
+OSS_FUZZ_FUZZERS = \
+	src/test/fuzz/liboss-fuzz-consensus.a \
+	src/test/fuzz/liboss-fuzz-descriptor.a \
+	src/test/fuzz/liboss-fuzz-diff.a \
+	src/test/fuzz/liboss-fuzz-diff-apply.a \
+	src/test/fuzz/liboss-fuzz-extrainfo.a \
+	src/test/fuzz/liboss-fuzz-hsdescv2.a \
+	src/test/fuzz/liboss-fuzz-http.a \
+	src/test/fuzz/liboss-fuzz-iptsv2.a \
+	src/test/fuzz/liboss-fuzz-microdesc.a \
+	src/test/fuzz/liboss-fuzz-vrs.a
+
+else
+OSS_FUZZ_FUZZERS =
+endif
+
+noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS)
+noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS)
+oss-fuzz-fuzzers:  oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS)
+fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS)
+
+test-fuzz-corpora: $(FUZZERS)
+	$(top_srcdir)/src/test/fuzz_static_testcases.sh
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
new file mode 100755
index 0000000000..87d3dda13c
--- /dev/null
+++ b/src/test/fuzz/minimize.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+if [ ! -d "$1" ] ; then
+    echo "I need a directory"
+    exit 1
+fi
+
+which=`basename "$1"`
+
+mkdir "$1.out"
+afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
+
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
new file mode 100755
index 0000000000..3cb45ad5e6
--- /dev/null
+++ b/src/test/fuzz_static_testcases.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Copyright (c) 2016-2017, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+set -e
+
+if [ -z "${TOR_FUZZ_CORPORA}" ] || [ ! -d "${TOR_FUZZ_CORPORA}" ] ; then
+    echo "You need to set TOR_FUZZ_CORPORA to point to a checkout of "
+    echo "the 'fuzzing-corpora' repository."
+    exit 77
+fi
+
+
+
+for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
+    f=`basename $fuzzer`
+    case="${f#fuzz-}"
+    if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
+        echo "Running tests for ${case}"
+        for entry in "${TOR_FUZZ_CORPORA}/${case}/"*; do
+	    "${fuzzer}" "--err" < "$entry"
+        done
+    else
+	echo "No tests found for ${case}"
+    fi
+done
diff --git a/src/test/hs_ntor_ref.py b/src/test/hs_ntor_ref.py
new file mode 100644
index 0000000000..2ed9324e1f
--- /dev/null
+++ b/src/test/hs_ntor_ref.py
@@ -0,0 +1,425 @@
+#!/usr/bin/python
+# Copyright 2017, The Tor Project, Inc
+# See LICENSE for licensing information
+
+"""
+hs_ntor_ref.py
+
+This module is a reference implementation of the modified ntor protocol
+proposed for Tor hidden services in proposal 224 (Next Generation Hidden
+Services) in section [NTOR-WITH-EXTRA-DATA].
+
+The modified ntor protocol is a single-round protocol, with three steps in total:
+
+    1: Client generates keys and sends them to service via INTRODUCE cell
+
+    2: Service computes key material based on client's keys, and sends its own
+       keys to client via RENDEZVOUS cell
+
+    3: Client computes key material as well.
+
+It's meant to be used to validate Tor's HS ntor implementation by conducting
+various integration tests. Specifically it conducts the following three tests:
+
+- Tests our Python implementation by running the whole protocol in Python and
+  making sure that results are consistent.
+
+- Tests little-t-tor ntor implementation. We use this Python code to instrument
+  little-t-tor and carry out the handshake by using little-t-tor code. The
+  small C wrapper at src/test/test-hs-ntor-cl is used for this Python module to
+  interface with little-t-tor.
+
+- Cross-tests Python and little-t-tor implementation by running half of the
+  protocol in Python code and the other in little-t-tor. This is actually two
+  tests so that all parts of the protocol are run both by little-t-tor and
+  Python.
+
+It requires the curve25519 python module from the curve25519-donna package.
+
+The whole logic and concept for this test suite was taken from ntor_ref.py.
+
+                *** DO NOT USE THIS IN PRODUCTION. ***
+"""
+
+import struct
+import os, sys
+import binascii
+import subprocess
+
+try:
+    import curve25519
+    curve25519mod = curve25519.keys
+except ImportError:
+    curve25519 = None
+    import slownacl_curve25519
+    curve25519mod = slownacl_curve25519
+
+import hashlib
+try:
+    import sha3
+except ImportError:
+    # In python 3.6, the sha3 functions are in hashlib whether we
+    # import sha3 or not.
+    sha3 = None
+
+try:
+    # Pull the sha3 functions in.
+    from hashlib import sha3_256, shake_256
+    shake_squeeze = shake_256.digest
+except ImportError:
+    if hasattr(sha3, "SHA3256"):
+        # If this happens, then we have the old "sha3" module which
+        # hashlib and pysha3 superseded.
+        sha3_256 = sha3.SHA3256
+        shake_256 = sha3.SHAKE256
+        shake_squeeze = shake_256.squeeze
+    else:
+        # error code 77 tells automake to skip this test
+        sys.exit(77)
+
+# Import Nick's ntor reference implementation in Python
+# We are gonna use a few of its utilities.
+from ntor_ref import hash_nil
+from ntor_ref import PrivateKey
+
+# String constants used in this protocol
+PROTOID = b"tor-hs-ntor-curve25519-sha3-256-1"
+T_HSENC    = PROTOID + b":hs_key_extract"
+T_HSVERIFY = PROTOID + b":hs_verify"
+T_HSMAC    = PROTOID + b":hs_mac"
+M_HSEXPAND = PROTOID + b":hs_key_expand"
+
+INTRO_SECRET_LEN = 161
+REND_SECRET_LEN = 225
+AUTH_INPUT_LEN = 199
+
+# Implements MAC(k,m) = H(htonll(len(k)) | k | m)
+def mac(k,m):
+    def htonll(num):
+        return struct.pack('!q', num)
+
+    s = sha3_256()
+    s.update(htonll(len(k)))
+    s.update(k)
+    s.update(m)
+    return s.digest()
+
+######################################################################
+
+# Functions that implement the modified HS ntor protocol
+
+"""As client compute key material for INTRODUCE cell as follows:
+
+  intro_secret_hs_input = EXP(B,x) | AUTH_KEY | X | B | PROTOID
+  info = m_hsexpand | subcredential
+  hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+  ENC_KEY = hs_keys[0:S_KEY_LEN]
+  MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+"""
+def intro2_ntor_client(intro_auth_pubkey_str, intro_enc_pubkey,
+                       client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential):
+
+    dh_result = client_ephemeral_enc_privkey.get_shared_key(intro_enc_pubkey, hash_nil)
+    secret =  dh_result + intro_auth_pubkey_str + client_ephemeral_enc_pubkey.serialize() + intro_enc_pubkey.serialize() + PROTOID
+    assert(len(secret) == INTRO_SECRET_LEN)
+    info = M_HSEXPAND + subcredential
+
+    kdf = shake_256()
+    kdf.update(secret + T_HSENC + info)
+    key_material = shake_squeeze(kdf, 64*8)
+
+    enc_key = key_material[0:32]
+    mac_key = key_material[32:64]
+
+    return enc_key, mac_key
+
+"""Wrapper over intro2_ntor_client()"""
+def client_part1(intro_auth_pubkey_str, intro_enc_pubkey,
+                 client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential):
+    enc_key, mac_key = intro2_ntor_client(intro_auth_pubkey_str, intro_enc_pubkey, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential)
+    assert(enc_key)
+    assert(mac_key)
+
+    return enc_key, mac_key
+
+"""As service compute key material for INTRODUCE cell as follows:
+
+  intro_secret_hs_input = EXP(X,b) | AUTH_KEY | X | B | PROTOID
+  info = m_hsexpand | subcredential
+  hs_keys = KDF(intro_secret_hs_input | t_hsenc | info, S_KEY_LEN+MAC_LEN)
+  HS_DEC_KEY = hs_keys[0:S_KEY_LEN]
+  HS_MAC_KEY = hs_keys[S_KEY_LEN:S_KEY_LEN+MAC_KEY_LEN]
+"""
+def intro2_ntor_service(intro_auth_pubkey_str, client_enc_pubkey, service_enc_privkey, service_enc_pubkey, subcredential):
+    dh_result = service_enc_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    secret = dh_result + intro_auth_pubkey_str + client_enc_pubkey.serialize() + service_enc_pubkey.serialize() + PROTOID
+    assert(len(secret) == INTRO_SECRET_LEN)
+    info = M_HSEXPAND + subcredential
+
+    kdf = shake_256()
+    kdf.update(secret + T_HSENC + info)
+    key_material = shake_squeeze(kdf, 64*8)
+
+    enc_key = key_material[0:32]
+    mac_key = key_material[32:64]
+
+    return enc_key, mac_key
+
+"""As service compute key material for INTRODUCE and REDNEZVOUS cells.
+
+  Use intro2_ntor_service() to calculate the INTRODUCE key material, and use
+  the following computations to do the RENDEZVOUS ones:
+
+      rend_secret_hs_input = EXP(X,y) | EXP(X,b) | AUTH_KEY | B | X | Y | PROTOID
+      NTOR_KEY_SEED = MAC(rend_secret_hs_input, t_hsenc)
+      verify = MAC(rend_secret_hs_input, t_hsverify)
+      auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+      AUTH_INPUT_MAC = MAC(auth_input, t_hsmac)
+"""
+def service_part1(intro_auth_pubkey_str, client_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential):
+    intro_enc_key, intro_mac_key = intro2_ntor_service(intro_auth_pubkey_str, client_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+    assert(intro_enc_key)
+    assert(intro_mac_key)
+
+    service_ephemeral_privkey = PrivateKey()
+    service_ephemeral_pubkey = service_ephemeral_privkey.get_public()
+
+    dh_result1 = service_ephemeral_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    dh_result2 = intro_enc_privkey.get_shared_key(client_enc_pubkey, hash_nil)
+    rend_secret_hs_input = dh_result1 + dh_result2 + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + client_enc_pubkey.serialize() + service_ephemeral_pubkey.serialize() + PROTOID
+    assert(len(rend_secret_hs_input) == REND_SECRET_LEN)
+
+    ntor_key_seed = mac(rend_secret_hs_input, T_HSENC)
+    verify = mac(rend_secret_hs_input, T_HSVERIFY)
+    auth_input = verify + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + service_ephemeral_pubkey.serialize() + client_enc_pubkey.serialize() + PROTOID + b"Server"
+    assert(len(auth_input) == AUTH_INPUT_LEN)
+    auth_input_mac = mac(auth_input, T_HSMAC)
+
+    assert(ntor_key_seed)
+    assert(auth_input_mac)
+    assert(service_ephemeral_pubkey)
+
+    return intro_enc_key, intro_mac_key, ntor_key_seed, auth_input_mac, service_ephemeral_pubkey
+
+"""As client compute key material for rendezvous cells as follows:
+
+  rend_secret_hs_input = EXP(Y,x) | EXP(B,x) | AUTH_KEY | B | X | Y | PROTOID
+  NTOR_KEY_SEED = MAC(ntor_secret_input, t_hsenc)
+  verify = MAC(ntor_secret_input, t_hsverify)
+  auth_input = verify | AUTH_KEY | B | Y | X | PROTOID | "Server"
+  AUTH_INPUT_MAC = MAC(auth_input, t_hsmac)
+"""
+def client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                 intro_enc_pubkey, service_ephemeral_rend_pubkey):
+    dh_result1 = client_ephemeral_enc_privkey.get_shared_key(service_ephemeral_rend_pubkey, hash_nil)
+    dh_result2 = client_ephemeral_enc_privkey.get_shared_key(intro_enc_pubkey, hash_nil)
+    rend_secret_hs_input = dh_result1 + dh_result2 + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + client_ephemeral_enc_pubkey.serialize() + service_ephemeral_rend_pubkey.serialize() + PROTOID
+    assert(len(rend_secret_hs_input) == REND_SECRET_LEN)
+
+    ntor_key_seed = mac(rend_secret_hs_input, T_HSENC)
+    verify = mac(rend_secret_hs_input, T_HSVERIFY)
+    auth_input = verify + intro_auth_pubkey_str + intro_enc_pubkey.serialize() + service_ephemeral_rend_pubkey.serialize() + client_ephemeral_enc_pubkey.serialize() + PROTOID + b"Server"
+    assert(len(auth_input) == AUTH_INPUT_LEN)
+    auth_input_mac = mac(auth_input, T_HSMAC)
+
+    assert(ntor_key_seed)
+    assert(auth_input_mac)
+
+    return ntor_key_seed, auth_input_mac
+
+#################################################################################
+
+"""
+Utilities for communicating with the little-t-tor ntor wrapper to conduct the
+integration tests
+"""
+
+PROG = b"./src/test/test-hs-ntor-cl"
+enhex=lambda s: binascii.b2a_hex(s)
+dehex=lambda s: binascii.a2b_hex(s.strip())
+
+def tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                client_ephemeral_enc_privkey, subcredential):
+    p = subprocess.Popen([PROG, "client1",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(intro_enc_pubkey.serialize()),
+                          enhex(client_ephemeral_enc_privkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+def tor_server1(intro_auth_pubkey_str, intro_enc_privkey,
+                client_ephemeral_enc_pubkey, subcredential):
+    p = subprocess.Popen([PROG, "server1",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(intro_enc_privkey.serialize()),
+                          enhex(client_ephemeral_enc_pubkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+def tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                intro_enc_pubkey, service_ephemeral_rend_pubkey, subcredential):
+    p = subprocess.Popen([PROG, "client2",
+                          enhex(intro_auth_pubkey_str),
+                          enhex(client_ephemeral_enc_privkey.serialize()),
+                          enhex(intro_enc_pubkey.serialize()),
+                          enhex(service_ephemeral_rend_pubkey.serialize()),
+                          enhex(subcredential)],
+                         stdout=subprocess.PIPE)
+    return map(dehex, p.stdout.readlines())
+
+##################################################################################
+
+# Perform a pure python ntor test
+def do_pure_python_ntor_test():
+    # Initialize all needed key material
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public()
+    intro_auth_pubkey_str = os.urandom(32)
+    subcredential = os.urandom(32)
+
+    client_enc_key, client_mac_key = client_part1(intro_auth_pubkey_str, intro_enc_pubkey, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey, subcredential)
+
+    service_enc_key, service_mac_key, service_ntor_key_seed, service_auth_input_mac, service_ephemeral_pubkey = service_part1(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    client_ntor_key_seed, client_auth_input_mac = client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                               intro_enc_pubkey, service_ephemeral_pubkey)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_auth_input_mac == service_auth_input_mac)
+
+    print("DONE: python dance [%s]" % repr(client_auth_input_mac))
+
+# Perform a pure little-t-tor integration test.
+def do_little_t_tor_ntor_test():
+    # Initialize all needed key material
+    subcredential = os.urandom(32)
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+    intro_auth_pubkey_str = os.urandom(32)
+
+    client_enc_key, client_mac_key = tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                 client_ephemeral_enc_privkey, subcredential)
+    assert(client_enc_key)
+    assert(client_mac_key)
+
+    service_enc_key, service_mac_key, service_ntor_auth_mac, service_ntor_key_seed, service_eph_pubkey = tor_server1(intro_auth_pubkey_str,
+                                                                                                                     intro_enc_privkey,
+                                                                                                                     client_ephemeral_enc_pubkey,
+                                                                                                                     subcredential)
+    assert(service_enc_key)
+    assert(service_mac_key)
+    assert(service_ntor_auth_mac)
+    assert(service_ntor_key_seed)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    # Turn from bytes to key
+    service_eph_pubkey = curve25519mod.Public(service_eph_pubkey)
+
+    client_ntor_auth_mac, client_ntor_key_seed  = tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                                                              intro_enc_pubkey, service_eph_pubkey, subcredential)
+    assert(client_ntor_auth_mac)
+    assert(client_ntor_key_seed)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_ntor_auth_mac == service_ntor_auth_mac)
+
+    print("DONE: tor dance [%s]" % repr(client_ntor_auth_mac))
+
+"""
+Do mixed test as follows:
+    1. C -> S (python mode)
+    2. C <- S (tor mode)
+    3. Client computes keys (python mode)
+"""
+def do_first_mixed_test():
+    subcredential = os.urandom(32)
+
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+
+    intro_auth_pubkey_str = os.urandom(32)
+
+    # Let's do mixed
+    client_enc_key, client_mac_key = client_part1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                  client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                  subcredential)
+
+    service_enc_key, service_mac_key, service_ntor_auth_mac, service_ntor_key_seed, service_eph_pubkey = tor_server1(intro_auth_pubkey_str,
+                                                                                                                     intro_enc_privkey,
+                                                                                                                     client_ephemeral_enc_pubkey,
+                                                                                                                     subcredential)
+    assert(service_enc_key)
+    assert(service_mac_key)
+    assert(service_ntor_auth_mac)
+    assert(service_ntor_key_seed)
+    assert(service_eph_pubkey)
+
+    assert(client_enc_key == service_enc_key)
+    assert(client_mac_key == service_mac_key)
+
+    # Turn from bytes to key
+    service_eph_pubkey = curve25519mod.Public(service_eph_pubkey)
+
+    client_ntor_key_seed, client_auth_input_mac = client_part2(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, client_ephemeral_enc_privkey,
+                                                               intro_enc_pubkey, service_eph_pubkey)
+
+    assert(client_auth_input_mac == service_ntor_auth_mac)
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+
+    print("DONE: 1st mixed dance [%s]" % repr(client_auth_input_mac))
+
+"""
+Do mixed test as follows:
+    1. C -> S (tor mode)
+    2. C <- S (python mode)
+    3. Client computes keys (tor mode)
+"""
+def do_second_mixed_test():
+    subcredential = os.urandom(32)
+
+    client_ephemeral_enc_privkey = PrivateKey()
+    client_ephemeral_enc_pubkey = client_ephemeral_enc_privkey.get_public()
+    intro_enc_privkey = PrivateKey()
+    intro_enc_pubkey = intro_enc_privkey.get_public() # service-side enc key
+
+    intro_auth_pubkey_str = os.urandom(32)
+
+    # Let's do mixed
+    client_enc_key, client_mac_key = tor_client1(intro_auth_pubkey_str, intro_enc_pubkey,
+                                                 client_ephemeral_enc_privkey, subcredential)
+    assert(client_enc_key)
+    assert(client_mac_key)
+
+    service_enc_key, service_mac_key, service_ntor_key_seed, service_ntor_auth_mac, service_ephemeral_pubkey = service_part1(intro_auth_pubkey_str, client_ephemeral_enc_pubkey, intro_enc_privkey, intro_enc_pubkey, subcredential)
+
+    client_ntor_auth_mac, client_ntor_key_seed  = tor_client2(intro_auth_pubkey_str, client_ephemeral_enc_privkey,
+                                                              intro_enc_pubkey, service_ephemeral_pubkey, subcredential)
+    assert(client_ntor_auth_mac)
+    assert(client_ntor_key_seed)
+
+    assert(client_ntor_key_seed == service_ntor_key_seed)
+    assert(client_ntor_auth_mac == service_ntor_auth_mac)
+
+    print("DONE: 2nd mixed dance [%s]" % repr(client_ntor_auth_mac))
+
+def do_mixed_tests():
+    do_first_mixed_test()
+    do_second_mixed_test()
+
+if __name__ == '__main__':
+    do_pure_python_ntor_test()
+    do_little_t_tor_ntor_test()
+    do_mixed_tests()
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
new file mode 100644
index 0000000000..dcd58bc5fd
--- /dev/null
+++ b/src/test/hs_test_helpers.c
@@ -0,0 +1,263 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "crypto_ed25519.h"
+#include "test.h"
+#include "torcert.h"
+
+#include "hs_test_helpers.h"
+
+hs_desc_intro_point_t *
+hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
+                            const char *addr, int legacy)
+{
+  int ret;
+  ed25519_keypair_t auth_kp;
+  hs_desc_intro_point_t *intro_point = NULL;
+  hs_desc_intro_point_t *ip = tor_malloc_zero(sizeof(*ip));
+  ip->link_specifiers = smartlist_new();
+
+  {
+    hs_desc_link_specifier_t *ls = tor_malloc_zero(sizeof(*ls));
+    if (legacy) {
+      ls->type = LS_LEGACY_ID;
+      memcpy(ls->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
+             DIGEST_LEN);
+    } else {
+      ls->u.ap.port = 9001;
+      int family = tor_addr_parse(&ls->u.ap.addr, addr);
+      switch (family) {
+        case AF_INET:
+          ls->type = LS_IPV4;
+          break;
+        case AF_INET6:
+          ls->type = LS_IPV6;
+          break;
+        default:
+          /* Stop the test, not suppose to have an error. */
+          tt_int_op(family, OP_EQ, AF_INET);
+      }
+    }
+    smartlist_add(ip->link_specifiers, ls);
+  }
+
+  ret = ed25519_keypair_generate(&auth_kp, 0);
+  tt_int_op(ret, ==, 0);
+  ip->auth_key_cert = tor_cert_create(signing_kp, CERT_TYPE_AUTH_HS_IP_KEY,
+                                      &auth_kp.pubkey, now,
+                                      HS_DESC_CERT_LIFETIME,
+                                      CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(ip->auth_key_cert);
+
+  if (legacy) {
+    ip->legacy.key = crypto_pk_new();
+    tt_assert(ip->legacy.key);
+    ret = crypto_pk_generate_key(ip->legacy.key);
+    tt_int_op(ret, ==, 0);
+    ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
+                                    &signing_kp->pubkey, ip->legacy.key,
+                                    now + HS_DESC_CERT_LIFETIME,
+                                    &ip->legacy.cert.encoded);
+    tt_assert(ip->legacy.cert.encoded);
+    tt_u64_op(cert_len, OP_GT, 0);
+    ip->legacy.cert.len = cert_len;
+  }
+
+  /* Encryption key. */
+  {
+    int signbit;
+    curve25519_keypair_t curve25519_kp;
+    ed25519_keypair_t ed25519_kp;
+    tor_cert_t *cross_cert;
+
+    ret = curve25519_keypair_generate(&curve25519_kp, 0);
+    tt_int_op(ret, ==, 0);
+    ed25519_keypair_from_curve25519_keypair(&ed25519_kp, &signbit,
+                                            &curve25519_kp);
+    cross_cert = tor_cert_create(signing_kp, CERT_TYPE_CROSS_HS_IP_KEYS,
+                                 &ed25519_kp.pubkey, time(NULL),
+                                 HS_DESC_CERT_LIFETIME,
+                                 CERT_FLAG_INCLUDE_SIGNING_KEY);
+    tt_assert(cross_cert);
+    ip->enc_key_cert = cross_cert;
+  }
+
+  intro_point = ip;
+ done:
+  if (intro_point == NULL)
+    tor_free(ip);
+
+  return intro_point;
+}
+
+/* Return a valid hs_descriptor_t object. If no_ip is set, no introduction
+ * points are added. */
+static hs_descriptor_t *
+hs_helper_build_hs_desc_impl(unsigned int no_ip,
+                             const ed25519_keypair_t *signing_kp)
+{
+  int ret;
+  time_t now = time(NULL);
+  ed25519_keypair_t blinded_kp;
+  hs_descriptor_t *descp = NULL, *desc = tor_malloc_zero(sizeof(*desc));
+
+  desc->plaintext_data.version = HS_DESC_SUPPORTED_FORMAT_VERSION_MAX;
+
+  /* Copy only the public key into the descriptor. */
+  memcpy(&desc->plaintext_data.signing_pubkey, &signing_kp->pubkey,
+         sizeof(ed25519_public_key_t));
+
+  ret = ed25519_keypair_generate(&blinded_kp, 0);
+  tt_int_op(ret, ==, 0);
+  /* Copy only the public key into the descriptor. */
+  memcpy(&desc->plaintext_data.blinded_pubkey, &blinded_kp.pubkey,
+         sizeof(ed25519_public_key_t));
+
+  desc->plaintext_data.signing_key_cert =
+    tor_cert_create(&blinded_kp, CERT_TYPE_SIGNING_HS_DESC,
+                    &signing_kp->pubkey, now, 3600,
+                    CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(desc->plaintext_data.signing_key_cert);
+  desc->plaintext_data.revision_counter = 42;
+  desc->plaintext_data.lifetime_sec = 3 * 60 * 60;
+
+  /* Setup encrypted data section. */
+  desc->encrypted_data.create2_ntor = 1;
+  desc->encrypted_data.intro_auth_types = smartlist_new();
+  desc->encrypted_data.single_onion_service = 1;
+  smartlist_add(desc->encrypted_data.intro_auth_types, tor_strdup("ed25519"));
+  desc->encrypted_data.intro_points = smartlist_new();
+  if (!no_ip) {
+    /* Add four intro points. */
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1));
+    smartlist_add(desc->encrypted_data.intro_points,
+              hs_helper_build_intro_point(signing_kp, now, "", 1));
+  }
+
+  descp = desc;
+ done:
+  if (descp == NULL)
+    tor_free(desc);
+
+  return descp;
+}
+
+/* Build a descriptor with introduction points. */
+hs_descriptor_t *
+hs_helper_build_hs_desc_with_ip(const ed25519_keypair_t *signing_kp)
+{
+  return hs_helper_build_hs_desc_impl(0, signing_kp);
+}
+
+/* Build a descriptor without any introduction points. */
+hs_descriptor_t *
+hs_helper_build_hs_desc_no_ip(const ed25519_keypair_t *signing_kp)
+{
+  return hs_helper_build_hs_desc_impl(1, signing_kp);
+}
+
+void
+hs_helper_desc_equal(const hs_descriptor_t *desc1,
+                     const hs_descriptor_t *desc2)
+{
+  char *addr1 = NULL, *addr2 = NULL;
+  /* Plaintext data section. */
+  tt_int_op(desc1->plaintext_data.version, OP_EQ,
+            desc2->plaintext_data.version);
+  tt_uint_op(desc1->plaintext_data.lifetime_sec, OP_EQ,
+             desc2->plaintext_data.lifetime_sec);
+  tt_assert(tor_cert_eq(desc1->plaintext_data.signing_key_cert,
+                        desc2->plaintext_data.signing_key_cert));
+  tt_mem_op(desc1->plaintext_data.signing_pubkey.pubkey, OP_EQ,
+            desc2->plaintext_data.signing_pubkey.pubkey,
+            ED25519_PUBKEY_LEN);
+  tt_mem_op(desc1->plaintext_data.blinded_pubkey.pubkey, OP_EQ,
+            desc2->plaintext_data.blinded_pubkey.pubkey,
+            ED25519_PUBKEY_LEN);
+  tt_u64_op(desc1->plaintext_data.revision_counter, ==,
+            desc2->plaintext_data.revision_counter);
+
+  /* NOTE: We can't compare the encrypted blob because when encoding the
+   * descriptor, the object is immutable thus we don't update it with the
+   * encrypted blob. As contrast to the decoding process where we populate a
+   * descriptor object. */
+
+  /* Encrypted data section. */
+  tt_uint_op(desc1->encrypted_data.create2_ntor, ==,
+             desc2->encrypted_data.create2_ntor);
+
+  /* Authentication type. */
+  tt_int_op(!!desc1->encrypted_data.intro_auth_types, ==,
+            !!desc2->encrypted_data.intro_auth_types);
+  if (desc1->encrypted_data.intro_auth_types &&
+      desc2->encrypted_data.intro_auth_types) {
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), ==,
+              smartlist_len(desc2->encrypted_data.intro_auth_types));
+    for (int i = 0;
+         i < smartlist_len(desc1->encrypted_data.intro_auth_types);
+         i++) {
+      tt_str_op(smartlist_get(desc1->encrypted_data.intro_auth_types, i),OP_EQ,
+                smartlist_get(desc2->encrypted_data.intro_auth_types, i));
+    }
+  }
+
+  /* Introduction points. */
+  {
+    tt_assert(desc1->encrypted_data.intro_points);
+    tt_assert(desc2->encrypted_data.intro_points);
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), ==,
+              smartlist_len(desc2->encrypted_data.intro_points));
+    for (int i=0; i < smartlist_len(desc1->encrypted_data.intro_points); i++) {
+      hs_desc_intro_point_t *ip1 = smartlist_get(desc1->encrypted_data
+                                                 .intro_points, i),
+                            *ip2 = smartlist_get(desc2->encrypted_data
+                                                 .intro_points, i);
+      tt_assert(tor_cert_eq(ip1->auth_key_cert, ip2->auth_key_cert));
+      if (ip1->legacy.key) {
+        tt_int_op(crypto_pk_cmp_keys(ip1->legacy.key, ip2->legacy.key),
+                  OP_EQ, 0);
+      } else {
+        tt_mem_op(&ip1->enc_key, OP_EQ, &ip2->enc_key, CURVE25519_PUBKEY_LEN);
+      }
+
+      tt_int_op(smartlist_len(ip1->link_specifiers), ==,
+                smartlist_len(ip2->link_specifiers));
+      for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
+        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(ls1->type, ==, ls2->type);
+        switch (ls1->type) {
+          case LS_IPV4:
+          case LS_IPV6:
+            {
+              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
+              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
+              tt_str_op(addr1, OP_EQ, addr2);
+              tor_free(addr1);
+              tor_free(addr2);
+              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+            }
+            break;
+          case LS_LEGACY_ID:
+            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
+                      sizeof(ls1->u.legacy_id));
+            break;
+          default:
+            /* Unknown type, caught it and print its value. */
+            tt_int_op(ls1->type, OP_EQ, -1);
+        }
+      }
+    }
+  }
+
+ done:
+  tor_free(addr1);
+  tor_free(addr2);
+}
+
diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h
new file mode 100644
index 0000000000..a7fedab136
--- /dev/null
+++ b/src/test/hs_test_helpers.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_HS_TEST_HELPERS_H
+#define TOR_HS_TEST_HELPERS_H
+
+#include "ed25519_cert.h"
+#include "hs_descriptor.h"
+
+/* Set of functions to help build and test descriptors. */
+hs_desc_intro_point_t *hs_helper_build_intro_point(
+                          const ed25519_keypair_t *signing_kp, time_t now,
+                          const char *addr, int legacy);
+hs_descriptor_t *hs_helper_build_hs_desc_no_ip(
+                                 const ed25519_keypair_t *signing_kp);
+hs_descriptor_t *hs_helper_build_hs_desc_with_ip(
+                                 const ed25519_keypair_t *signing_kp);
+void hs_helper_desc_equal(const hs_descriptor_t *desc1,
+                          const hs_descriptor_t *desc2);
+
+#endif /* TOR_HS_TEST_HELPERS_H */
+
diff --git a/src/test/include.am b/src/test/include.am
index 0ee3d1169f..723b4964e1 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -5,10 +5,15 @@ TESTS_ENVIRONMENT = \
 	export PYTHON="$(PYTHON)"; \
 	export SHELL="$(SHELL)"; \
 	export abs_top_srcdir="$(abs_top_srcdir)"; \
+	export abs_top_builddir="$(abs_top_builddir)"; \
 	export builddir="$(builddir)"; \
-	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)";
+	export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \
+	export CARGO="$(CARGO)"; \
+	export CARGO_ONLINE="$(CARGO_ONLINE)";
 
-TESTSCRIPTS = src/test/test_zero_length_keys.sh \
+TESTSCRIPTS = \
+	src/test/fuzz_static_testcases.sh \
+	src/test/test_zero_length_keys.sh \
 	src/test/test_workqueue_cancel.sh \
 	src/test/test_workqueue_efd.sh \
 	src/test/test_workqueue_efd2.sh \
@@ -17,8 +22,13 @@ TESTSCRIPTS = src/test/test_zero_length_keys.sh \
 	src/test/test_workqueue_socketpair.sh \
 	src/test/test_switch_id.sh
 
+if USE_RUST
+TESTSCRIPTS += \
+	src/test/test_rust.sh
+endif
+
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
 endif
 
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
@@ -67,6 +77,7 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
 
 src_test_test_SOURCES = \
 	src/test/log_test_helpers.c \
+	src/test/hs_test_helpers.c \
 	src/test/rend_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
@@ -77,17 +88,24 @@ src_test_test_SOURCES = \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
+	src/test/test_channelpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
 	src/test/test_circuitmux.c \
+	src/test/test_circuitbuild.c \
+	src/test/test_circuituse.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
 	src/test/test_connection.c \
+	src/test/test_conscache.c \
+	src/test/test_consdiff.c \
+	src/test/test_consdiffmgr.c \
 	src/test/test_containers.c \
 	src/test/test_controller.c \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
+	src/test/test_crypto_openssl.c \
 	src/test/test_dos.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
@@ -98,7 +116,11 @@ src_test_test_SOURCES = \
 	src/test/test_guardfraction.c \
 	src/test/test_extorport.c \
 	src/test/test_hs.c \
+	src/test/test_hs_service.c \
+	src/test/test_hs_intropoint.c \
 	src/test/test_handles.c \
+	src/test/test_hs_cache.c \
+	src/test/test_hs_descriptor.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
@@ -120,10 +142,12 @@ src_test_test_SOURCES = \
 	src/test/test_routerkeys.c \
 	src/test/test_routerlist.c \
 	src/test/test_routerset.c \
+	src/test/test_rust.c \
 	src/test/test_scheduler.c \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
 	src/test/test_status.c \
+	src/test/test_storagedir.c \
 	src/test/test_threads.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
@@ -132,6 +156,7 @@ src_test_test_SOURCES = \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
 	src/test/testing_common.c \
+	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
 
 src_test_test_slow_SOURCES = \
@@ -139,6 +164,7 @@ src_test_test_slow_SOURCES = \
 	src/test/test_crypto_slow.c \
 	src/test/test_util_slow.c \
 	src/test/testing_common.c \
+	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
 
 src_test_test_memwipe_SOURCES = \
@@ -167,7 +193,9 @@ src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@
 src_test_test_switch_id_LDADD = \
 	src/common/libor-testing.a \
 	src/common/libor-ctime-testing.a \
-	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
 
 src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
@@ -179,9 +207,11 @@ src_test_test_LDADD = src/or/libtor-testing.a \
 	src/common/libor-ctime-testing.a \
 	src/common/libor-event-testing.a \
 	src/trunnel/libor-trunnel-testing.a \
+	src/trace/libor-trace.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
 	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@
+	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
 
 src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS)
 src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
@@ -202,9 +232,11 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \
 	src/common/libor-ctime.a \
 	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event.a src/trunnel/libor-trunnel.a \
+	src/trace/libor-trace.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
 	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
-	@TOR_SYSTEMD_LIBS@
+	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
 
 src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \
         @TOR_LDFLAGS_libevent@
@@ -213,8 +245,11 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \
 	src/common/libor-ctime-testing.a \
 	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	src/common/libor-event-testing.a \
+	src/trace/libor-trace.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ \
+	$(rust_ldadd)
 
 src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS)
 src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS)
@@ -224,11 +259,14 @@ src_test_test_timers_LDADD = \
 	src/common/libor-event-testing.a \
 	src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	$(rust_ldadd)
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
 noinst_HEADERS+= \
 	src/test/fakechans.h \
+	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
 	src/test/rend_test_helpers.h \
 	src/test/test.h \
@@ -239,37 +277,58 @@ noinst_HEADERS+= \
 	src/test/failing_routerdescs.inc \
 	src/test/ed25519_vectors.inc \
 	src/test/test_descriptors.inc \
+	src/test/test_hs_descriptor.inc \
 	src/test/vote_descriptors.inc
 
 noinst_PROGRAMS+= src/test/test-ntor-cl
+noinst_PROGRAMS+= src/test/test-hs-ntor-cl
 src_test_test_ntor_cl_SOURCES  = src/test/test_ntor_cl.c
 src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \
 	src/common/libor-ctime.a \
 	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	src/trace/libor-trace.a \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
+	@TOR_LZMA_LIBS@ \
+	$(rust_ldadd)
 src_test_test_ntor_cl_AM_CPPFLAGS =	       \
 	-I"$(top_srcdir)/src/or"
 
+src_test_test_hs_ntor_cl_SOURCES  = src/test/test_hs_ntor_cl.c
+src_test_test_hs_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
+src_test_test_hs_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \
+	src/common/libor-ctime.a \
+	src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \
+	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
+	@TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+src_test_test_hs_ntor_cl_AM_CPPFLAGS =	       \
+	-I"$(top_srcdir)/src/or"
+
+
 noinst_PROGRAMS += src/test/test-bt-cl
 src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c
 src_test_test_bt_cl_LDADD = src/common/libor-testing.a \
 	src/common/libor-ctime-testing.a \
+	src/trace/libor-trace.a \
 	@TOR_LIB_MATH@ \
-	@TOR_LIB_WS32@ @TOR_LIB_GDI@
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ \
+	$(rust_ldadd)
 src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
 
 EXTRA_DIST += \
 	src/test/bt_test.py \
 	src/test/ntor_ref.py \
+	src/test/hs_ntor_ref.py \
+	src/test/fuzz_static_testcases.sh \
 	src/test/slownacl_curve25519.py \
 	src/test/zero_length_keys.sh \
 	src/test/test_keygen.sh \
 	src/test/test_zero_length_keys.sh \
-	src/test/test_ntor.sh src/test/test_bt.sh \
+	src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh \
 	src/test/test-network.sh \
+	src/test/test_rust.sh \
 	src/test/test_switch_id.sh \
 	src/test/test_workqueue_cancel.sh \
 	src/test/test_workqueue_efd.sh \
diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c
index c788a33c17..d5a39cfeee 100644
--- a/src/test/log_test_helpers.c
+++ b/src/test/log_test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define LOG_PRIVATE
 #include "torlog.h"
diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h
index 922c68b42f..f7798c0249 100644
--- a/src/test/log_test_helpers.h
+++ b/src/test/log_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -71,14 +71,14 @@ void mock_dump_saved_logs(void);
                                                                         \
     assert_log_predicate(mock_saved_log_has_message_containing(str) &&  \
                          mock_saved_log_n_entries() == 1,               \
-                  "expected log to contain exactly 1 message: " # str); \
+                  "expected log to contain exactly 1 message " # str); \
   } while (0);
 
 #define expect_single_log_msg_containing(str) \
   do {                                                                  \
     assert_log_predicate(mock_saved_log_has_message_containing(str)&&   \
                          mock_saved_log_n_entries() == 1 ,              \
-            "expected log to contain 1 message, containing" # str);     \
+            "expected log to contain 1 message, containing " # str);     \
   } while (0);
 
 #define expect_no_log_msg(str) \
diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py
index df065853f3..c753588f97 100755
--- a/src/test/ntor_ref.py
+++ b/src/test/ntor_ref.py
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright 2012-2015, The Tor Project, Inc
+# Copyright 2012-2017, The Tor Project, Inc
 # See LICENSE for licensing information
 
 """
diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c
index 377337bcb9..f7880046fb 100644
--- a/src/test/rend_test_helpers.c
+++ b/src/test/rend_test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h
index 180a4e8fde..486adba436 100644
--- a/src/test/rend_test_helpers.h
+++ b/src/test/rend_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/test-child.c b/src/test/test-child.c
index fdf3ccec0a..f0bdb3ea26 100644
--- a/src/test/test-child.c
+++ b/src/test/test-child.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2016, The Tor Project, Inc. */
+/* Copyright (c) 2011-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index fd6457416a..484f13dd05 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -36,7 +36,7 @@ const char *s = NULL;
     sum += (unsigned char)buf[i];                                       \
   }
 
-#ifdef __OpenBSD__
+#ifdef OpenBSD
 /* Disable some of OpenBSD's malloc protections for this test. This helps
  * us do bad things, such as access freed buffers, without crashing. */
 const char *malloc_options="sufjj";
diff --git a/src/test/test-network.sh b/src/test/test-network.sh
index 4d9776822b..6e0f286573 100755
--- a/src/test/test-network.sh
+++ b/src/test/test-network.sh
@@ -1,103 +1,45 @@
-#! /bin/sh
+#!/bin/sh
 
-# Please do not modify this script, it has been moved to chutney/tools
+# This script calls the equivalent script in chutney/tools
 
-ECHO_N="/bin/echo -n"
+# If we already know CHUTNEY_PATH, don't bother with argument parsing
+TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
+# Call the chutney version of this script, if it exists, and we can find it
+if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+    # we can't produce any output, because we might be --quiet
+    # this preserves arguments with spaces correctly
+    exec "$TEST_NETWORK" "$@"
+fi
+
+# We need to go looking for CHUTNEY_PATH
 
+# Do we output anything at all?
+ECHO="${ECHO:-echo}"
 # Output is prefixed with the name of the script
 myname=$(basename $0)
 
-# We need to find CHUTNEY_PATH, so that we can call the version of this script
-# in chutney/tools. And we want to pass any arguments to that script as well.
-# So we source this script, which processes its arguments to find CHUTNEY_PATH.
-
-# Avoid recursively sourcing this script, and don't call the chutney version
-# while recursing, either
-if [ "$TEST_NETWORK_RECURSING" != true ]; then
-    # Process the arguments into environmental variables with this script
-    # to make sure $CHUTNEY_PATH is set
-    # When we switch to using test-network.sh in chutney/tools, --dry-run
-    # can be removed, because this script will find chutney, then pass all
-    # arguments to chutney's test-network.sh
-    echo "$myname: Parsing command-line arguments to find \$CHUTNEY_PATH"
-    export TEST_NETWORK_RECURSING=true
-    . "$0" --dry-run "$@"
-
-    # Call the chutney version of this script, if it exists, and we can find it
-    if [ -d "$CHUTNEY_PATH" -a -x "$CHUTNEY_PATH/tools/test-network.sh" ]; then
-        unset NETWORK_DRY_RUN
-        echo "$myname: Calling newer chutney script \
-$CHUTNEY_PATH/tools/test-network.sh"
-        "$CHUTNEY_PATH/tools/test-network.sh" "$@"
-        exit $?
-    else
-        echo "$myname: This script has moved to chutney/tools."
-        echo "$myname: Please update your chutney using 'git pull'."
-        # When we switch to using test-network.sh in chutney/tools, we should
-        # exit with a very loud failure here
-        echo "$myname: Falling back to the old tor version of the script."
-    fi
-fi
+# Save the arguments before we destroy them
+# This might not preserve arguments with spaces in them
+ORIGINAL_ARGS="$@"
 
+# We need to find CHUTNEY_PATH, so that we can call the version of this script
+# in chutney/tools with the same arguments. We also need to respect --quiet.
 until [ -z "$1" ]
 do
   case "$1" in
     --chutney-path)
-      export CHUTNEY_PATH="$2"
+      CHUTNEY_PATH="$2"
       shift
     ;;
     --tor-path)
-      export TOR_DIR="$2"
-      shift
-    ;;
-    # When we switch to using test-network.sh in chutney/tools, only the
-    # --chutney-path and --tor-path arguments need to be processed by this
-    # script, everything else can be handled by chutney's test-network.sh
-    --flavor|--flavour|--network-flavor|--network-flavour)
-      export NETWORK_FLAVOUR="$2"
-      shift
-    ;;
-    --delay|--sleep|--bootstrap-time|--time)
-      export BOOTSTRAP_TIME="$2"
-      shift
-    ;;
-    # Environmental variables used by chutney verify performance tests
-    # Send this many bytes per client connection (10 KBytes)
-    --data|--data-bytes|--data-byte|--bytes|--byte)
-      export CHUTNEY_DATA_BYTES="$2"
+      TOR_DIR="$2"
       shift
     ;;
-    # Make this many connections per client (1)
-    # Note: If you create 7 or more connections to a hidden service from
-    # a single Tor 0.2.7 client, you'll likely get a verification failure due
-    # to #15937. This is fixed in 0.2.8.
-    --connections|--connection|--connection-count|--count)
-      export CHUTNEY_CONNECTIONS="$2"
-      shift
+    --quiet)
+      ECHO=true
     ;;
-    # Make each client connect to each HS (0)
-    # 0 means a single client connects to each HS
-    # 1 means every client connects to every HS
-    --hs-multi-client|--hs-multi-clients|--hs-client|--hs-clients)
-      export CHUTNEY_HS_MULTI_CLIENT="$2"
-      shift
-      ;;
-    --coverage)
-      export USE_COVERAGE_BINARY=true
-      ;;
-    --dry-run)
-      # process arguments, but don't call any other scripts
-      export NETWORK_DRY_RUN=true
-      ;;
     *)
-      echo "$myname: Sorry, I don't know what to do with '$1'."
-      echo "$myname: Maybe chutney's test-network.sh understands '$1'."
-      echo "$myname: Please update your chutney using 'git pull', and set \
-\$CHUTNEY_PATH"
-      # continue processing arguments during a dry run
-      if [ "$NETWORK_DRY_RUN" != true ]; then
-          exit 2
-      fi
+      # maybe chutney's test-network.sh can handle it
     ;;
   esac
   shift
@@ -106,22 +48,22 @@ done
 # optional: $TOR_DIR is the tor build directory
 # it's used to find the location of tor binaries
 # if it's not set:
-#  - set it ro $BUILDDIR, or
+#  - set it to $BUILDDIR, or
 #  - if $PWD looks like a tor build directory, set it to $PWD, or
 #  - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH
 if [ ! -d "$TOR_DIR" ]; then
     if [ -d "$BUILDDIR/src/or" -a -d "$BUILDDIR/src/tools" ]; then
         # Choose the build directory
         # But only if it looks like one
-        echo "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
-        export TOR_DIR="$BUILDDIR"
+        $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
+        TOR_DIR="$BUILDDIR"
     elif [ -d "$PWD/src/or" -a -d "$PWD/src/tools" ]; then
         # Guess the tor directory is the current directory
         # But only if it looks like one
-        echo "$myname: \$TOR_DIR not set, trying \$PWD"
-        export TOR_DIR="$PWD"
+        $ECHO "$myname: \$TOR_DIR not set, trying \$PWD"
+        TOR_DIR="$PWD"
     else
-        echo "$myname: no \$TOR_DIR, chutney will use \$PATH for tor binaries"
+        $ECHO "$myname: no \$TOR_DIR, chutney will use \$PATH for tor binaries"
         unset TOR_DIR
     fi
 fi
@@ -133,63 +75,34 @@ fi
 #  - fail and tell the user how to clone the chutney repository
 if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then
     if [ -x "$PWD/chutney" ]; then
-        echo "$myname: \$CHUTNEY_PATH not valid, trying \$PWD"
-        export CHUTNEY_PATH="$PWD"
+        $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD"
+        CHUTNEY_PATH="$PWD"
     elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \
            -x "$TOR_DIR/../chutney/chutney" ]; then
-        echo "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney"
-        export CHUTNEY_PATH="$TOR_DIR/../chutney"
+        $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney"
+        CHUTNEY_PATH="$TOR_DIR/../chutney"
     else
-        # TODO: work out how to package and install chutney,
-        # so users can find it in $PATH
-        echo "$myname: missing 'chutney' in \$CHUTNEY_PATH ($CHUTNEY_PATH)"
-        echo "$myname: Get chutney: git clone https://git.torproject.org/\
+        $ECHO "$myname: missing 'chutney' in \$CHUTNEY_PATH ($CHUTNEY_PATH)"
+        $ECHO "$myname: Get chutney: git clone https://git.torproject.org/\
 chutney.git"
-        echo "$myname: Set \$CHUTNEY_PATH to a non-standard location: export \
+        $ECHO "$myname: Set \$CHUTNEY_PATH to a non-standard location: export \
 CHUTNEY_PATH=\`pwd\`/chutney"
         unset CHUTNEY_PATH
         exit 1
     fi
 fi
 
-# When we switch to using test-network.sh in chutney/tools, this comment and
-# everything below it can be removed
-
-# For picking up the right tor binaries.
-# If these varibles aren't set, chutney looks for tor binaries in $PATH
-if [ -d "$TOR_DIR" ]; then
-    tor_name=tor
-    tor_gencert_name=tor-gencert
-    if [ "$USE_COVERAGE_BINARY" = true ]; then
-        tor_name=tor-cov
-    fi
-    export CHUTNEY_TOR="${TOR_DIR}/src/or/${tor_name}"
-    export CHUTNEY_TOR_GENCERT="${TOR_DIR}/src/tools/${tor_gencert_name}"
-fi
-
-# Set the variables for the chutney network flavour
-export NETWORK_FLAVOUR=${NETWORK_FLAVOUR:-"bridges+hs"}
-export CHUTNEY_NETWORK=networks/$NETWORK_FLAVOUR
-
-# And finish up if we're doing a dry run
-if [ "$NETWORK_DRY_RUN" = true ]; then
-    # we can't exit here, it breaks argument processing
-    return
+TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
+# Call the chutney version of this script, if it exists, and we can find it
+if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+    $ECHO "$myname: Calling newer chutney script $TEST_NETWORK"
+    # this may fail if some arguments have spaces in them
+    # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces
+    # will be handled correctly
+    exec "$TEST_NETWORK" $ORIGINAL_ARGS
+else
+    $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH."
+    $ECHO "$myname: Please update your chutney using 'git pull'."
+    # We have failed to do what the user asked
+    exit 1
 fi
-
-cd "$CHUTNEY_PATH"
-./tools/bootstrap-network.sh $NETWORK_FLAVOUR || exit 2
-
-# Sleep some, waiting for the network to bootstrap.
-# TODO: Add chutney command 'bootstrap-status' and use that instead.
-BOOTSTRAP_TIME=${BOOTSTRAP_TIME:-35}
-$ECHO_N "$myname: sleeping for $BOOTSTRAP_TIME seconds"
-n=$BOOTSTRAP_TIME; while [ $n -gt 0 ]; do
-    sleep 1; n=$(expr $n - 1); $ECHO_N .
-done; echo ""
-./chutney verify $CHUTNEY_NETWORK
-VERIFY_EXIT_STATUS=$?
-# work around a bug/feature in make -j2 (or more)
-# where make hangs if any child processes are still alive
-./chutney stop $CHUTNEY_NETWORK
-exit $VERIFY_EXIT_STATUS
diff --git a/src/test/test-timers.c b/src/test/test-timers.c
index b5fcade7f8..99715f4333 100644
--- a/src/test/test-timers.c
+++ b/src/test/test-timers.c
@@ -1,4 +1,4 @@
-/* Copyright 2016, The Tor Project, Inc. */
+/* Copyright 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test.c b/src/test/test.c
index 0fef697909..911ef0c24e 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,13 +44,13 @@ double fabs(double x);
 #include "buffers.h"
 #include "circuitlist.h"
 #include "circuitstats.h"
+#include "compress.h"
 #include "config.h"
 #include "connection_edge.h"
 #include "geoip.h"
 #include "rendcommon.h"
 #include "rendcache.h"
 #include "test.h"
-#include "torgzip.h"
 #include "main.h"
 #include "memarea.h"
 #include "onion.h"
@@ -1205,17 +1205,24 @@ struct testgroup_t testgroups[] = {
   { "cellfmt/", cell_format_tests },
   { "cellqueue/", cell_queue_tests },
   { "channel/", channel_tests },
+  { "channelpadding/", channelpadding_tests },
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
+  { "circuitbuild/", circuitbuild_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
+  { "circuituse/", circuituse_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
   { "connection/", connection_tests },
+  { "conscache/", conscache_tests },
+  { "consdiff/", consdiff_tests },
+  { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
   { "control/", controller_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
+  { "crypto/openssl/", crypto_openssl_tests },
   { "dos/", dos_tests },
   { "dir/", dir_tests },
   { "dir_handle_get/", dir_handle_get_tests },
@@ -1224,7 +1231,11 @@ struct testgroup_t testgroups[] = {
   { "entrynodes/", entrynodes_tests },
   { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
-  { "hs/", hs_tests },
+  { "legacy_hs/", hs_tests },
+  { "hs_cache/", hs_cache },
+  { "hs_descriptor/", hs_descriptor },
+  { "hs_service/", hs_service_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
   { "link-handshake/", link_handshake_tests },
@@ -1243,10 +1254,12 @@ struct testgroup_t testgroups[] = {
   { "routerkeys/", routerkeys_tests },
   { "routerlist/", routerlist_tests },
   { "routerset/" , routerset_tests },
+  { "rust/", rust_tests },
   { "scheduler/", scheduler_tests },
   { "socks/", socks_tests },
   { "shared-random/", sr_tests },
   { "status/" , status_tests },
+  { "storagedir/", storagedir_tests },
   { "tortls/", tortls_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
diff --git a/src/test/test.h b/src/test/test.h
index 028082386e..ea1b16adee 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_H
@@ -75,6 +75,8 @@
 const char *get_fname(const char *name);
 const char *get_fname_rnd(const char *name);
 struct crypto_pk_t *pk_generate(int idx);
+void init_pregenerated_keys(void);
+void free_pregenerated_keys(void);
 
 #define US2_CONCAT_2__(a, b) a ## __ ## b
 #define US_CONCAT_2__(a, b) a ## _ ## b
@@ -180,17 +182,24 @@ extern struct testcase_t buffer_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
+extern struct testcase_t channelpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
+extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
+extern struct testcase_t circuituse_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
 extern struct testcase_t connection_tests[];
+extern struct testcase_t conscache_tests[];
+extern struct testcase_t consdiff_tests[];
+extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
 extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
 extern struct testcase_t crypto_tests[];
+extern struct testcase_t crypto_openssl_tests[];
 extern struct testcase_t dos_tests[];
 extern struct testcase_t dir_tests[];
 extern struct testcase_t dir_handle_get_tests[];
@@ -199,6 +208,10 @@ extern struct testcase_t entrynodes_tests[];
 extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
 extern struct testcase_t hs_tests[];
+extern struct testcase_t hs_cache[];
+extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_service_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
@@ -221,7 +234,9 @@ extern struct testcase_t router_tests[];
 extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
+extern struct testcase_t rust_tests[];
 extern struct testcase_t scheduler_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t socks_tests[];
 extern struct testcase_t status_tests[];
 extern struct testcase_t thread_tests[];
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index be440a0925..2f591bdfe7 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESSMAP_PRIVATE
@@ -9,6 +9,24 @@
 #include "test.h"
 #include "addressmap.h"
 
+/** Mocking replacement: only handles localhost. */
+static int
+mock_tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out)
+{
+  if (!strcmp(name, "localhost")) {
+    if (family == AF_INET || family == AF_UNSPEC) {
+      tor_addr_from_ipv4h(addr_out, 0x7f000001);
+      return 0;
+    } else if (family == AF_INET6) {
+      char bytes[16] = { 0, 0, 0, 0, 0, 0, 0, 0,
+                         0, 0, 0, 0, 0, 0, 0, 1 };
+      tor_addr_from_ipv6_bytes(addr_out, bytes);
+      return 0;
+    }
+  }
+  return -1;
+}
+
 static void
 test_addr_basic(void *arg)
 {
@@ -29,6 +47,9 @@ test_addr_basic(void *arg)
   tt_int_op(u32,OP_EQ, 0x04030201u);
   tt_int_op(u16,OP_EQ, 99);
   tor_free(cp);
+
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup);
+
   tt_assert(!addr_port_lookup(LOG_WARN, "nonexistent.address:4040",
                                &cp, NULL, &u16));
   tt_str_op(cp,OP_EQ, "nonexistent.address");
@@ -36,8 +57,8 @@ test_addr_basic(void *arg)
   tor_free(cp);
   tt_assert(!addr_port_lookup(LOG_WARN, "localhost:9999", &cp, &u32, &u16));
   tt_str_op(cp,OP_EQ, "localhost");
-  tt_int_op(u32,OP_EQ, 0x7f000001u);
   tt_int_op(u16,OP_EQ, 9999);
+  tt_int_op(u32,OP_EQ, 0x7f000001u);
   tor_free(cp);
   u32 = 3;
   tt_assert(!addr_port_lookup(LOG_WARN, "localhost", NULL, &u32, &u16));
@@ -75,6 +96,7 @@ test_addr_basic(void *arg)
   }
 
  done:
+  UNMOCK(tor_addr_lookup);
   tor_free(cp);
 }
 
diff --git a/src/test/test_address.c b/src/test/test_address.c
index 0d142ad483..50a0574522 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESS_PRIVATE
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 95b4f48f11..ed588ecc5b 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -19,14 +19,12 @@ static int crashtype = 0;
 
 #ifdef __GNUC__
 #define NOINLINE __attribute__((noinline))
-#define NORETURN __attribute__((noreturn))
 #endif
 
 int crash(int x) NOINLINE;
 int oh_what(int x) NOINLINE;
 int a_tangled_web(int x) NOINLINE;
 int we_weave(int x) NOINLINE;
-static void abort_handler(int s) NORETURN;
 
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
 DISABLE_GCC_WARNING(null-dereference)
@@ -76,13 +74,6 @@ we_weave(int x)
   return a_tangled_web(x) + a_tangled_web(x+1);
 }
 
-static void
-abort_handler(int s)
-{
-  (void)s;
-  exit(0);
-}
-
 int
 main(int argc, char **argv)
 {
@@ -120,8 +111,6 @@ main(int argc, char **argv)
 
   configure_backtrace_handler(NULL);
 
-  signal(SIGABRT, abort_handler);
-
   printf("%d\n", we_weave(2));
 
   clean_up_backtrace_handler();
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3408da3aa9..07114a8571 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
@@ -578,120 +578,150 @@ test_buffer_time_tracking(void *arg)
 }
 
 static void
-test_buffers_zlib_impl(int finalize_with_nil)
+test_buffers_compress_fin_at_chunk_end_impl(compress_method_t method,
+                                            compression_level_t level)
 {
   char *msg = NULL;
   char *contents = NULL;
   char *expanded = NULL;
   buf_t *buf = NULL;
-  tor_zlib_state_t *zlib_state = NULL;
+  tor_compress_state_t *compress_state = NULL;
   size_t out_len, in_len;
-  int done;
+  size_t sz, headerjunk;
 
   buf = buf_new_with_capacity(128); /* will round up */
-  zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
+  sz = buf_get_default_chunk_size(buf);
+  msg = tor_malloc_zero(sz);
 
-  msg = tor_malloc(512);
-  crypto_rand(msg, 512);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+128, 128, 0), OP_EQ, 0);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+256, 256, 0), OP_EQ, 0);
-  done = !finalize_with_nil;
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, "all done", 9, done), OP_EQ, 0);
-  if (finalize_with_nil) {
-    tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
-  }
+  write_to_buf(msg, 1, buf);
+  tt_assert(buf->head);
+
+  /* Fill up the chunk so the compression stuff won't fit in one chunk. */
+  tt_uint_op(buf->head->memlen, OP_LT, sz);
+  headerjunk = buf->head->memlen - 7;
+  write_to_buf(msg, headerjunk-1, buf);
+  tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
+  tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
+  /* Write an empty string, with finalization on. */
+  compress_state = tor_compress_new(1, method, level);
+  tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
   tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
 
-  tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len,
-                                       contents, in_len,
-                                       ZLIB_METHOD, 1,
-                                       LOG_WARN));
+  if (method == NO_METHOD) {
+    tt_uint_op(in_len, OP_EQ, headerjunk);
+  } else {
+    tt_uint_op(in_len, OP_GT, headerjunk);
+  }
 
-  tt_int_op(out_len, OP_GE, 128);
-  tt_mem_op(msg, OP_EQ, expanded, 128);
-  tt_int_op(out_len, OP_GE, 512);
-  tt_mem_op(msg, OP_EQ, expanded, 512);
-  tt_int_op(out_len, OP_EQ, 512+9);
-  tt_mem_op("all done", OP_EQ, expanded+512, 9);
+  tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
+                                     contents + headerjunk,
+                                     in_len - headerjunk,
+                                     method, 1,
+                                     LOG_WARN));
+
+  tt_int_op(out_len, OP_EQ, 0);
+  tt_assert(expanded);
 
  done:
   buf_free(buf);
-  tor_zlib_free(zlib_state);
+  tor_compress_free(compress_state);
   tor_free(contents);
   tor_free(expanded);
   tor_free(msg);
 }
 
 static void
-test_buffers_zlib(void *arg)
-{
-  (void) arg;
-  test_buffers_zlib_impl(0);
-}
-static void
-test_buffers_zlib_fin_with_nil(void *arg)
-{
-  (void) arg;
-  test_buffers_zlib_impl(1);
-}
-
-static void
-test_buffers_zlib_fin_at_chunk_end(void *arg)
+test_buffers_compress_impl(compress_method_t method,
+                           compression_level_t level,
+                           int finalize_with_nil)
 {
   char *msg = NULL;
   char *contents = NULL;
   char *expanded = NULL;
   buf_t *buf = NULL;
-  tor_zlib_state_t *zlib_state = NULL;
+  tor_compress_state_t *compress_state = NULL;
   size_t out_len, in_len;
-  size_t sz, headerjunk;
-  (void) arg;
+  int done;
 
   buf = buf_new_with_capacity(128); /* will round up */
-  sz = buf_get_default_chunk_size(buf);
-  msg = tor_malloc_zero(sz);
+  compress_state = tor_compress_new(1, method, level);
 
-  write_to_buf(msg, 1, buf);
-  tt_assert(buf->head);
-
-  /* Fill up the chunk so the zlib stuff won't fit in one chunk. */
-  tt_uint_op(buf->head->memlen, OP_LT, sz);
-  headerjunk = buf->head->memlen - 7;
-  write_to_buf(msg, headerjunk-1, buf);
-  tt_uint_op(buf->head->datalen, OP_EQ, headerjunk);
-  tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk);
-  /* Write an empty string, with finalization on. */
-  zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
-  tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0);
+  msg = tor_malloc(512);
+  crypto_rand(msg, 512);
+  tt_int_op(write_to_buf_compress(buf, compress_state,
+                                  msg, 128, 0), OP_EQ, 0);
+  tt_int_op(write_to_buf_compress(buf, compress_state,
+                                  msg+128, 128, 0), OP_EQ, 0);
+  tt_int_op(write_to_buf_compress(buf, compress_state,
+                                  msg+256, 256, 0), OP_EQ, 0);
+  done = !finalize_with_nil;
+  tt_int_op(write_to_buf_compress(buf, compress_state,
+                                  "all done", 9, done), OP_EQ, 0);
+  if (finalize_with_nil) {
+    tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0);
+  }
 
   in_len = buf_datalen(buf);
   contents = tor_malloc(in_len);
 
   tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0);
 
-  tt_uint_op(in_len, OP_GT, headerjunk);
-
-  tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len,
-                                  contents + headerjunk, in_len - headerjunk,
-                                  ZLIB_METHOD, 1,
-                                  LOG_WARN));
+  tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len,
+                                     contents, in_len,
+                                     method, 1,
+                                     LOG_WARN));
 
-  tt_int_op(out_len, OP_EQ, 0);
-  tt_assert(expanded);
+  tt_int_op(out_len, OP_GE, 128);
+  tt_mem_op(msg, OP_EQ, expanded, 128);
+  tt_int_op(out_len, OP_GE, 512);
+  tt_mem_op(msg, OP_EQ, expanded, 512);
+  tt_int_op(out_len, OP_EQ, 512+9);
+  tt_mem_op("all done", OP_EQ, expanded+512, 9);
 
  done:
   buf_free(buf);
-  tor_zlib_free(zlib_state);
+  tor_compress_free(compress_state);
   tor_free(contents);
   tor_free(expanded);
   tor_free(msg);
 }
 
+static void
+test_buffers_compress(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  compression_level_t levels[] = {
+    BEST_COMPRESSION,
+    HIGH_COMPRESSION,
+    MEDIUM_COMPRESSION,
+    LOW_COMPRESSION
+  };
+
+  for (unsigned l = 0; l < ARRAY_LENGTH(levels); ++l) {
+    compression_level_t level = levels[l];
+
+    test_buffers_compress_impl(method, level, 0);
+    test_buffers_compress_impl(method, level, 1);
+    test_buffers_compress_fin_at_chunk_end_impl(method, level);
+  }
+
+ done:
+  ;
+}
+
 static const uint8_t *tls_read_ptr;
 static int n_remaining;
 static int next_reply_val[16];
@@ -765,6 +795,49 @@ test_buffers_chunk_size(void *arg)
   ;
 }
 
+static void
+test_buffers_find_contentlen(void *arg)
+{
+  static const struct {
+    const char *headers;
+    int r;
+    int contentlen;
+  } results[] = {
+    { "Blah blah\r\nContent-Length: 1\r\n\r\n", 1, 1 },
+    { "Blah blah\r\n\r\n", 0, 0 }, /* no content-len */
+    { "Blah blah Content-Length: 1\r\n", 0, 0 }, /* no content-len. */
+    { "Blah blah\r\nContent-Length: 100000\r\n", 1, 100000},
+    { "Blah blah\r\nContent-Length: 1000000000000000000000000\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 0\r\n", 1, 0},
+    { "Blah blah\r\nContent-Length: -1\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1x\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1 x\r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: 1 \r\n", 1, 1},
+    { "Blah blah\r\nContent-Length:  \r\n", -1, 0},
+    { "Blah blah\r\nContent-Length: ", -1, 0},
+    { "Blah blah\r\nContent-Length: 5050", -1, 0},
+    { NULL, 0, 0 }
+  };
+  int i;
+
+  (void)arg;
+
+  for (i = 0; results[i].headers; ++i) {
+    int r;
+    size_t sz;
+    size_t headerlen = strlen(results[i].headers);
+    char * tmp = tor_memdup(results[i].headers, headerlen);/* ensure no eos */
+    sz = 999; /* to ensure it gets set */
+    r = buf_http_find_content_length(tmp, headerlen, &sz);
+    tor_free(tmp);
+    log_debug(LD_DIR, "%d: %s", i, escaped(results[i].headers));
+    tt_int_op(r, ==, results[i].r);
+    tt_int_op(sz, ==, results[i].contentlen);
+  }
+ done:
+  ;
+}
+
 struct testcase_t buffer_tests[] = {
   { "basic", test_buffers_basic, TT_FORK, NULL, NULL },
   { "copy", test_buffer_copy, TT_FORK, NULL, NULL },
@@ -773,13 +846,22 @@ struct testcase_t buffer_tests[] = {
   { "allocation_tracking", test_buffer_allocation_tracking, TT_FORK,
     NULL, NULL },
   { "time_tracking", test_buffer_time_tracking, TT_FORK, NULL, NULL },
-  { "zlib", test_buffers_zlib, TT_FORK, NULL, NULL },
-  { "zlib_fin_with_nil", test_buffers_zlib_fin_with_nil, TT_FORK, NULL, NULL },
-  { "zlib_fin_at_chunk_end", test_buffers_zlib_fin_at_chunk_end, TT_FORK,
-    NULL, NULL},
   { "tls_read_mocked", test_buffers_tls_read_mocked, 0,
     NULL, NULL },
   { "chunk_size", test_buffers_chunk_size, 0, NULL, NULL },
+  { "find_contentlen", test_buffers_find_contentlen, 0, NULL, NULL },
+
+  { "compress/zlib", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"deflate" },
+  { "compress/gzip", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"gzip" },
+  { "compress/zstd", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"x-zstd" },
+  { "compress/lzma", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"x-tor-lzma" },
+  { "compress/none", test_buffers_compress, TT_FORK,
+    &passthrough_setup, (char*)"identity" },
+
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c
index f429f4291d..007f7e3d3e 100644
--- a/src/test/test_cell_formats.c
+++ b/src/test/test_cell_formats.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,6 +11,7 @@
 #include "channel.h"
 #include "connection_edge.h"
 #include "connection_or.h"
+#include "config.h"
 #include "onion.h"
 #include "onion_tap.h"
 #include "onion_fast.h"
@@ -698,6 +699,7 @@ test_cfmt_extend_cells(void *arg)
   tt_int_op(61681, OP_EQ, ec.orport_ipv4.port);
   tt_str_op("2002::f0:c51e", OP_EQ, fmt_addr(&ec.orport_ipv6.addr));
   tt_int_op(4370, OP_EQ, ec.orport_ipv6.port);
+  tt_assert(ed25519_public_key_is_zero(&ec.ed_pubkey));
   tt_mem_op(ec.node_id,OP_EQ, "anthropomorphization", 20);
   tt_int_op(cc->cell_type, OP_EQ, CELL_CREATE2);
   tt_int_op(cc->handshake_type, OP_EQ, 0x105);
@@ -717,6 +719,37 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op(p2+1+8+22+4,OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
 
+  /* Now let's add an ed25519 key to that extend2 cell. */
+  memcpy(ec.ed_pubkey.pubkey,
+         "brownshoesdontmakeit/brownshoesd", 32);
+
+  /* As before, since we aren't extending by ed25519. */
+  get_options_mutable()->ExtendByEd25519ID = 0;
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  test_memeq_hex(p2,
+                 "02000612F40001F0F1"
+                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 "01050063");
+
+  /* Now try with the ed25519 ID. */
+  get_options_mutable()->ExtendByEd25519ID = 1;
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_len, OP_EQ, 89+99-34-20 + 34);
+  test_memeq_hex(p2,
+                 "03000612F40001F0F1"
+                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 // ed digest follows:
+                 "0320" "62726f776e73686f6573646f6e746d616b656"
+                        "9742f62726f776e73686f657364"
+                 "01050063");
+  /* Can we parse that? Did the key come through right? */
+  memset(&ec, 0, sizeof(ec));
+  tt_int_op(0, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
+                                        p2, p2_len));
+  tt_mem_op("brownshoesdontmakeit/brownshoesd", OP_EQ,
+            ec.ed_pubkey.pubkey, 32);
+
   /* == Now try parsing some junk */
 
   /* Try a too-long handshake */
@@ -1257,7 +1290,7 @@ struct testcase_t cell_format_tests[] = {
   TEST(connected_cells, 0),
   TEST(create_cells, 0),
   TEST(created_cells, 0),
-  TEST(extend_cells, 0),
+  TEST(extend_cells, TT_FORK),
   TEST(extended_cells, 0),
   TEST(resolved_cells, 0),
   TEST(is_destroy, 0),
diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c
index 93ac9854d8..69e89b69b0 100644
--- a/src/test/test_cell_queue.c
+++ b/src/test/test_cell_queue.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index a9e0634d9e..f5999b8e67 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
@@ -1405,10 +1405,14 @@ test_channel_queue_impossible(void *arg)
 
   /* Let it drain and check that the bad entry is discarded */
   test_chan_accept_cells = 1;
+  tor_capture_bugs_(1);
   channel_change_state(ch, CHANNEL_STATE_OPEN);
   tt_assert(test_cells_written == old_count);
   tt_int_op(chan_cell_queue_len(&(ch->outgoing_queue)), ==, 0);
 
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), ==, 1);
+  tor_end_capture_bugs_();
+
  done:
   free_fake_channel(ch);
 
@@ -1764,6 +1768,112 @@ test_channel_write(void *arg)
   return;
 }
 
+static void
+test_channel_id_map(void *arg)
+{
+  (void)arg;
+#define N_CHAN 6
+  char rsa_id[N_CHAN][DIGEST_LEN];
+  ed25519_public_key_t *ed_id[N_CHAN];
+  channel_t *chan[N_CHAN];
+  int i;
+  ed25519_public_key_t ed_zero;
+  memset(&ed_zero, 0, sizeof(ed_zero));
+
+  tt_assert(sizeof(rsa_id[0]) == DIGEST_LEN); // Do I remember C?
+
+  for (i = 0; i < N_CHAN; ++i) {
+    crypto_rand(rsa_id[i], DIGEST_LEN);
+    ed_id[i] = tor_malloc_zero(sizeof(*ed_id[i]));
+    crypto_rand((char*)ed_id[i]->pubkey, sizeof(ed_id[i]->pubkey));
+  }
+
+  /* For channel 3, have no Ed identity. */
+  tor_free(ed_id[3]);
+
+  /* Channel 2 and 4 have same ROSA identity */
+  memcpy(rsa_id[4], rsa_id[2], DIGEST_LEN);
+
+  /* Channel 2 and 4 and 5 have same RSA identity */
+  memcpy(rsa_id[4], rsa_id[2], DIGEST_LEN);
+  memcpy(rsa_id[5], rsa_id[2], DIGEST_LEN);
+
+  /* Channels 2 and 5 have same Ed25519 identity */
+  memcpy(ed_id[5], ed_id[2], sizeof(*ed_id[2]));
+
+  for (i = 0; i < N_CHAN; ++i) {
+    chan[i] = new_fake_channel();
+    channel_register(chan[i]);
+    channel_set_identity_digest(chan[i], rsa_id[i], ed_id[i]);
+  }
+
+  /* Lookup by RSA id only */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], NULL));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], NULL));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], NULL));
+  channel_t *ch;
+  ch = channel_find_by_remote_identity(rsa_id[2], NULL);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == NULL);
+
+  /* As above, but with zero Ed25519 ID (meaning "any ID") */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], &ed_zero));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], &ed_zero));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], &ed_zero));
+  ch = channel_find_by_remote_identity(rsa_id[2], &ed_zero);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == chan[2] || ch == chan[4] || ch == chan[5]);
+  ch = channel_next_with_rsa_identity(ch);
+  tt_assert(ch == NULL);
+
+  /* Lookup nonexistent RSA identity */
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity("!!!!!!!!!!!!!!!!!!!!", NULL));
+
+  /* Look up by full identity pair */
+  tt_ptr_op(chan[0], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[0], ed_id[0]));
+  tt_ptr_op(chan[1], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[1], ed_id[1]));
+  tt_ptr_op(chan[3], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], ed_id[3] /*NULL*/));
+  tt_ptr_op(chan[4], OP_EQ,
+            channel_find_by_remote_identity(rsa_id[4], ed_id[4]));
+  ch = channel_find_by_remote_identity(rsa_id[2], ed_id[2]);
+  tt_assert(ch == chan[2] || ch == chan[5]);
+
+  /* Look up RSA identity with wrong ed25519 identity */
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[4], ed_id[0]));
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[2], ed_id[1]));
+  tt_ptr_op(NULL, OP_EQ,
+            channel_find_by_remote_identity(rsa_id[3], ed_id[1]));
+
+ done:
+  for (i = 0; i < N_CHAN; ++i) {
+    channel_clear_identity_digest(chan[i]);
+    channel_unregister(chan[i]);
+    free_fake_channel(chan[i]);
+    tor_free(ed_id[i]);
+  }
+#undef N_CHAN
+}
+
 struct testcase_t channel_tests[] = {
   { "dumpstats", test_channel_dumpstats, TT_FORK, NULL, NULL },
   { "flush", test_channel_flush, TT_FORK, NULL, NULL },
@@ -1776,6 +1886,7 @@ struct testcase_t channel_tests[] = {
   { "queue_incoming", test_channel_queue_incoming, TT_FORK, NULL, NULL },
   { "queue_size", test_channel_queue_size, TT_FORK, NULL, NULL },
   { "write", test_channel_write, TT_FORK, NULL, NULL },
+  { "id_map", test_channel_id_map, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
new file mode 100644
index 0000000000..d399738ce6
--- /dev/null
+++ b/src/test/test_channelpadding.c
@@ -0,0 +1,1129 @@
+#define TOR_CHANNEL_INTERNAL_
+#define MAIN_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define TOR_TIMERS_PRIVATE
+#include "or.h"
+#include "test.h"
+#include "testsupport.h"
+#include "connection.h"
+#include "connection_or.h"
+#include "channel.h"
+#include "channeltls.h"
+#include "channelpadding.h"
+#include "compat_libevent.h"
+#include "config.h"
+#include <event2/event.h>
+#include "compat_time.h"
+#include "main.h"
+#include "networkstatus.h"
+#include "log_test_helpers.h"
+
+int channelpadding_get_netflow_inactive_timeout_ms(channel_t *chan);
+int64_t channelpadding_compute_time_until_pad_for_netflow(channel_t *chan);
+int channelpadding_send_disable_command(channel_t*);
+int channelpadding_find_timerslot(channel_t *chan);
+
+void test_channelpadding_timers(void *arg);
+void test_channelpadding_consensus(void *arg);
+void test_channelpadding_negotiation(void *arg);
+void test_channelpadding_decide_to_pad_channel(void *arg);
+void test_channelpadding_killonehop(void *arg);
+
+void dummy_nop_timer(void);
+
+#define NSEC_PER_MSEC (1000*1000)
+
+/* Thing to cast to fake tor_tls_t * to appease assert_connection_ok() */
+static int fake_tortls = 0; /* Bleh... */
+
+static int dont_stop_libevent = 0;
+
+// From test_channel.c
+channel_t * new_fake_channel(void);
+void free_fake_channel(channel_t*);
+
+static int
+mock_channel_has_queued_writes(channel_t *chan)
+{
+  (void)chan;
+  return 0;
+}
+
+static int tried_to_write_cell = 0;
+
+static channel_t *relay1_relay2;
+static channel_t *relay2_relay1;
+static channel_t *relay3_client;
+static channel_t *client_relay3;
+
+static int
+mock_channel_write_cell_relay2(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay1_relay2)->conn);
+  event_base_loopbreak(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_relay1(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay2_relay1)->conn);
+  event_base_loopbreak(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_relay3(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)client_relay3)->conn);
+  event_base_loopbreak(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell_client(channel_t *chan, cell_t *cell)
+{
+  (void)chan;
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)relay3_client)->conn);
+  event_base_loopbreak(tor_libevent_get_base());
+  return 0;
+}
+
+static int
+mock_channel_write_cell(channel_t *chan, cell_t *cell)
+{
+  tried_to_write_cell++;
+  channel_tls_handle_cell(cell, ((channel_tls_t*)chan)->conn);
+  if (!dont_stop_libevent)
+    event_base_loopbreak(tor_libevent_get_base());
+  return 0;
+}
+
+static void
+setup_fake_connection_for_channel(channel_tls_t *chan)
+{
+  or_connection_t *conn = (or_connection_t*)connection_new(CONN_TYPE_OR,
+                                                           AF_INET);
+
+  conn->base_.conn_array_index = smartlist_len(connection_array);
+  smartlist_add(connection_array, conn);
+
+  conn->chan = chan;
+  chan->conn = conn;
+
+  conn->base_.magic = OR_CONNECTION_MAGIC;
+  conn->base_.state = OR_CONN_STATE_OPEN;
+  conn->base_.type = CONN_TYPE_OR;
+  conn->base_.socket_family = AF_INET;
+  conn->base_.address = tor_strdup("<fake>");
+
+  conn->base_.port = 4242;
+
+  conn->tls = (tor_tls_t *)((void *)(&fake_tortls));
+
+  conn->link_proto = MIN_LINK_PROTO_FOR_CHANNEL_PADDING;
+
+  connection_or_set_canonical(conn, 1);
+}
+
+static channel_tls_t *
+new_fake_channeltls(uint8_t id)
+{
+  channel_tls_t *chan = tor_realloc(new_fake_channel(), sizeof(channel_tls_t));
+  chan->base_.magic = TLS_CHAN_MAGIC;
+  setup_fake_connection_for_channel(chan);
+  chan->base_.channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+  chan->base_.has_queued_writes = mock_channel_has_queued_writes;
+  chan->base_.write_cell = mock_channel_write_cell;
+  chan->base_.padding_enabled = 1;
+
+  chan->base_.identity_digest[0] = id;
+  channel_register(&chan->base_);
+
+  return chan;
+}
+
+static void
+free_fake_channeltls(channel_tls_t *chan)
+{
+  channel_unregister(&chan->base_);
+
+  tor_free(((channel_tls_t*)chan)->conn->base_.address);
+  buf_free(((channel_tls_t*)chan)->conn->base_.inbuf);
+  buf_free(((channel_tls_t*)chan)->conn->base_.outbuf);
+  tor_free(((channel_tls_t*)chan)->conn);
+
+  timer_free(chan->base_.padding_timer);
+  channel_handle_free(chan->base_.timer_handle);
+  channel_handles_clear(&chan->base_);
+
+  free_fake_channel(&chan->base_);
+
+  return;
+}
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus
+        = tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+  channelpadding_new_consensus_params(current_md_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+setup_mock_network(void)
+{
+  routerstatus_t *relay;
+  connection_array = smartlist_new();
+
+  relay1_relay2 = (channel_t*)new_fake_channeltls(2);
+  relay1_relay2->write_cell = mock_channel_write_cell_relay1;
+  channel_timestamp_active(relay1_relay2);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 1;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  relay2_relay1 = (channel_t*)new_fake_channeltls(1);
+  relay2_relay1->write_cell = mock_channel_write_cell_relay2;
+  channel_timestamp_active(relay2_relay1);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 2;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  relay3_client = (channel_t*)new_fake_channeltls(0);
+  relay3_client->write_cell = mock_channel_write_cell_relay3;
+  relay3_client->is_client = 1;
+  channel_timestamp_active(relay3_client);
+  relay = tor_malloc_zero(sizeof(routerstatus_t));
+  relay->identity_digest[0] = 3;
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+
+  client_relay3 = (channel_t*)new_fake_channeltls(3);
+  client_relay3->write_cell = mock_channel_write_cell_client;
+  channel_timestamp_active(client_relay3);
+
+  channel_do_open_actions(relay1_relay2);
+  channel_do_open_actions(relay2_relay1);
+  channel_do_open_actions(relay3_client);
+  channel_do_open_actions(client_relay3);
+}
+
+static void
+free_mock_network(void)
+{
+  free_fake_channeltls((channel_tls_t*)relay1_relay2);
+  free_fake_channeltls((channel_tls_t*)relay2_relay1);
+  free_fake_channeltls((channel_tls_t*)relay3_client);
+  free_fake_channeltls((channel_tls_t*)client_relay3);
+
+  smartlist_free(connection_array);
+}
+
+static void
+dummy_timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono)
+{
+  (void)t; (void)arg; (void)now_mono;
+  event_base_loopbreak(tor_libevent_get_base());
+  return;
+}
+
+// This hack adds a dummy timer so that the libevent base loop
+// actually returns when we don't expect any timers to fire. Otherwise,
+// the global_timer_event gets scheduled an hour from now, and the
+// base loop never returns.
+void
+dummy_nop_timer(void)
+{
+  tor_timer_t *dummy_timer = timer_new(dummy_timer_cb, NULL);
+  struct timeval timeout;
+  timeout.tv_sec = 1;
+  timeout.tv_usec = 0;
+
+  timer_schedule(dummy_timer, &timeout);
+
+  event_base_loop(tor_libevent_get_base(), 0);
+  timer_free(dummy_timer);
+}
+
+#define CHANNELPADDING_MAX_TIMERS 25
+#define CHANNELS_TO_TEST (CHANNELPADDING_MAX_TIMERS*4)
+/**
+ * Tests to ensure that we handle more than the max number of pending
+ * timers properly.
+ */
+void
+test_channelpadding_timers(void *arg)
+{
+  channelpadding_decision_t decision;
+  channel_t *chans[CHANNELS_TO_TEST];
+  int64_t new_time;
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  connection_array = smartlist_new();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+
+  timers_initialize();
+  channelpadding_new_consensus_params(NULL);
+
+  for (int i = 0; i < CHANNELS_TO_TEST; i++) {
+    chans[i] = (channel_t*)new_fake_channeltls(0);
+    channel_timestamp_active(chans[i]);
+  }
+
+  for (int j = 0; j < 2; j++) {
+    tried_to_write_cell = 0;
+    int i = 0;
+
+    /* This loop fills our timerslot array with timers of increasing time
+     * until they fire */
+    for (; i < CHANNELPADDING_MAX_TIMERS; i++) {
+      chans[i]->next_padding_time_ms = monotime_coarse_absolute_msec()
+                                        + 10 + i*4;
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to the first position in the timerslot
+     * array, since its timeout is before all other timers. */
+    for (; i < CHANNELS_TO_TEST/3; i++) {
+      chans[i]->next_padding_time_ms = monotime_coarse_absolute_msec() + 1;
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to our existing lists in a weak
+     * pseudorandom pattern.  It ensures that the lists can grow with multiple
+     * timers in them. */
+    for (; i < CHANNELS_TO_TEST/2; i++) {
+      chans[i]->next_padding_time_ms = monotime_coarse_absolute_msec() + 10 +
+          i*3 % CHANNELPADDING_MAX_TIMERS;
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    /* This loop should add timers to the last position in the timerslot
+     * array, since its timeout is after all other timers. */
+    for (; i < CHANNELS_TO_TEST; i++) {
+      chans[i]->next_padding_time_ms = monotime_coarse_absolute_msec() + 500 +
+          i % CHANNELPADDING_MAX_TIMERS;
+      decision = channelpadding_decide_to_pad_channel(chans[i]);
+      tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+      tt_assert(chans[i]->pending_padding_callback);
+      tt_int_op(tried_to_write_cell, OP_EQ, 0);
+    }
+
+    // Wait for the timers and then kill the event loop.
+    new_time = (monotime_coarse_absolute_msec()+1001)*NSEC_PER_MSEC;
+    monotime_coarse_set_mock_time_nsec(new_time);
+    monotime_set_mock_time_nsec(new_time);
+    timers_run_pending();
+
+    tt_int_op(tried_to_write_cell, OP_EQ, CHANNELS_TO_TEST);
+
+    // Test that we have no pending callbacks and all empty slots now
+    for (i = 0; i < CHANNELS_TO_TEST; i++) {
+      tt_assert(!chans[i]->pending_padding_callback);
+    }
+  }
+
+ done:
+  for (int i = 0; i < CHANNELS_TO_TEST; i++) {
+    free_fake_channeltls((channel_tls_t*)chans[i]);
+  }
+  smartlist_free(connection_array);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_killonehop(void *arg)
+{
+  channelpadding_decision_t decision;
+  int64_t new_time;
+  (void)arg;
+  tor_libevent_postfork();
+
+  routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+
+  timers_initialize();
+  setup_mock_consensus();
+  setup_mock_network();
+
+  /* Do we disable padding if tor2webmode or rsos are enabled, and
+   * the consensus says don't pad?  */
+
+  /* Ensure we can kill tor2web and rsos padding if we want. */
+  // First, test that padding works if either is enabled
+  smartlist_clear(current_md_consensus->net_params);
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  tried_to_write_cell = 0;
+  get_options_mutable()->Tor2webMode = 1;
+  client_relay3->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Then test disabling each via consensus param
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_tor2web=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  // Before the client tries to pad, the relay will still pad:
+  tried_to_write_cell = 0;
+  relay3_client->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->Tor2webMode = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(relay3_client->pending_padding_callback);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test client side (it should stop immediately, but send a negotiate)
+  tried_to_write_cell = 0;
+  tt_assert(relay3_client->padding_enabled);
+  tt_assert(client_relay3->padding_enabled);
+  get_options_mutable()->Tor2webMode = 1;
+  /* For the relay to recieve the negotiate: */
+  get_options_mutable()->ORPort_set = 1;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+  tt_assert(!relay3_client->padding_enabled);
+
+  // Test relay side (it should have gotten the negotiation to disable)
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->Tor2webMode = 0;
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+
+  /* Repeat for SOS */
+  // First, test that padding works if either is enabled
+  smartlist_clear(current_md_consensus->net_params);
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  tried_to_write_cell = 0;
+  get_options_mutable()->ORPort_set = 0;
+  get_options_mutable()->HiddenServiceSingleHopMode = 1;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 1;
+  client_relay3->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Then test disabling each via consensus param
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_single_onion=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  // Before the client tries to pad, the relay will still pad:
+  tried_to_write_cell = 0;
+  relay3_client->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->HiddenServiceSingleHopMode = 0;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(relay3_client->pending_padding_callback);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test client side (it should stop immediately)
+  get_options_mutable()->HiddenServiceSingleHopMode = 1;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 1;
+  /* For the relay to recieve the negotiate: */
+  get_options_mutable()->ORPort_set = 1;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!client_relay3->pending_padding_callback);
+
+  // Test relay side (it should have gotten the negotiation to disable)
+  get_options_mutable()->ORPort_set = 1;
+  get_options_mutable()->HiddenServiceSingleHopMode = 0;
+  get_options_mutable()->HiddenServiceNonAnonymousMode = 0;
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+
+ done:
+  free_mock_consensus();
+  free_mock_network();
+  tor_free(relay);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+}
+
+void
+test_channelpadding_consensus(void *arg)
+{
+  channelpadding_decision_t decision;
+  or_options_t *options = get_options_mutable();
+  int64_t val;
+  int64_t new_time;
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  /*
+   * Params tested:
+   *   nf_pad_before_usage
+   *   nf_pad_relays
+   *   nf_ito_low
+   *   nf_ito_high
+   *
+   * Plan:
+   * 1. Padding can be completely disabled via consensus
+   * 2. Negotiation can't re-enable consensus-disabled padding
+   * 3. Negotiation can't increase padding from relays beyond
+   *    consensus defaults
+   * 4. Relay-to-relay padding can be enabled/disabled in consensus
+   * 5. Can enable/disable padding before actually using a connection
+   * 6. Can we control circ and TLS conn lifetime from the consensus?
+   */
+  channel_t *chan;
+  routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  timers_initialize();
+
+  connection_array = smartlist_new();
+  chan = (channel_t*)new_fake_channeltls(0);
+  channel_timestamp_active(chan);
+
+  setup_mock_consensus();
+
+  get_options_mutable()->ORPort_set = 1;
+
+  /* Test 1: Padding can be completely disabled via consensus */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=0");
+  get_options_mutable()->ConnectionPadding = 1;
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_EQ, 0);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_EQ, -2);
+
+  /* Test 2: Negotiation can't re-enable consensus-disabled padding */
+  channelpadding_send_enable_command(chan, 100, 200);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_EQ, 0);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_EQ, -2);
+  tt_assert(!chan->next_padding_time_ms);
+
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Test 3: Negotiation can't increase padding from relays beyond consensus
+   * values */
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=100");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=200");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 100);
+  tt_i64_op(val, OP_LE, 200);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 200);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+201)*NSEC_PER_MSEC;
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_clear(current_md_consensus->net_params);
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_low=1500");
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_ito_high=4500");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  channelpadding_send_enable_command(chan, 100, 200);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 1500);
+  tt_i64_op(val, OP_LE, 4500);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 4500);
+
+  /* Test 4: Relay-to-relay padding can be enabled/disabled in consensus */
+  /* Make this channel a relay's channel */
+  memcpy(relay->identity_digest,
+          ((channel_tls_t *)chan)->conn->identity_digest, DIGEST_LEN);
+  smartlist_add(current_md_consensus->routerstatus_list, relay);
+  relay = NULL; /* Prevent double-free */
+
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_relays=1");
+  channelpadding_new_consensus_params(current_md_consensus);
+
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(chan);
+  tt_i64_op(val, OP_GE, 1500);
+  tt_i64_op(val, OP_LE, 4500);
+  val = channelpadding_compute_time_until_pad_for_netflow(chan);
+  tt_i64_op(val, OP_LE, 4500);
+
+  /* Test 5: If we disable padding before channel usage, does that work? */
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_pad_before_usage=0");
+  channelpadding_new_consensus_params(current_md_consensus);
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test 6: Can we control circ and TLS conn lifetime from the consensus? */
+  val = channelpadding_get_channel_idle_timeout(NULL, 0);
+  tt_i64_op(val, OP_GE, 180);
+  tt_i64_op(val, OP_LE, 180+90);
+  val = channelpadding_get_channel_idle_timeout(chan, 0);
+  tt_i64_op(val, OP_GE, 180);
+  tt_i64_op(val, OP_LE, 180+90);
+  options->ReducedConnectionPadding = 1;
+  val = channelpadding_get_channel_idle_timeout(chan, 0);
+  tt_i64_op(val, OP_GE, 180/2);
+  tt_i64_op(val, OP_LE, (180+90)/2);
+
+  options->ReducedConnectionPadding = 0;
+  options->ORPort_set = 1;
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_conntimeout_relays=600");
+  channelpadding_new_consensus_params(current_md_consensus);
+  val = channelpadding_get_channel_idle_timeout(chan, 1);
+  tt_i64_op(val, OP_GE, 450);
+  tt_i64_op(val, OP_LE, 750);
+
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 30*60);
+  tt_i64_op(val, OP_LE, 30*60*2);
+
+  options->ReducedConnectionPadding = 1;
+  smartlist_add(current_md_consensus->net_params,
+                (void*)"nf_conntimeout_clients=600");
+  channelpadding_new_consensus_params(current_md_consensus);
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 600/2);
+  tt_i64_op(val, OP_LE, 600*2/2);
+
+  options->ReducedConnectionPadding = 0;
+  options->CircuitsAvailableTimeout = 24*60*60;
+  val = channelpadding_get_circuits_available_timeout();
+  tt_i64_op(val, OP_GE, 24*60*60);
+  tt_i64_op(val, OP_LE, 24*60*60*2);
+
+ done:
+  tor_free(relay);
+
+  free_mock_consensus();
+  free_fake_channeltls((channel_tls_t*)chan);
+  smartlist_free(connection_array);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_negotiation(void *arg)
+{
+  channelpadding_negotiate_t disable;
+  cell_t cell;
+  channelpadding_decision_t decision;
+  int val;
+  (void)arg;
+
+  /* Plan:
+   * 1. Clients reject negotiation, relays accept it.
+   *    * Bridges accept negotiation from their clients,
+   *      but not from relays.
+   * 2. Torrc options can override client-side negotiation
+   * 3. Test a version issue in channelpadidng cell
+   * 4. Test channelpadding_reduced_padding
+   */
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  timers_initialize();
+  setup_mock_consensus();
+  setup_mock_network();
+
+  /* Test case #1: Do the right things ignore negotiation? */
+  /* relay-to-client case: */
+  channelpadding_send_disable_command(relay3_client);
+  tt_assert(client_relay3->padding_enabled);
+
+  /* client-to-relay case: */
+  get_options_mutable()->ORPort_set = 1;
+  channelpadding_disable_padding_on_channel(client_relay3);
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->padding_enabled);
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  /* Bridge case from relay */
+  get_options_mutable()->BridgeRelay = 1;
+  channelpadding_disable_padding_on_channel(relay2_relay1);
+  tt_assert(relay1_relay2->padding_enabled);
+
+  /* Bridge case from client */
+  channelpadding_disable_padding_on_channel(client_relay3);
+  tt_assert(!relay3_client->padding_enabled);
+  tt_int_op(channelpadding_decide_to_pad_channel(relay3_client), OP_EQ,
+      CHANNELPADDING_WONTPAD);
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+  get_options_mutable()->BridgeRelay = 0;
+  get_options_mutable()->ORPort_set = 0;
+
+  /* Test case #2: Torrc options */
+  /* ConnectionPadding auto; Relay doesn't suport us */
+  ((channel_tls_t*)relay3_client)->conn->link_proto = 4;
+  relay3_client->padding_enabled = 0;
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!relay3_client->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  ((channel_tls_t*)relay3_client)->conn->link_proto = 5;
+  relay3_client->padding_enabled = 1;
+
+  /* ConnectionPadding 1; Relay doesn't suport us */
+  get_options_mutable()->ConnectionPadding = 1;
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!client_relay3->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  get_options_mutable()->ConnectionPadding = 0;
+
+  /* Test case #3: Test a version issue in channelpadding cell */
+  get_options_mutable()->ORPort_set = 1;
+  client_relay3->padding_enabled = 1;
+  relay3_client->padding_enabled = 1;
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&disable, 0, sizeof(channelpadding_negotiate_t));
+  cell.command = CELL_PADDING_NEGOTIATE;
+
+  channelpadding_negotiate_set_command(&disable, CHANNELPADDING_COMMAND_STOP);
+  disable.version = 1;
+  channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, &disable);
+  client_relay3->write_cell(client_relay3, &cell);
+  tt_assert(relay3_client->padding_enabled);
+  tt_int_op(channelpadding_update_padding_for_channel(client_relay3, &disable),
+          OP_EQ, -1);
+  tt_assert(client_relay3->padding_enabled);
+
+  disable.version = 0;
+  channelpadding_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, &disable);
+  client_relay3->write_cell(client_relay3, &cell);
+  tt_assert(!relay3_client->padding_enabled);
+
+  /* Test case 4: Reducing padding actually reduces it */
+  relay3_client->padding_enabled = 1;
+  client_relay3->padding_enabled = 1;
+
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+
+  channelpadding_reduce_padding_on_channel(client_relay3);
+
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(relay3_client);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+
+  get_options_mutable()->ORPort_set = 0;
+  decision = channelpadding_decide_to_pad_channel(client_relay3);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+
+  tt_assert(!client_relay3->pending_padding_callback);
+  val = channelpadding_get_netflow_inactive_timeout_ms(client_relay3);
+  tt_int_op(val, OP_GE, 9000);
+  tt_int_op(val, OP_LE, 14000);
+  int64_t val64 =
+    channelpadding_compute_time_until_pad_for_netflow(client_relay3);
+  tt_i64_op(val64, OP_LE, 14000);
+
+ done:
+  free_mock_network();
+  free_mock_consensus();
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  channel_free_all();
+
+  return;
+}
+
+void
+test_channelpadding_decide_to_pad_channel(void *arg)
+{
+  channelpadding_decision_t decision;
+  /**
+   * Test case plan:
+   *
+   * 1. Channel that has "sent a packet" before the timeout.
+   *    + We should decide to pad later
+   * 2. Channel that has not "sent a packet" before the timeout:
+   * 2a. Not within 1.1s of the timeout.
+   *    + We should decide to pad later
+   * 2b. Within 1.1s of the timemout.
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2c. Within 0.1s of the timeout.
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2d. Channel that asks to pad while timeout is scheduled
+   *    + We should schedule padding
+   *    + We should get feedback that we wrote a cell
+   * 2e. 0s of the timeout
+   *    + We should send padding immediately
+   *    + We should get feedback that we wrote a cell
+   * 2f. <0s of the timeout
+   *    + We should send padding immediately
+   *    + We should get feedback that we wrote a cell
+   * 3. Channel that sends a packet while timeout is scheduled
+   *    + We should not get feedback that we wrote a cell
+   * 4. Channel that closes while timeout is scheduled
+   *    + We should not get feedback that we wrote a cell
+   * 5. Make sure the channel still would work if repaired
+   *    + We should be able to schedule padding and resend
+   * 6. Channel is not used for full circuits
+   * 7. Channel that disappears while timeout is scheduled
+   *    + We should not send padding
+   */
+  channel_t *chan;
+  int64_t new_time;
+  connection_array = smartlist_new();
+  (void)arg;
+
+  tor_libevent_postfork();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1);
+  monotime_coarse_set_mock_time_nsec(1);
+  timers_initialize();
+  setup_full_capture_of_logs(LOG_WARN);
+  channelpadding_new_consensus_params(NULL);
+
+  chan = (channel_t*)new_fake_channeltls(0);
+  channel_timestamp_active(chan);
+
+  /* Test case #1: Channel that has "sent a packet" before the timeout. */
+  tried_to_write_cell = 0;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2a: > 1.1s until timeout */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 1200;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADLATER);
+  tt_assert(!chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2b: >= 1.0s until timeout */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 1000;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  // Wait for the timer from case #2b
+  new_time = (monotime_coarse_absolute_msec() + 1000)*NSEC_PER_MSEC;
+  monotime_set_mock_time_nsec(new_time);
+  monotime_coarse_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2c: > 0.1s until timeout */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  /* Test case #2d: Channel that asks to pad while timeout is scheduled */
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_ALREADY_SCHEDULED);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2e: 0s until timeout */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec();
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SENT);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #2f: <0s until timeout */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() - 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SENT);
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #3: Channel that sends a packet while timeout is scheduled */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Pretend the channel sent a packet
+  channel_timestamp_active(chan);
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #4: Channel that closes while a timeout is scheduled */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Pretend the channel is temporarily down
+  chan->state = CHANNEL_STATE_MAINT;
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(!chan->pending_padding_callback);
+  chan->state = CHANNEL_STATE_OPEN;
+
+  /* Test case #5: Make sure previous test case didn't break everything */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_assert(chan->pending_padding_callback);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+  // Wait for the timer
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 1);
+  tt_assert(!chan->pending_padding_callback);
+
+  /* Test case #6. Channel is not used for full circuits */
+  chan->channel_usage = CHANNEL_USED_NOT_USED_FOR_FULL_CIRCS;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_WONTPAD);
+  tt_assert(!chan->pending_padding_callback);
+  chan->channel_usage = CHANNEL_USED_FOR_FULL_CIRCS;
+
+  /* Test case #7. Channel is closed while timeout is scheduled.
+   *
+   * NOTE: This test deliberately breaks the channel callback mechanism.
+   * It must be last.
+   */
+  tried_to_write_cell = 0;
+  chan->next_padding_time_ms = monotime_coarse_absolute_msec() + 100;
+  decision = channelpadding_decide_to_pad_channel(chan);
+  tt_int_op(decision, OP_EQ, CHANNELPADDING_PADDING_SCHEDULED);
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+  tt_assert(chan->pending_padding_callback);
+
+  // Close the connection while the timer is scheduled
+  free_fake_channeltls((channel_tls_t*)chan);
+
+  // We don't expect any timer callbacks here. Make a dummy one to be sure.
+  new_time = (monotime_coarse_absolute_msec()+101)*NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(new_time);
+  monotime_set_mock_time_nsec(new_time);
+  timers_run_pending();
+
+  tt_int_op(tried_to_write_cell, OP_EQ, 0);
+
+ done:
+  smartlist_free(connection_array);
+
+  teardown_capture_of_logs();
+  monotime_disable_test_mocking();
+  timers_shutdown();
+  channel_free_all();
+
+  return;
+}
+
+#define TEST_CHANNELPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t channelpadding_tests[] = {
+  //TEST_CHANNELPADDING(channelpadding_decide_to_pad_channel, 0),
+  TEST_CHANNELPADDING(channelpadding_decide_to_pad_channel, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_negotiation, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_consensus, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_killonehop, TT_FORK),
+  TEST_CHANNELPADDING(channelpadding_timers, TT_FORK),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 08442e01b6..96c5eba9a5 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -32,6 +32,7 @@ static or_connection_t * tlschan_connection_or_connect_mock(
     const tor_addr_t *addr,
     uint16_t port,
     const char *digest,
+    const ed25519_public_key_t *ed_id,
     channel_tls_t *tlschan);
 static int tlschan_is_local_addr_mock(const tor_addr_t *addr);
 
@@ -70,7 +71,7 @@ test_channeltls_create(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
   tt_assert(ch != NULL);
 
  done:
@@ -119,7 +120,7 @@ test_channeltls_num_bytes_queued(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
   tt_assert(ch != NULL);
 
   /*
@@ -204,7 +205,7 @@ test_channeltls_overhead_estimate(void *arg)
   MOCK(connection_or_connect, tlschan_connection_or_connect_mock);
 
   /* Try connecting */
-  ch = channel_tls_connect(&test_addr, 567, test_digest);
+  ch = channel_tls_connect(&test_addr, 567, test_digest, NULL);
   tt_assert(ch != NULL);
 
   /* First case: silly low ratios should get clamped to 1.0 */
@@ -266,9 +267,11 @@ static or_connection_t *
 tlschan_connection_or_connect_mock(const tor_addr_t *addr,
                                    uint16_t port,
                                    const char *digest,
+                                   const ed25519_public_key_t *ed_id,
                                    channel_tls_t *tlschan)
 {
   or_connection_t *result = NULL;
+  (void) ed_id; // XXXX Not yet used.
 
   tt_assert(addr != NULL);
   tt_assert(port != 0);
diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c
index fbb33f87f6..38f3360b61 100644
--- a/src/test/test_checkdir.c
+++ b/src/test/test_checkdir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
new file mode 100644
index 0000000000..a5282df69d
--- /dev/null
+++ b/src/test/test_circuitbuild.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITBUILD_PRIVATE
+
+#include "or.h"
+#include "test.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+#include "config.h"
+#include "circuitbuild.h"
+
+/* Dummy nodes smartlist for testing */
+static smartlist_t dummy_nodes;
+/* Dummy exit extend_info for testing */
+static extend_info_t dummy_ei;
+
+static int
+mock_count_acceptable_nodes(smartlist_t *nodes)
+{
+  (void)nodes;
+
+  return DEFAULT_ROUTE_LEN + 1;
+}
+
+/* Test route lengths when the caller of new_route_len() doesn't
+ * specify exit_ei. */
+static void
+test_new_route_len_noexit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  r = new_route_len(CIRCUIT_PURPOSE_C_GENERAL, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  r = new_route_len(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  r = new_route_len(CIRCUIT_PURPOSE_S_CONNECT_REND, NULL, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Test route lengths where someone else chose the "exit" node, which
+ * require an extra hop for safety. */
+static void
+test_new_route_len_unsafe_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  /* connecting to hidden service directory */
+  r = new_route_len(CIRCUIT_PURPOSE_C_GENERAL, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+  /* client connecting to introduction point */
+  r = new_route_len(CIRCUIT_PURPOSE_C_INTRODUCING, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+  /* hidden service connecting to rendezvous point */
+  r = new_route_len(CIRCUIT_PURPOSE_S_CONNECT_REND, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Test route lengths where we chose the "exit" node, which don't
+ * require an extra hop for safety. */
+static void
+test_new_route_len_safe_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  /* hidden service connecting to introduction point */
+  r = new_route_len(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, &dummy_ei,
+                    &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+  /* router testing its own reachability */
+  r = new_route_len(CIRCUIT_PURPOSE_TESTING, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r);
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+/* Make sure a non-fatal assertion fails when new_route_len() gets an
+ * unexpected circuit purpose. */
+static void
+test_new_route_len_unhandled_exit(void *arg)
+{
+  int r;
+
+  (void)arg;
+  MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
+
+  tor_capture_bugs_(1);
+  setup_full_capture_of_logs(LOG_WARN);
+  r = new_route_len(CIRCUIT_PURPOSE_CONTROLLER, &dummy_ei, &dummy_nodes);
+  tt_int_op(DEFAULT_ROUTE_LEN + 1, OP_EQ, r);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(exit_ei && !known_purpose)");
+  expect_single_log_msg_containing("Unhandled purpose");
+  expect_single_log_msg_containing("with a chosen exit; assuming routelen");
+  teardown_capture_of_logs();
+  tor_end_capture_bugs_();
+
+ done:
+  UNMOCK(count_acceptable_nodes);
+}
+
+struct testcase_t circuitbuild_tests[] = {
+  { "noexit", test_new_route_len_noexit, 0, NULL, NULL },
+  { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL },
+  { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
+  { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c
index e996c42115..344ab27921 100644
--- a/src/test/test_circuitlist.c
+++ b/src/test/test_circuitlist.c
@@ -1,13 +1,15 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
+#define HS_CIRCUITMAP_PRIVATE
 #include "or.h"
 #include "channel.h"
 #include "circuitbuild.h"
 #include "circuitlist.h"
+#include "hs_circuitmap.h"
 #include "test.h"
 #include "log_test_helpers.h"
 
@@ -185,6 +187,9 @@ test_rend_token_maps(void *arg)
 
   (void)arg;
   (void)tok1; //xxxx
+
+  hs_circuitmap_init();
+
   c1 = or_circuit_new(0, NULL);
   c2 = or_circuit_new(0, NULL);
   c3 = or_circuit_new(0, NULL);
@@ -196,68 +201,68 @@ test_rend_token_maps(void *arg)
   tt_int_op(tok3[REND_TOKEN_LEN-1], OP_EQ, '.');
 
   /* No maps; nothing there. */
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok1));
 
-  circuit_set_rendezvous_cookie(c1, tok1);
-  circuit_set_intro_point_digest(c2, tok2);
+  hs_circuitmap_register_rend_circ_relay_side(c1, tok1);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c2, tok2);
 
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok3));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok3));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok2));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok1));
 
   /* Without purpose set, we don't get the circuits */
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   c1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
   c2->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
 
   /* Okay, make sure they show up now. */
-  tt_ptr_op(c1, OP_EQ, circuit_get_rendezvous(tok1));
-  tt_ptr_op(c2, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(c1, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  tt_ptr_op(c2, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   /* Two items at the same place with the same token. */
   c3->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
-  circuit_set_rendezvous_cookie(c3, tok2);
-  tt_ptr_op(c2, OP_EQ, circuit_get_intro_point(tok2));
-  tt_ptr_op(c3, OP_EQ, circuit_get_rendezvous(tok2));
+  hs_circuitmap_register_rend_circ_relay_side(c3, tok2);
+  tt_ptr_op(c2, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
 
   /* Marking a circuit makes it not get returned any more */
   circuit_mark_for_close(TO_CIRCUIT(c1), END_CIRC_REASON_FINISHED);
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok1));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
   circuit_free(TO_CIRCUIT(c1));
   c1 = NULL;
 
   /* Freeing a circuit makes it not get returned any more. */
   circuit_free(TO_CIRCUIT(c2));
   c2 = NULL;
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok2));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
 
   /* c3 -- are you still there? */
-  tt_ptr_op(c3, OP_EQ, circuit_get_rendezvous(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
   /* Change its cookie.  This never happens in Tor per se, but hey. */
   c3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
-  circuit_set_intro_point_digest(c3, tok3);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c3, tok3);
 
-  tt_ptr_op(NULL, OP_EQ, circuit_get_rendezvous(tok2));
-  tt_ptr_op(c3, OP_EQ, circuit_get_intro_point(tok3));
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+  tt_ptr_op(c3, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
   /* Now replace c3 with c4. */
   c4->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
-  circuit_set_intro_point_digest(c4, tok3);
+  hs_circuitmap_register_intro_circ_v2_relay_side(c4, tok3);
 
-  tt_ptr_op(c4, OP_EQ, circuit_get_intro_point(tok3));
+  tt_ptr_op(c4, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
-  tt_ptr_op(c3->rendinfo, OP_EQ, NULL);
-  tt_ptr_op(c4->rendinfo, OP_NE, NULL);
-  tt_mem_op(c4->rendinfo, OP_EQ, tok3, REND_TOKEN_LEN);
+  tt_ptr_op(TO_CIRCUIT(c3)->hs_token, OP_EQ, NULL);
+  tt_ptr_op(TO_CIRCUIT(c4)->hs_token, OP_NE, NULL);
+  tt_mem_op(TO_CIRCUIT(c4)->hs_token->token, OP_EQ, tok3, REND_TOKEN_LEN);
 
   /* Now clear c4's cookie. */
-  circuit_set_intro_point_digest(c4, NULL);
-  tt_ptr_op(c4->rendinfo, OP_EQ, NULL);
-  tt_ptr_op(NULL, OP_EQ, circuit_get_intro_point(tok3));
+  hs_circuitmap_remove_circuit(TO_CIRCUIT(c4));
+  tt_ptr_op(TO_CIRCUIT(c4)->hs_token, OP_EQ, NULL);
+  tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok3));
 
  done:
   if (c1)
@@ -365,10 +370,89 @@ test_pick_circid(void *arg)
   UNMOCK(channel_dump_statistics);
 }
 
+/** Test that the circuit pools of our HS circuitmap are isolated based on
+ *  their token type. */
+static void
+test_hs_circuitmap_isolation(void *arg)
+{
+  or_circuit_t *circ1 = NULL;
+  origin_circuit_t *circ2 = NULL;
+  or_circuit_t *circ3 = NULL;
+  origin_circuit_t *circ4 = NULL;
+
+  (void)arg;
+
+  hs_circuitmap_init();
+
+  {
+    const uint8_t tok1[REND_TOKEN_LEN] = "bet i got some of th";
+
+    circ1 = or_circuit_new(0, NULL);
+    tt_assert(circ1);
+    circ1->base_.purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
+
+    /* check that circuitmap is empty right? */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+
+    /* Register circ1 with tok1 as relay-side rend circ */
+    hs_circuitmap_register_rend_circ_relay_side(circ1, tok1);
+
+    /* check that service-side getters don't work */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_service_side(tok1));
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_intro_circ_v2_service_side(tok1));
+
+    /* Check that the right getter works. */
+    tt_ptr_op(circ1, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok1));
+  }
+
+  {
+    const uint8_t tok2[REND_TOKEN_LEN] = "you dont know anythi";
+
+    circ2 = origin_circuit_new();
+    tt_assert(circ2);
+    circ2->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+    circ3 = or_circuit_new(0, NULL);
+    tt_assert(circ3);
+    circ3->base_.purpose = CIRCUIT_PURPOSE_INTRO_POINT;
+    circ4 = origin_circuit_new();
+    tt_assert(circ4);
+    circ4->base_.purpose = CIRCUIT_PURPOSE_S_ESTABLISH_INTRO;
+
+    /* Register circ2 with tok2 as service-side intro v2 circ */
+    hs_circuitmap_register_intro_circ_v2_service_side(circ2, tok2);
+    /* Register circ3 with tok2 again but for different purpose */
+    hs_circuitmap_register_intro_circ_v2_relay_side(circ3, tok2);
+
+    /* Check that the getters work */
+    tt_ptr_op(circ2, OP_EQ,
+              hs_circuitmap_get_intro_circ_v2_service_side(tok2));
+    tt_ptr_op(circ3, OP_EQ, hs_circuitmap_get_intro_circ_v2_relay_side(tok2));
+
+    /* Register circ4 with tok2: it should override circ2 */
+    hs_circuitmap_register_intro_circ_v2_service_side(circ4, tok2);
+
+    /* check that relay-side getters don't work */
+    tt_ptr_op(NULL, OP_EQ, hs_circuitmap_get_rend_circ_relay_side(tok2));
+
+    /* Check that the getter returns circ4; the last circuit registered with
+     * that token. */
+    tt_ptr_op(circ4, OP_EQ,
+              hs_circuitmap_get_intro_circ_v2_service_side(tok2));
+  }
+
+ done:
+  circuit_free(TO_CIRCUIT(circ1));
+  circuit_free(TO_CIRCUIT(circ2));
+  circuit_free(TO_CIRCUIT(circ3));
+  circuit_free(TO_CIRCUIT(circ4));
+}
+
 struct testcase_t circuitlist_tests[] = {
   { "maps", test_clist_maps, TT_FORK, NULL, NULL },
   { "rend_token_maps", test_rend_token_maps, TT_FORK, NULL, NULL },
   { "pick_circid", test_pick_circid, TT_FORK, NULL, NULL },
+  { "hs_circuitmap_isolation", test_hs_circuitmap_isolation,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c
index 1ffa17247d..99abdbc685 100644
--- a/src/test/test_circuitmux.c
+++ b/src/test/test_circuitmux.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_CHANNEL_INTERNAL_
diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c
new file mode 100644
index 0000000000..5cc9fe571e
--- /dev/null
+++ b/src/test/test_circuituse.c
@@ -0,0 +1,304 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITLIST_PRIVATE
+
+#include "or.h"
+#include "test.h"
+#include "test_helpers.h"
+#include "config.h"
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "circuitbuild.h"
+#include "nodelist.h"
+
+static void
+test_circuit_is_available_for_use_ret_false_when_marked_for_close(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->marked_for_close = 1;
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_when_timestamp_dirty(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->timestamp_dirty = 1;
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_general_purpose(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_REND_POINT_WAITING;
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_general_origin(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT;
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_for_non_origin_purpose(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = tor_malloc(sizeof(circuit_t));
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    tor_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_ret_false_unusable_for_new_conns(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ));
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_returns_false_for_onehop_tunnel(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->onehop_tunnel = 1;
+
+  tt_int_op(0, ==, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static void
+test_circuit_is_available_for_use_returns_true_for_clean_circuit(void *arg)
+{
+  (void)arg;
+
+  circuit_t *circ = dummy_origin_circuit_new(30);
+  origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ);
+  oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  oc->build_state->onehop_tunnel = 0;
+
+  tt_int_op(1, ==, circuit_is_available_for_use(circ));
+
+  done:
+    circuit_free(circ);
+}
+
+static int
+mock_circuit_all_predicted_ports_handled(time_t now,
+                                         int *need_uptime,
+                                         int *need_capacity)
+{
+  (void)now;
+
+  if (need_uptime && need_capacity)
+    return 0;
+  return 1;
+}
+
+static consensus_path_type_t
+mock_router_have_unknown_consensus_path(void)
+{
+  return CONSENSUS_PATH_UNKNOWN;
+}
+
+static consensus_path_type_t
+mock_router_have_exit_consensus_path(void)
+{
+  return CONSENSUS_PATH_EXIT;
+}
+
+static void
+test_needs_exit_circuits_ret_false_for_predicted_ports_and_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 0;
+
+  time_t now = time(NULL);
+  tt_int_op(0, ==, needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+}
+
+static void
+test_needs_exit_circuits_ret_false_for_non_exit_consensus_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 1;
+  MOCK(router_have_consensus_path, mock_router_have_unknown_consensus_path);
+
+  time_t now = time(NULL);
+  tt_int_op(0, ==, needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_exit_circuits_ret_true_for_predicted_ports_and_path(void *arg)
+{
+  (void)arg;
+
+  MOCK(circuit_all_predicted_ports_handled,
+       mock_circuit_all_predicted_ports_handled);
+  int needs_uptime = 1;
+  int needs_capacity = 1;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+
+  time_t now = time(NULL);
+  tt_int_op(1, ==, needs_exit_circuits(now, &needs_uptime, &needs_capacity));
+
+  done:
+    UNMOCK(circuit_all_predicted_ports_handled);
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_circuits_for_build_ret_false_consensus_path_unknown(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_unknown_consensus_path);
+  tt_int_op(0, ==, needs_circuits_for_build(0));
+  done: ;
+}
+
+static void
+test_needs_circuits_for_build_ret_false_if_num_less_than_max(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+  tt_int_op(0, ==, needs_circuits_for_build(13));
+  done:
+    UNMOCK(router_have_consensus_path);
+}
+
+static void
+test_needs_circuits_for_build_returns_true_when_more_are_needed(void *arg)
+{
+  (void)arg;
+  MOCK(router_have_consensus_path, mock_router_have_exit_consensus_path);
+  tt_int_op(1, ==, needs_circuits_for_build(0));
+  done:
+    UNMOCK(router_have_consensus_path);
+}
+
+struct testcase_t circuituse_tests[] = {
+ { "marked",
+   test_circuit_is_available_for_use_ret_false_when_marked_for_close,
+   TT_FORK, NULL, NULL
+ },
+ { "timestamp",
+   test_circuit_is_available_for_use_ret_false_when_timestamp_dirty,
+   TT_FORK, NULL, NULL
+ },
+ { "non_general",
+   test_circuit_is_available_for_use_ret_false_for_non_general_purpose,
+   TT_FORK, NULL, NULL
+ },
+ { "non_general",
+  test_circuit_is_available_for_use_ret_false_for_non_general_origin,
+   TT_FORK, NULL, NULL
+ },
+ { "origin",
+   test_circuit_is_available_for_use_ret_false_for_non_origin_purpose,
+   TT_FORK, NULL, NULL
+ },
+ { "clean",
+   test_circuit_is_available_for_use_ret_false_unusable_for_new_conns,
+   TT_FORK, NULL, NULL
+ },
+ { "onehop",
+   test_circuit_is_available_for_use_returns_false_for_onehop_tunnel,
+   TT_FORK, NULL, NULL
+ },
+ { "clean_circ",
+   test_circuit_is_available_for_use_returns_true_for_clean_circuit,
+   TT_FORK, NULL, NULL
+ },
+ { "exit_f",
+   test_needs_exit_circuits_ret_false_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "exit_t",
+   test_needs_exit_circuits_ret_true_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "non_exit",
+   test_needs_exit_circuits_ret_false_for_non_exit_consensus_path,
+   TT_FORK, NULL, NULL
+ },
+ { "true",
+   test_needs_exit_circuits_ret_true_for_predicted_ports_and_path,
+   TT_FORK, NULL, NULL
+ },
+ { "consensus_path_unknown",
+   test_needs_circuits_for_build_ret_false_consensus_path_unknown,
+   TT_FORK, NULL, NULL
+ },
+ { "less_than_max",
+   test_needs_circuits_for_build_ret_false_if_num_less_than_max,
+   TT_FORK, NULL, NULL
+ },
+ { "more_needed",
+   test_needs_circuits_for_build_returns_true_when_more_are_needed,
+   TT_FORK, NULL, NULL
+ },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 0443cc0b1c..7dd8e65194 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define COMPAT_LIBEVENT_PRIVATE
diff --git a/src/test/test_config.c b/src/test/test_config.c
index 89f9b3e2aa..beaab00f73 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,6 +11,7 @@
 #include "or.h"
 #include "address.h"
 #include "addressmap.h"
+#include "bridges.h"
 #include "circuitmux_ewma.h"
 #include "circuitbuild.h"
 #include "config.h"
@@ -45,6 +46,8 @@
 #include "transports.h"
 #include "util.h"
 
+#include "test_helpers.h"
+
 static void
 test_config_addressmap(void *arg)
 {
@@ -851,9 +854,23 @@ static void
 test_config_fix_my_family(void *arg)
 {
   char *err = NULL;
-  const char *family = "$1111111111111111111111111111111111111111, "
-                       "1111111111111111111111111111111111111112, "
-                       "$1111111111111111111111111111111111111113";
+  config_line_t *family = tor_malloc_zero(sizeof(config_line_t));
+  family->key = tor_strdup("MyFamily");
+  family->value = tor_strdup("$1111111111111111111111111111111111111111, "
+                             "1111111111111111111111111111111111111112, "
+                             "$1111111111111111111111111111111111111113");
+
+  config_line_t *family2 = tor_malloc_zero(sizeof(config_line_t));
+  family2->key = tor_strdup("MyFamily");
+  family2->value = tor_strdup("1111111111111111111111111111111111111114");
+
+  config_line_t *family3 = tor_malloc_zero(sizeof(config_line_t));
+  family3->key = tor_strdup("MyFamily");
+  family3->value = tor_strdup("$1111111111111111111111111111111111111115");
+
+  family->next = family2;
+  family2->next = family3;
+  family3->next = NULL;
 
   or_options_t* options = options_new();
   or_options_t* defaults = options_new();
@@ -861,7 +878,7 @@ test_config_fix_my_family(void *arg)
 
   options_init(options);
   options_init(defaults);
-  options->MyFamily = tor_strdup(family);
+  options->MyFamily_lines = family;
 
   options_validate(NULL, options, defaults, 0, &err) ;
 
@@ -869,18 +886,23 @@ test_config_fix_my_family(void *arg)
     TT_FAIL(("options_validate failed: %s", err));
   }
 
-  tt_str_op(options->MyFamily,OP_EQ,
-                                "$1111111111111111111111111111111111111111, "
-                                "$1111111111111111111111111111111111111112, "
-                                "$1111111111111111111111111111111111111113");
-
-  done:
-    if (err != NULL) {
-      tor_free(err);
-    }
+  const char *valid[] = { "$1111111111111111111111111111111111111111",
+                          "$1111111111111111111111111111111111111112",
+                          "$1111111111111111111111111111111111111113",
+                          "$1111111111111111111111111111111111111114",
+                          "$1111111111111111111111111111111111111115" };
+  int ret_size = 0;
+  config_line_t *ret;
+  for (ret = options->MyFamily; ret && ret_size < 5; ret = ret->next) {
+    tt_str_op(ret->value, OP_EQ, valid[ret_size]);
+    ret_size++;
+  }
+  tt_int_op(ret_size, OP_EQ, 5);
 
-    or_options_free(options);
-    or_options_free(defaults);
+ done:
+  tor_free(err);
+  or_options_free(options);
+  or_options_free(defaults);
 }
 
 static int n_hostname_01010101 = 0;
@@ -3861,144 +3883,6 @@ mock_config_line(const char *key, const char *val)
 }
 
 static void
-test_config_parse_port_config__listenaddress(void *data)
-{
-  (void)data;
-  int ret;
-  config_line_t *config_listen_address = NULL, *config_listen_address2 = NULL,
-    *config_listen_address3 = NULL;
-  config_line_t *config_port1 = NULL, *config_port2 = NULL,
-    *config_port3 = NULL, *config_port4 = NULL, *config_port5 = NULL;
-  smartlist_t *slout = NULL;
-  port_cfg_t *port_cfg = NULL;
-
-  // Test basic invocation with no arguments
-  ret = parse_port_config(NULL, NULL, NULL, NULL, 0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Setup some test data
-  config_listen_address = mock_config_line("DNSListenAddress", "127.0.0.1");
-  config_listen_address2 = mock_config_line("DNSListenAddress", "x$$$:::345");
-  config_listen_address3 = mock_config_line("DNSListenAddress",
-                                            "127.0.0.1:1442");
-  config_port1 = mock_config_line("DNSPort", "42");
-  config_port2 = mock_config_line("DNSPort", "43");
-  config_port1->next = config_port2;
-  config_port3 = mock_config_line("DNSPort", "auto");
-  config_port4 = mock_config_line("DNSPort", "55542");
-  config_port5 = mock_config_line("DNSPort", "666777");
-
-  // Test failure when we have a ListenAddress line and several
-  // Port lines for the same portname
-  ret = parse_port_config(NULL, config_port1, config_listen_address, "DNS", 0,
-                          NULL, 0, 0);
-
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test case when we have a listen address, no default port and allow
-  // spurious listen address lines
-  ret = parse_port_config(NULL, NULL, config_listen_address, "DNS", 0, NULL,
-                          0, CL_PORT_ALLOW_EXTRA_LISTENADDR);
-  tt_int_op(ret, OP_EQ, 1);
-
-  // Test case when we have a listen address, no default port but doesn't
-  // allow spurious listen address lines
-  ret = parse_port_config(NULL, NULL, config_listen_address, "DNS", 0, NULL,
-                          0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test case when we have a listen address, and a port that points to auto,
-  // should use the AUTO port
-  slout = smartlist_new();
-  ret = parse_port_config(slout, config_port3, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 1);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
-  tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
-
-  // Test when we have a listen address and a custom port
-  ret = parse_port_config(slout, config_port4, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 2);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 1);
-  tt_int_op(port_cfg->port, OP_EQ, 55542);
-
-  // Test when we have a listen address and an invalid custom port
-  ret = parse_port_config(slout, config_port5, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test we get a server port configuration when asked for it
-  ret = parse_port_config(slout, NULL, config_listen_address, "DNS", 0, NULL,
-                          123, CL_PORT_SERVER_OPTIONS);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 4);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 2);
-  tt_int_op(port_cfg->port, OP_EQ, 123);
-  tt_int_op(port_cfg->server_cfg.no_listen, OP_EQ, 1);
-  tt_int_op(port_cfg->server_cfg.bind_ipv4_only, OP_EQ, 1);
-
-  // Test an invalid ListenAddress configuration
-  ret = parse_port_config(NULL, NULL, config_listen_address2, "DNS", 0, NULL,
-                          222, 0);
-  tt_int_op(ret, OP_EQ, -1);
-
-  // Test default to the port in the listen address if available
-  ret = parse_port_config(slout, config_port2, config_listen_address3, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(smartlist_len(slout), OP_EQ, 5);
-  port_cfg = (port_cfg_t *)smartlist_get(slout, 4);
-  tt_int_op(port_cfg->port, OP_EQ, 1442);
-
-  // Test we work correctly without an out, but with a listen address
-  // and a port
-  ret = parse_port_config(NULL, config_port2, config_listen_address, "DNS",
-                          0, NULL, 0, 0);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal control
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_CONTROL_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal ext or listener
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_EXT_OR_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal other
-  SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
-  smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port2, config_listen_address, "DNS",
-                          0, NULL, 0, CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
-  // Test warning nonlocal control without an out
-  ret = parse_port_config(NULL, config_port2, config_listen_address, "DNS",
-                          CONN_TYPE_CONTROL_LISTENER, NULL, 0,
-                          CL_PORT_WARN_NONLOCAL);
-  tt_int_op(ret, OP_EQ, 0);
-
- done:
-  config_free_lines(config_listen_address);
-  config_free_lines(config_listen_address2);
-  config_free_lines(config_listen_address3);
-  config_free_lines(config_port1);
-  /* 2 was linked from 1. */
-  config_free_lines(config_port3);
-  config_free_lines(config_port4);
-  config_free_lines(config_port5);
-  if (slout)
-    SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
-  smartlist_free(slout);
-}
-
-static void
 test_config_parse_port_config__ports__no_ports_given(void *data)
 {
   (void)data;
@@ -4009,40 +3893,40 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   slout = smartlist_new();
 
   // Test no defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and with out
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and out, adds a new port cfg
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
@@ -4053,7 +3937,7 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   // for a unix address
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, NULL, "DNS", 0, "/foo/bar/unixdomain",
+  ret = parse_port_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
                           42, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4082,28 +3966,28 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters an empty unix domain specification
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "unix:");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters a unix domain specification but the listener
   // doesn't support domain sockets
   config_port_valid = mock_config_line("DNSPort", "unix:/tmp/foo/bar");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS",
+  ret = parse_port_config(NULL, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test valid unix domain
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0, 0);
 #ifdef _WIN32
   tt_int_op(ret, OP_EQ, -1);
@@ -4126,8 +4010,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("SOCKSPort",
                                          "unix:/tmp/foo/bar NoIPv4Traffic "
+                                         "NoIPv6Traffic "
                                          "NoOnionTraffic");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "SOCKS",
+  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4136,7 +4021,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.1:80 NoDNSRequest");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS",
+  ret = parse_port_config(NULL, config_port_invalid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4147,8 +4032,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.1:80 "
+                                       "NoIPv6Traffic "
                                        "NoIPv4Traffic NoOnionTraffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS",
+  ret = parse_port_config(slout, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4162,8 +4048,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure if we have DNS but no ipv4 and no ipv6
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("SOCKSPort",
+                                         "NoIPv6Traffic "
                                          "unix:/tmp/foo/bar NoIPv4Traffic");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "SOCKS",
+  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4174,8 +4061,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4195,8 +4083,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar\" "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4216,8 +4105,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4227,8 +4117,9 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"\" "
+                                       "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4239,7 +4130,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "OnionTrafficOnly");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4260,7 +4151,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4279,7 +4170,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "IPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4295,28 +4186,28 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure if we specify world writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 WorldWritable");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 GroupWritable");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 RelaxDirModeCheck");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with only a port (this will fail without a default address)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4325,7 +4216,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestPort");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4338,7 +4229,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateDestPorts");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4351,7 +4242,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestAddr");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4364,7 +4255,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4377,7 +4268,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientProtocol");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4390,7 +4281,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientAddr");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4401,7 +4292,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with ignored unknown options
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42 ThisOptionDoesntExist");
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4410,7 +4301,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4423,7 +4314,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort",
                                        "42 IPv6Traffic PreferIPv6");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.42", 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4436,7 +4327,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4449,7 +4340,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv6DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4462,7 +4353,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoCacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4475,7 +4366,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheDNS");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4488,7 +4379,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv4Cache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4501,7 +4392,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv6Cache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4514,7 +4405,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseDNSCache");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4527,7 +4418,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoPreferIPv6Automap");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4539,7 +4430,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 PreferSOCKSNoAuth");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4554,14 +4445,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("DNSPort", "0");
   config_port_valid = mock_config_line("DNSPort", "42");
   config_port_invalid->next = config_port_valid;
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with warn non-local control
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "Control",
+  ret = parse_port_config(slout, config_port_valid, "Control",
                           CONN_TYPE_CONTROL_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4569,7 +4460,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local listener
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "ExtOR",
+  ret = parse_port_config(slout, config_port_valid, "ExtOR",
                           CONN_TYPE_EXT_OR_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4577,12 +4468,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local other
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test success with warn non-local other without out
-  ret = parse_port_config(NULL, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4593,7 +4484,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IPv4Traffic "
                                        "IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0,
                           CL_PORT_TAKES_HOSTNAMES |
                           CL_PORT_NO_STREAM_OPTIONS);
@@ -4601,14 +4492,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
-  tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 0);
+  tt_int_op(port_cfg->entry_cfg.ipv6_traffic, OP_EQ, 1);
 
   // Test failure for a SessionGroup argument with invalid value
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=invalid");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4620,7 +4511,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4630,7 +4521,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123 "
                                          "SessionGroup=321");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4639,7 +4530,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 SessionGroup=1111122");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4651,7 +4542,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "0");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
@@ -4661,7 +4552,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "something");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4674,7 +4565,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "auto");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4688,7 +4579,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.122:auto");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4700,8 +4591,10 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure when asked to parse an invalid address followed by auto
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "invalidstuff!!:auto");
-  ret = parse_port_config(NULL, config_port_invalid, NULL, "DNS", 0,
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
+  UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with parsing both an address and a real port
@@ -4709,7 +4602,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.123:656");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4723,7 +4616,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "something wrong");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4732,7 +4625,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.1.0:123:auto");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4742,7 +4635,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/somewhere");
-  ret = parse_port_config(slout, config_port_valid, NULL, "SOCKS",
+  ret = parse_port_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.46", 0,
                           CL_PORT_DFLT_GROUP_WRITABLE);
 #ifdef _WIN32
@@ -4777,7 +4670,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort",
                                        "127.0.0.124:656 NoAdvertise");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4790,7 +4683,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4804,7 +4697,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen "
                                          "NoAdvertise");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4813,7 +4706,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4826,7 +4719,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "[::1]:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4840,7 +4733,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 IPv6Only "
                                          "IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4849,7 +4742,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 unknown");
-  ret = parse_port_config(slout, config_port_valid, NULL, "DNS", 0, NULL, 0,
+  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4860,7 +4753,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.124:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4869,7 +4762,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "[::1]:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "DNS", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4878,7 +4771,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("ORPort", "unix:\"\"");
-  ret = parse_port_config(slout, config_port_invalid, NULL, "ORPort", 0, NULL,
+  ret = parse_port_config(slout, config_port_invalid, "ORPort", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4890,6 +4783,580 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
 }
 
+static void
+test_config_parse_log_severity(void *data)
+{
+  int ret;
+  const char *severity_log_lines[] = {
+    "debug file /tmp/debug.log",
+    "debug\tfile /tmp/debug.log",
+    "[handshake]debug [~net,~mm]info notice stdout",
+    "[handshake]debug\t[~net,~mm]info\tnotice\tstdout",
+    NULL
+  };
+  int i;
+  log_severity_list_t *severity;
+
+  (void) data;
+
+  severity = tor_malloc(sizeof(log_severity_list_t));
+  for (i = 0; severity_log_lines[i]; i++) {
+    memset(severity, 0, sizeof(log_severity_list_t));
+    ret = parse_log_severity_config(&severity_log_lines[i], severity);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  tor_free(severity);
+}
+
+static void
+test_config_include_limit(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_limit"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrc_path[PATH_MAX+1];
+  tor_snprintf(torrc_path, sizeof(torrc_path), "%s"PATH_SEPARATOR"torrc", dir);
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               torrc_path);
+  tt_int_op(write_str_to_file(torrc_path, torrc_contents, 0), OP_EQ, 0);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_does_not_exist(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_does_not_exist"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char missing_path[PATH_MAX+1];
+  tor_snprintf(missing_path, sizeof(missing_path), "%s"PATH_SEPARATOR"missing",
+               dir);
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               missing_path);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_error_in_included_file(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *dir = tor_strdup(get_fname("test_error_in_included_file"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char invalid_path[PATH_MAX+1];
+  tor_snprintf(invalid_path, sizeof(invalid_path), "%s"PATH_SEPARATOR"invalid",
+               dir);
+  tt_int_op(write_str_to_file(invalid_path, "unclosed \"", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s",
+               invalid_path);
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, NULL),
+            OP_EQ, -1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_empty_file_folder(void *data)
+{
+  (void)data;
+  config_line_t *result = NULL;
+
+  char *dir = tor_strdup(get_fname("test_include_empty_file_folder"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char folder_path[PATH_MAX+1];
+  tor_snprintf(folder_path, sizeof(folder_path), "%s"PATH_SEPARATOR"empty_dir",
+               dir);
+#ifdef _WIN32
+  tt_int_op(mkdir(folder_path), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(folder_path, 0700), OP_EQ, 0);
+#endif
+  char file_path[PATH_MAX+1];
+  tor_snprintf(file_path, sizeof(file_path), "%s"PATH_SEPARATOR"empty_file",
+               dir);
+  tt_int_op(write_str_to_file(file_path, "", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n"
+               "%%include %s\n",
+               folder_path, file_path);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used),
+            OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_recursion_before_after(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_recursion_before_after"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrc_path[PATH_MAX+1];
+  tor_snprintf(torrc_path, sizeof(torrc_path), "%s"PATH_SEPARATOR"torrc", dir);
+
+  char file_contents[1000];
+  const int limit = MAX_INCLUDE_RECURSION_LEVEL;
+  int i;
+  // Loop backwards so file_contents has the contents of the first file by the
+  // end of the loop
+  for (i = limit; i > 0; i--) {
+    if (i < limit) {
+      tor_snprintf(file_contents, sizeof(file_contents),
+                   "Test %d\n"
+                   "%%include %s%d\n"
+                   "Test %d\n",
+                   i, torrc_path, i + 1, 2 * limit - i);
+    } else {
+      tor_snprintf(file_contents, sizeof(file_contents), "Test %d\n", i);
+    }
+
+    if (i > 1) {
+      char file_path[PATH_MAX+1];
+      tor_snprintf(file_path, sizeof(file_path), "%s%d", torrc_path, i);
+      tt_int_op(write_str_to_file(file_path, file_contents, 0), OP_EQ, 0);
+    }
+  }
+
+  int include_used;
+  tt_int_op(config_get_lines_include(file_contents, &result, 0, &include_used),
+            OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 2 * limit - 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_recursion_after_only(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_recursion_after_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrc_path[PATH_MAX+1];
+  tor_snprintf(torrc_path, sizeof(torrc_path), "%s"PATH_SEPARATOR"torrc", dir);
+
+  char file_contents[1000];
+  const int limit = MAX_INCLUDE_RECURSION_LEVEL;
+  int i;
+  // Loop backwards so file_contents has the contents of the first file by the
+  // end of the loop
+  for (i = limit; i > 0; i--) {
+    int n = (i - limit - 1) * -1;
+    if (i < limit) {
+      tor_snprintf(file_contents, sizeof(file_contents),
+                   "%%include %s%d\n"
+                   "Test %d\n",
+                   torrc_path, i + 1, n);
+    } else {
+      tor_snprintf(file_contents, sizeof(file_contents), "Test %d\n", n);
+    }
+
+    if (i > 1) {
+      char file_path[PATH_MAX+1];
+      tor_snprintf(file_path, sizeof(file_path), "%s%d", torrc_path, i);
+      tt_int_op(write_str_to_file(file_path, file_contents, 0), OP_EQ, 0);
+    }
+  }
+
+  int include_used;
+  tt_int_op(config_get_lines_include(file_contents, &result, 0, &include_used),
+            OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, limit);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_folder_order(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_folder_order"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrcd[PATH_MAX+1];
+  tor_snprintf(torrcd, sizeof(torrcd), "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  // test that files in subfolders are ignored
+  char path[PATH_MAX+1];
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", torrcd,
+               "subfolder");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(path), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(path, 0700), OP_EQ, 0);
+#endif
+
+  char path2[PATH_MAX+1];
+  tor_snprintf(path2, sizeof(path2), "%s"PATH_SEPARATOR"%s", path,
+               "01_ignore");
+  tt_int_op(write_str_to_file(path2, "ShouldNotSee 1\n", 0), OP_EQ, 0);
+
+  // test that files starting with . are ignored
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", torrcd, ".dot");
+  tt_int_op(write_str_to_file(path, "ShouldNotSee 2\n", 0), OP_EQ, 0);
+
+  // test file order
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", torrcd, "01_1st");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", torrcd, "02_2nd");
+  tt_int_op(write_str_to_file(path, "Test 2\n", 0), OP_EQ, 0);
+
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", torrcd, "aa_3rd");
+  tt_int_op(write_str_to_file(path, "Test 3\n", 0), OP_EQ, 0);
+
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", torrcd, "ab_4th");
+  tt_int_op(write_str_to_file(path, "Test 4\n", 0), OP_EQ, 0);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used),
+            OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 4);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_path_syntax(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_path_syntax"));
+  char *esc_dir = NULL, *dir_with_pathsep = NULL,
+    *esc_dir_with_pathsep = NULL, *torrc_contents = NULL;
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  esc_dir = esc_for_log(dir);
+  tor_asprintf(&dir_with_pathsep, "%s%s", dir, PATH_SEPARATOR);
+  esc_dir_with_pathsep = esc_for_log(dir_with_pathsep);
+
+  tor_asprintf(&torrc_contents,
+               "%%include %s\n"
+               "%%include %s%s \n" // space to avoid suppressing newline
+               "%%include %s\n",
+               esc_dir,
+               dir, PATH_SEPARATOR,
+               esc_dir_with_pathsep);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used),
+            OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+  tor_free(torrc_contents);
+  tor_free(esc_dir);
+  tor_free(dir_with_pathsep);
+  tor_free(esc_dir_with_pathsep);
+}
+
+static void
+test_config_include_not_processed(void *data)
+{
+  (void)data;
+
+  char torrc_contents[1000] = "%include does_not_exist\n";
+  config_line_t *result = NULL;
+  tt_int_op(config_get_lines(torrc_contents, &result, 0),OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    tt_str_op(next->key, OP_EQ, "%include");
+    tt_str_op(next->value, OP_EQ, "does_not_exist");
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+}
+
+static void
+test_config_include_has_include(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_has_include"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char torrc_contents[1000] = "Test 1\n";
+  int include_used;
+
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used),
+            OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 0);
+  config_free_lines(result);
+
+  tor_snprintf(torrc_contents, sizeof(torrc_contents), "%%include %s\n", dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0,&include_used),
+            OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 1);
+
+ done:
+  config_free_lines(result);
+  tor_free(dir);
+}
+
+static void
+test_config_include_flag_both_without(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  // test with defaults-torrc and torrc without include
+  int ret = options_init_from_string(conf_empty, conf_empty, CMD_RUN_UNITTESTS,
+                                     NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 0);
+
+ done:
+  tor_free(errmsg);
+}
+
+static void
+test_config_include_flag_torrc_only(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char *dir = tor_strdup(get_fname("test_include_flag_torrc_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char path[PATH_MAX+1];
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", dir, "dummy");
+  tt_int_op(write_str_to_file(path, "\n", 0), OP_EQ, 0);
+
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  char conf_include[1000];
+  tor_snprintf(conf_include, sizeof(conf_include), "%%include %s", path);
+
+  // test with defaults-torrc without include and torrc with include
+  int ret = options_init_from_string(conf_empty, conf_include,
+                                     CMD_RUN_UNITTESTS, NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 1);
+
+ done:
+  tor_free(errmsg);
+  tor_free(dir);
+}
+
+static void
+test_config_include_flag_defaults_only(void *data)
+{
+  (void)data;
+
+  char *errmsg = NULL;
+  char *dir = tor_strdup(get_fname("test_include_flag_defaults_only"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  char path[PATH_MAX+1];
+  tor_snprintf(path, sizeof(path), "%s"PATH_SEPARATOR"%s", dir, "dummy");
+  tt_int_op(write_str_to_file(path, "\n", 0), OP_EQ, 0);
+
+  char conf_empty[1000];
+  tor_snprintf(conf_empty, sizeof(conf_empty),
+               "DataDirectory %s\n",
+               get_fname(NULL));
+  char conf_include[1000];
+  tor_snprintf(conf_include, sizeof(conf_include), "%%include %s", path);
+
+  // test with defaults-torrc with include and torrc without include
+  int ret = options_init_from_string(conf_include, conf_empty,
+                                     CMD_RUN_UNITTESTS, NULL, &errmsg);
+  tt_int_op(ret, OP_EQ, 0);
+
+  const or_options_t *options = get_options();
+  tt_int_op(options->IncludeUsed, OP_EQ, 0);
+
+ done:
+  tor_free(errmsg);
+  tor_free(dir);
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -4912,10 +5379,23 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(fix_my_family, 0),
   CONFIG_TEST(directory_fetch, 0),
   CONFIG_TEST(port_cfg_line_extract_addrport, 0),
-  CONFIG_TEST(parse_port_config__listenaddress, 0),
   CONFIG_TEST(parse_port_config__ports__no_ports_given, 0),
   CONFIG_TEST(parse_port_config__ports__server_options, 0),
   CONFIG_TEST(parse_port_config__ports__ports_given, 0),
+  CONFIG_TEST(parse_log_severity, 0),
+  CONFIG_TEST(include_limit, 0),
+  CONFIG_TEST(include_does_not_exist, 0),
+  CONFIG_TEST(include_error_in_included_file, 0),
+  CONFIG_TEST(include_empty_file_folder, 0),
+  CONFIG_TEST(include_recursion_before_after, 0),
+  CONFIG_TEST(include_recursion_after_only, 0),
+  CONFIG_TEST(include_folder_order, 0),
+  CONFIG_TEST(include_path_syntax, 0),
+  CONFIG_TEST(include_not_processed, 0),
+  CONFIG_TEST(include_has_include, 0),
+  CONFIG_TEST(include_flag_both_without, TT_FORK),
+  CONFIG_TEST(include_flag_torrc_only, TT_FORK),
+  CONFIG_TEST(include_flag_defaults_only, TT_FORK),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_connection.c b/src/test/test_connection.c
index d394fc9852..7e5193b203 100644
--- a/src/test/test_connection.c
+++ b/src/test/test_connection.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -10,6 +10,7 @@
 #include "test.h"
 
 #include "connection.h"
+#include "hs_common.h"
 #include "main.h"
 #include "microdesc.h"
 #include "networkstatus.h"
@@ -264,14 +265,10 @@ test_conn_get_rend_setup(const struct testcase_t *tc)
 
   rend_cache_init();
 
-  /* TODO: use directory_initiate_command_rend() to do this - maybe? */
-  conn->rend_data = tor_malloc_zero(sizeof(rend_data_t));
+  /* TODO: use directory_initiate_request() to do this - maybe? */
   tor_assert(strlen(TEST_CONN_REND_ADDR) == REND_SERVICE_ID_LEN_BASE32);
-  memcpy(conn->rend_data->onion_address,
-         TEST_CONN_REND_ADDR,
-         REND_SERVICE_ID_LEN_BASE32+1);
-  conn->rend_data->hsdirs_fp = smartlist_new();
-
+  conn->rend_data = rend_data_client_create(TEST_CONN_REND_ADDR, NULL, NULL,
+                                            REND_NO_AUTH);
   assert_connection_ok(&conn->base_, time(NULL));
   return conn;
 
@@ -551,7 +548,8 @@ test_conn_get_rend(void *arg)
   tt_assert(connection_get_by_type_state_rendquery(
                                             conn->base_.type,
                                             conn->base_.state,
-                                            conn->rend_data->onion_address)
+                                            rend_data_get_address(
+                                                      conn->rend_data))
             == TO_CONN(conn));
   tt_assert(connection_get_by_type_state_rendquery(
                                             TEST_CONN_TYPE,
diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c
new file mode 100644
index 0000000000..aee1ba8a06
--- /dev/null
+++ b/src/test/test_conscache.c
@@ -0,0 +1,340 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_conscache_open_failure(void *arg)
+{
+  (void) arg;
+  /* Try opening a directory that doesn't exist and which we shouldn't be
+   * able to create. */
+  consensus_cache_t *cache = consensus_cache_open("a/b/c/d/e/f/g", 128);
+  tt_ptr_op(cache, OP_EQ, NULL);
+
+ done:
+  ;
+}
+
+static void
+test_conscache_simple_usage(void *arg)
+{
+  (void)arg;
+  consensus_cache_entry_t *ent = NULL, *ent2 = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create object; make sure it exists. */
+  config_line_t *labels = NULL;
+  config_line_append(&labels, "Hello", "world");
+  config_line_append(&labels, "Adios", "planetas");
+  ent = consensus_cache_add(cache,
+                            labels, (const uint8_t *)"A\0B\0C", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent);
+
+  /* Make a second object */
+  config_line_append(&labels, "Hello", "mundo");
+  config_line_append(&labels, "Adios", "planets");
+  ent2 = consensus_cache_add(cache,
+                             labels, (const uint8_t *)"xyzzy", 5);
+  config_free_lines(labels);
+  labels = NULL;
+  tt_assert(ent2);
+  tt_assert(! consensus_cache_entry_is_mapped(ent2));
+  consensus_cache_entry_decref(ent2);
+  ent2 = NULL;
+
+  /* Check get_value */
+  tt_ptr_op(NULL, OP_EQ, consensus_cache_entry_get_value(ent, "hebbo"));
+  tt_str_op("world", OP_EQ, consensus_cache_entry_get_value(ent, "Hello"));
+
+  /* Check find_first */
+  ent2 = consensus_cache_find_first(cache, "Hello", "world!");
+  tt_ptr_op(ent2, OP_EQ, NULL);
+  ent2 = consensus_cache_find_first(cache, "Hello", "world");
+  tt_ptr_op(ent2, OP_EQ, ent);
+  ent2 = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_ptr_op(ent2, OP_NE, ent);
+
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+
+  /* Check get_body */
+  const uint8_t *bp = NULL;
+  size_t sz = 0;
+  int r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "A\0B\0C", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* Free and re-create the cache, to rescan the directory. */
+  consensus_cache_free(cache);
+  consensus_cache_entry_decref(ent);
+  cache = consensus_cache_open("cons", 128);
+
+  /* Make sure the entry is still there */
+  ent = consensus_cache_find_first(cache, "Hello", "mundo");
+  tt_assert(ent);
+  ent2 = consensus_cache_find_first(cache, "Adios", "planets");
+  tt_ptr_op(ent, OP_EQ, ent2);
+  consensus_cache_entry_incref(ent);
+  tt_assert(! consensus_cache_entry_is_mapped(ent));
+  r = consensus_cache_entry_get_body(ent, &bp, &sz);
+  tt_int_op(r, OP_EQ, 0);
+  tt_u64_op(sz, OP_EQ, 5);
+  tt_mem_op(bp, OP_EQ, "xyzzy", 5);
+  tt_assert(consensus_cache_entry_is_mapped(ent));
+
+  /* There should be two entries total. */
+  smartlist_t *entries = smartlist_new();
+  consensus_cache_find_all(entries, cache, NULL, NULL);
+  int n = smartlist_len(entries);
+  smartlist_free(entries);
+  tt_int_op(n, OP_EQ, 2);
+
+ done:
+  consensus_cache_entry_decref(ent);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_cleanup(void *arg)
+{
+  (void)arg;
+  const int N = 20;
+  consensus_cache_entry_t **ents =
+    tor_calloc(N, sizeof(consensus_cache_entry_t*));
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries. */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "cleanup");
+    config_line_append(&labels, "index", num);
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    ents[i] = consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ents[i]);
+    /* We're still holding a reference to each entry at this point. */
+  }
+
+  /* Page all of the entries into RAM */
+  for (i = 0; i < N; ++i) {
+    const uint8_t *bp;
+    size_t sz;
+    tt_assert(! consensus_cache_entry_is_mapped(ents[i]));
+    consensus_cache_entry_get_body(ents[i], &bp, &sz);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as deletable. */
+  for (i = 7; i < N; i += 7) {
+    consensus_cache_entry_mark_for_removal(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Mark some of the entries as aggressively unpaged. */
+  for (i = 3; i < N; i += 3) {
+    consensus_cache_entry_mark_for_aggressive_release(ents[i]);
+    tt_assert(consensus_cache_entry_is_mapped(ents[i]));
+  }
+
+  /* Incref some of the entries again */
+  for (i = 0; i < N; i += 2) {
+    consensus_cache_entry_incref(ents[i]);
+  }
+
+  /* Now we're going to decref everything. We do so at a specific time.  I'm
+   * picking the moment when I was writing this test, at 2017-04-05 12:16:48
+   * UTC. */
+  const time_t example_time = 1491394608;
+  update_approx_time(example_time);
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    if (i % 2) {
+      ents[i] = NULL; /* We're no longer holding any reference here. */
+    }
+  }
+
+  /* At this point, the aggressively-released items with refcount 1 should
+   * be unmapped. Nothing should be deleted. */
+  consensus_cache_entry_t *e_tmp;
+  e_tmp = consensus_cache_find_first(cache, "index", "3");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "5");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "6");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  e_tmp = consensus_cache_find_first(cache, "index", "7");
+  tt_assert(e_tmp == NULL); // not found because pending deletion.
+
+  /* Delete the pending-deletion items. */
+  consensus_cache_delete_pending(cache, 0);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 20 - 2); /* 1 entry was deleted; 1 is not-found. */
+  }
+  e_tmp = consensus_cache_find_first(cache, "index", "7"); // refcnt == 1...
+  tt_assert(e_tmp == NULL); // so deleted.
+  e_tmp = consensus_cache_find_first(cache, "index", "14"); // refcnt == 2
+  tt_assert(e_tmp == NULL); // not deleted; but not found.
+
+  /* Now do lazy unmapping. */
+  // should do nothing.
+  consensus_cache_unmap_lazy(cache, example_time - 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+  // should actually unmap
+  consensus_cache_unmap_lazy(cache, example_time + 10);
+  e_tmp = consensus_cache_find_first(cache, "index", "11");
+  tt_assert(e_tmp);
+  tt_assert(! consensus_cache_entry_is_mapped(e_tmp));
+  // This one will still be mapped, since it has a reference.
+  e_tmp = consensus_cache_find_first(cache, "index", "16");
+  tt_assert(e_tmp);
+  tt_assert(consensus_cache_entry_is_mapped(e_tmp));
+
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+    ents[i] = NULL;
+  }
+
+  /* Free and re-create the cache, to rescan the directory. Make sure the
+   * deleted thing is still deleted, along with the other deleted thing. */
+  consensus_cache_free(cache);
+  cache = consensus_cache_open("cons", 128);
+  {
+    smartlist_t *entries = smartlist_new();
+    consensus_cache_find_all(entries, cache, NULL, NULL);
+    int n = smartlist_len(entries);
+    smartlist_free(entries);
+    tt_int_op(n, OP_EQ, 18);
+  }
+
+ done:
+  for (i = 0; i < N; ++i) {
+    consensus_cache_entry_decref(ents[i]);
+  }
+  tor_free(ents);
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+}
+
+static void
+test_conscache_filter(void *arg)
+{
+  (void)arg;
+  const int N = 30;
+  smartlist_t *lst = NULL;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cache"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+  consensus_cache_t *cache = consensus_cache_open("cons", 128);
+
+  tt_assert(cache);
+
+  /* Create a bunch of entries with different labels */
+  int i;
+  for (i = 0; i < N; ++i) {
+    config_line_t *labels = NULL;
+    char num[8];
+    tor_snprintf(num, sizeof(num), "%d", i);
+    config_line_append(&labels, "test-id", "filter");
+    config_line_append(&labels, "index", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 3);
+    config_line_append(&labels, "mod3", num);
+    tor_snprintf(num, sizeof(num), "%d", i % 5);
+    config_line_append(&labels, "mod5", num);
+
+    size_t bodylen = i * 3;
+    uint8_t *body = tor_malloc(bodylen);
+    memset(body, i, bodylen);
+    consensus_cache_entry_t *ent =
+      consensus_cache_add(cache, labels, body, bodylen);
+    tor_free(body);
+    config_free_lines(labels);
+    tt_assert(ent);
+    consensus_cache_entry_decref(ent);
+  }
+
+  lst = smartlist_new();
+  /* Find nothing. */
+  consensus_cache_find_all(lst, cache, "mod5", "5");
+  tt_int_op(smartlist_len(lst), OP_EQ, 0);
+  /* Find everything. */
+  consensus_cache_find_all(lst, cache, "test-id", "filter");
+  tt_int_op(smartlist_len(lst), OP_EQ, N);
+
+  /* Now filter to find the entries that have i%3 == 1 */
+  consensus_cache_filter_list(lst, "mod3", "1");
+  tt_int_op(smartlist_len(lst), OP_EQ, 10);
+  /* Now filter to find the entries that also have i%5 == 3 */
+  consensus_cache_filter_list(lst, "mod5", "3");
+  tt_int_op(smartlist_len(lst), OP_EQ, 2);
+  /* So now we have those entries for which i%15 == 13. */
+
+  consensus_cache_entry_t *ent1 = smartlist_get(lst, 0);
+  consensus_cache_entry_t *ent2 = smartlist_get(lst, 1);
+  const char *idx1 = consensus_cache_entry_get_value(ent1, "index");
+  const char *idx2 = consensus_cache_entry_get_value(ent2, "index");
+  tt_assert( (!strcmp(idx1, "28") && !strcmp(idx2, "13")) ||
+             (!strcmp(idx1, "13") && !strcmp(idx2, "28")) );
+
+ done:
+  tor_free(ddir_fname);
+  consensus_cache_free(cache);
+  smartlist_free(lst);
+}
+
+#define ENT(name)                                               \
+  { #name, test_conscache_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t conscache_tests[] = {
+  ENT(open_failure),
+  ENT(simple_usage),
+  ENT(cleanup),
+  ENT(filter),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
new file mode 100644
index 0000000000..7cf8d6ba2b
--- /dev/null
+++ b/src/test/test_consdiff.c
@@ -0,0 +1,1184 @@
+/* Copyright (c) 2014, Daniel Martí
+ * Copyright (c) 2014, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFF_PRIVATE
+
+#include "or.h"
+#include "test.h"
+
+#include "consdiff.h"
+#include "memarea.h"
+#include "log_test_helpers.h"
+
+#define tt_str_eq_line(a,b) \
+  tt_assert(line_str_eq((b),(a)))
+
+static void
+test_consdiff_smartlist_slice(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_slice_t *sls;
+
+  /* Create a regular smartlist. */
+  (void)arg;
+  smartlist_add(sl, (void*)1);
+  smartlist_add(sl, (void*)2);
+  smartlist_add(sl, (void*)3);
+  smartlist_add(sl, (void*)4);
+  smartlist_add(sl, (void*)5);
+
+  /* See if the slice was done correctly. */
+  sls = smartlist_slice(sl, 2, 5);
+  tt_ptr_op(sl, OP_EQ, sls->list);
+  tt_ptr_op((void*)3, OP_EQ, smartlist_get(sls->list, sls->offset));
+  tt_ptr_op((void*)5, OP_EQ,
+      smartlist_get(sls->list, sls->offset + (sls->len-1)));
+  tor_free(sls);
+
+  /* See that using -1 as the end does get to the last element. */
+  sls = smartlist_slice(sl, 2, -1);
+  tt_ptr_op(sl, OP_EQ, sls->list);
+  tt_ptr_op((void*)3, OP_EQ, smartlist_get(sls->list, sls->offset));
+  tt_ptr_op((void*)5, OP_EQ,
+      smartlist_get(sls->list, sls->offset + (sls->len-1)));
+
+ done:
+  tor_free(sls);
+  smartlist_free(sl);
+}
+
+static void
+test_consdiff_smartlist_slice_string_pos(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_slice_t *sls;
+  memarea_t *area = memarea_new();
+
+  /* Create a regular smartlist. */
+  (void)arg;
+  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+
+  /* See that smartlist_slice_string_pos respects the bounds of the slice. */
+  sls = smartlist_slice(sl, 2, 5);
+  cdline_t a_line = { "a", 1 };
+  tt_int_op(3, OP_EQ, smartlist_slice_string_pos(sls, &a_line));
+  cdline_t d_line = { "d", 1 };
+  tt_int_op(-1, OP_EQ, smartlist_slice_string_pos(sls, &d_line));
+
+ done:
+  tor_free(sls);
+  smartlist_free(sl);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_lcs_lengths(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2;
+  int *lengths1, *lengths2;
+  memarea_t *area = memarea_new();
+
+  /* Expected lcs lengths in regular and reverse order. */
+  int e_lengths1[] = { 0, 1, 2, 3, 3, 4 };
+  int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+
+  lengths1 = lcs_lengths(sls1, sls2, 1);
+  lengths2 = lcs_lengths(sls1, sls2, -1);
+  tt_mem_op(e_lengths1, OP_EQ, lengths1, sizeof(int) * 6);
+  tt_mem_op(e_lengths2, OP_EQ, lengths2, sizeof(int) * 6);
+
+ done:
+  tor_free(lengths1);
+  tor_free(lengths2);
+  tor_free(sls1);
+  tor_free(sls2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_trim_slices(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_t *sl3 = smartlist_new();
+  smartlist_t *sl4 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2, *sls3, *sls4;
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  sls3 = smartlist_slice(sl3, 0, -1);
+  sls4 = smartlist_slice(sl4, 0, -1);
+
+  /* They should be trimmed by one line at each end. */
+  tt_int_op(5, OP_EQ, sls1->len);
+  tt_int_op(5, OP_EQ, sls2->len);
+  trim_slices(sls1, sls2);
+  tt_int_op(3, OP_EQ, sls1->len);
+  tt_int_op(3, OP_EQ, sls2->len);
+
+  /* They should not be trimmed at all. */
+  tt_int_op(5, OP_EQ, sls3->len);
+  tt_int_op(5, OP_EQ, sls4->len);
+  trim_slices(sls3, sls4);
+  tt_int_op(5, OP_EQ, sls3->len);
+  tt_int_op(5, OP_EQ, sls4->len);
+
+ done:
+  tor_free(sls1);
+  tor_free(sls2);
+  tor_free(sls3);
+  tor_free(sls4);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  smartlist_free(sl3);
+  smartlist_free(sl4);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_set_changed(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  bitarray_t *changed1 = bitarray_init_zero(4);
+  bitarray_t *changed2 = bitarray_init_zero(4);
+  smartlist_slice_t *sls1, *sls2;
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines(sl2, "a\na\na\na\n", area);
+
+  /* Length of sls1 is 0. */
+  sls1 = smartlist_slice(sl1, 0, 0);
+  sls2 = smartlist_slice(sl2, 1, 3);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The former is not changed, the latter changes all of its elements. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+  bitarray_clear(changed2, 1);
+  bitarray_clear(changed2, 2);
+
+  /* Length of sls1 is 1 and its element is in sls2. */
+  tor_free(sls1);
+  sls1 = smartlist_slice(sl1, 0, 1);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The latter changes all elements but the (first) common one. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(!bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+  bitarray_clear(changed2, 2);
+
+  /* Length of sls1 is 1 and its element is not in sls2. */
+  tor_free(sls1);
+  sls1 = smartlist_slice(sl1, 1, 2);
+  set_changed(changed1, changed2, sls1, sls2);
+
+  /* The former changes its element, the latter changes all elements. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+
+ done:
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  tor_free(sls1);
+  tor_free(sls2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_calc_changes(void *arg)
+{
+  smartlist_t *sl1 = smartlist_new();
+  smartlist_t *sl2 = smartlist_new();
+  smartlist_slice_t *sls1, *sls2;
+  bitarray_t *changed1 = bitarray_init_zero(4);
+  bitarray_t *changed2 = bitarray_init_zero(4);
+  memarea_t *area = memarea_new();
+
+  (void)arg;
+  consensus_split_lines(sl1, "a\na\na\na\n", area);
+  consensus_split_lines(sl2, "a\na\na\na\n", area);
+
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* Nothing should be set to changed. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(!bitarray_is_set(changed1, 1));
+  tt_assert(!bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(!bitarray_is_set(changed2, 1));
+  tt_assert(!bitarray_is_set(changed2, 2));
+  tt_assert(!bitarray_is_set(changed2, 3));
+
+  smartlist_clear(sl2);
+  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  tor_free(sls1);
+  tor_free(sls2);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* Two elements are changed. */
+  tt_assert(!bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(bitarray_is_set(changed1, 2));
+  tt_assert(!bitarray_is_set(changed1, 3));
+  bitarray_clear(changed1, 1);
+  bitarray_clear(changed1, 2);
+
+  tt_assert(!bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(!bitarray_is_set(changed2, 2));
+  tt_assert(bitarray_is_set(changed2, 3));
+  bitarray_clear(changed1, 1);
+  bitarray_clear(changed1, 3);
+
+  smartlist_clear(sl2);
+  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  tor_free(sls1);
+  tor_free(sls2);
+  sls1 = smartlist_slice(sl1, 0, -1);
+  sls2 = smartlist_slice(sl2, 0, -1);
+  calc_changes(sls1, sls2, changed1, changed2);
+
+  /* All elements are changed. */
+  tt_assert(bitarray_is_set(changed1, 0));
+  tt_assert(bitarray_is_set(changed1, 1));
+  tt_assert(bitarray_is_set(changed1, 2));
+  tt_assert(bitarray_is_set(changed1, 3));
+
+  tt_assert(bitarray_is_set(changed2, 0));
+  tt_assert(bitarray_is_set(changed2, 1));
+  tt_assert(bitarray_is_set(changed2, 2));
+  tt_assert(bitarray_is_set(changed2, 3));
+
+ done:
+  bitarray_free(changed1);
+  bitarray_free(changed2);
+  smartlist_free(sl1);
+  smartlist_free(sl2);
+  tor_free(sls1);
+  tor_free(sls2);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_get_id_hash(void *arg)
+{
+  (void)arg;
+
+  cdline_t line1 = { "r name", 6 };
+  cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 };
+  cdline_t line3 = { "r name hash+valid+base64 etc", 28 };
+  cdline_t tmp;
+
+  /* No hash. */
+  tt_int_op(-1, OP_EQ, get_id_hash(&line1, &tmp));
+  /* The hash contains characters that are not base64. */
+  tt_int_op(-1, OP_EQ, get_id_hash(&line2, &tmp));
+
+  /* valid hash. */
+  tt_int_op(0, OP_EQ, get_id_hash(&line3, &tmp));
+  tt_ptr_op(tmp.s, OP_EQ, line3.s + 7);
+  tt_uint_op(tmp.len, OP_EQ, line3.len - 11);
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_is_valid_router_entry(void *arg)
+{
+  /* Doesn't start with "r ". */
+  (void)arg;
+  cdline_t line0 = { "foo", 3 };
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line0));
+
+  /* These are already tested with get_id_hash, but make sure it's run
+   * properly. */
+
+  cdline_t line1 = { "r name", 6 };
+  cdline_t line2 = { "r name _hash_isnt_base64 etc", 28 };
+  cdline_t line3 = { "r name hash+valid+base64 etc", 28 };
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line1));
+  tt_int_op(0, OP_EQ, is_valid_router_entry(&line2));
+  tt_int_op(1, OP_EQ, is_valid_router_entry(&line3));
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_next_router(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  memarea_t *area = memarea_new();
+  (void)arg;
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area,
+      "r name hash+longer+than+27+chars+and+valid+base64 etc");
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area, "foo");
+  smartlist_add_linecpy(sl, area,
+      "r name hash+longer+than+27+chars+and+valid+base64 etc");
+  smartlist_add_linecpy(sl, area, "foo");
+
+  /* Not currently on a router entry line, finding the next one. */
+  tt_int_op(1, OP_EQ, next_router(sl, 0));
+  tt_int_op(4, OP_EQ, next_router(sl, 2));
+
+  /* Already at the beginning of a router entry line, ignore it. */
+  tt_int_op(4, OP_EQ, next_router(sl, 1));
+
+  /* There are no more router entries, so return the line after the last. */
+  tt_int_op(6, OP_EQ, next_router(sl, 4));
+  tt_int_op(6, OP_EQ, next_router(sl, 5));
+
+ done:
+  smartlist_free(sl);
+  memarea_drop_all(area);
+}
+
+static int
+base64cmp_wrapper(const char *a, const char *b)
+{
+  cdline_t aa = { a, a ? (uint32_t) strlen(a) : 0 };
+  cdline_t bb = { b, b ? (uint32_t) strlen(b) : 0 };
+  return base64cmp(&aa, &bb);
+}
+
+static void
+test_consdiff_base64cmp(void *arg)
+{
+  /* NULL arguments. */
+  (void)arg;
+  tt_int_op(0, OP_EQ, base64cmp_wrapper(NULL, NULL));
+  tt_int_op(-1, OP_EQ, base64cmp_wrapper(NULL, "foo"));
+  tt_int_op(1, OP_EQ, base64cmp_wrapper("bar", NULL));
+
+  /* Nil base64 values. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("", ""));
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("_", "&"));
+
+  /* Exact same valid strings. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+", "abcABC/+"));
+  /* Both end with an invalid base64 char other than '\0'. */
+  tt_int_op(0, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+ "));
+  /* Only one ends with an invalid base64 char other than '\0'. */
+  tt_int_op(-1, OP_EQ, base64cmp_wrapper("abcABC/+ ", "abcABC/+a"));
+
+  /* Comparisons that would return differently with strcmp(). */
+  tt_int_op(strcmp("/foo", "Afoo"), OP_LT, 0);
+  tt_int_op(base64cmp_wrapper("/foo", "Afoo"), OP_GT, 0);
+  tt_int_op(strcmp("Afoo", "0foo"), OP_GT, 0);
+  tt_int_op(base64cmp_wrapper("Afoo", "0foo"), OP_LT, 0);
+
+  /* Comparisons that would return the same as with strcmp(). */
+  tt_int_op(strcmp("afoo", "Afoo"), OP_GT, 0);
+  tt_int_op(base64cmp_wrapper("afoo", "Afoo"), OP_GT, 0);
+
+  /* Different lengths */
+  tt_int_op(base64cmp_wrapper("afoo", "afooo"), OP_LT, 0);
+  tt_int_op(base64cmp_wrapper("afooo", "afoo"), OP_GT, 0);
+
+ done:
+  ;
+}
+
+static void
+test_consdiff_gen_ed_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  int i;
+  memarea_t *area = memarea_new();
+  setup_capture_of_logs(LOG_WARN);
+
+  (void)arg;
+  cons1 = smartlist_new();
+  cons2 = smartlist_new();
+
+  /* Identity hashes are not sorted properly, return NULL. */
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  smartlist_add_linecpy(cons2, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+  smartlist_add_linecpy(cons2, area, "foo");
+  smartlist_add_linecpy(cons2, area, "r name ccccccccccccccccccccccccccc etc");
+  smartlist_add_linecpy(cons2, area, "bar");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Same, but now with the second consensus. */
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the target consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Same as the two above, but with the reversed thing immediately after a
+     match. (The code handles this differently) */
+  smartlist_del(cons1, 0);
+  smartlist_add_linecpy(cons1, area, "r name aaaaaaaaaaaaaaaaaaaaaaaaaaa etc");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the target consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* Identity hashes are repeated, return NULL. */
+  smartlist_clear(cons1);
+
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "r name bbbbbbbbbbbbbbbbbbbbbbbbbbb etc");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because the base consensus doesn't have its router entries sorted "
+         "properly.");
+
+  /* We have to add a line that is just a dot, return NULL. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  smartlist_add_linecpy(cons1, area, "foo1");
+  smartlist_add_linecpy(cons1, area, "foo2");
+
+  smartlist_add_linecpy(cons2, area, "foo1");
+  smartlist_add_linecpy(cons2, area, ".");
+  smartlist_add_linecpy(cons2, area, "foo2");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Cannot generate consensus diff "
+         "because one of the lines to be added is \".\".");
+
+#define MAX_LINE_COUNT (10000)
+  /* Too many lines to be fed to the quadratic-time function. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "a");
+  for (i=0; i < MAX_LINE_COUNT; ++i) smartlist_add_linecpy(cons1, area, "b");
+
+  mock_clean_saved_logs();
+  diff = gen_ed_diff(cons1, cons2, area);
+
+  tt_ptr_op(NULL, OP_EQ, diff);
+  expect_single_log_msg_containing("Refusing to generate consensus diff "
+         "because we found too few common router ids.");
+
+  /* We have dot lines, but they don't interfere with the script format. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  smartlist_add_linecpy(cons1, area, "foo1");
+  smartlist_add_linecpy(cons1, area, ".");
+  smartlist_add_linecpy(cons1, area, ".");
+  smartlist_add_linecpy(cons1, area, "foo2");
+
+  smartlist_add_linecpy(cons2, area, "foo1");
+  smartlist_add_linecpy(cons2, area, ".");
+  smartlist_add_linecpy(cons2, area, "foo2");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  smartlist_free(diff);
+
+  /* Empty diff tests. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(0, OP_EQ, smartlist_len(diff));
+  smartlist_free(diff);
+
+  smartlist_add_linecpy(cons1, area, "foo");
+  smartlist_add_linecpy(cons1, area, "bar");
+
+  smartlist_add_linecpy(cons2, area, "foo");
+  smartlist_add_linecpy(cons2, area, "bar");
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(0, OP_EQ, smartlist_len(diff));
+  smartlist_free(diff);
+
+  /* Everything is deleted. */
+  smartlist_clear(cons2);
+
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(1, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("1,2d", smartlist_get(diff, 0));
+
+  smartlist_free(diff);
+
+  /* Everything is added. */
+  diff = gen_ed_diff(cons2, cons1, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(4, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("0a", smartlist_get(diff, 0));
+  tt_str_eq_line("foo", smartlist_get(diff, 1));
+  tt_str_eq_line("bar", smartlist_get(diff, 2));
+  tt_str_eq_line(".", smartlist_get(diff, 3));
+
+  smartlist_free(diff);
+
+  /* Everything is changed. */
+  smartlist_add_linecpy(cons2, area, "foo2");
+  smartlist_add_linecpy(cons2, area, "bar2");
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(4, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("1,2c", smartlist_get(diff, 0));
+  tt_str_eq_line("foo2", smartlist_get(diff, 1));
+  tt_str_eq_line("bar2", smartlist_get(diff, 2));
+  tt_str_eq_line(".", smartlist_get(diff, 3));
+
+  smartlist_free(diff);
+
+  /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(7, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("5a", smartlist_get(diff, 0));
+  tt_str_eq_line("U", smartlist_get(diff, 1));
+  tt_str_eq_line(".", smartlist_get(diff, 2));
+  tt_str_eq_line("4c", smartlist_get(diff, 3));
+  tt_str_eq_line("O", smartlist_get(diff, 4));
+  tt_str_eq_line(".", smartlist_get(diff, 5));
+  tt_str_eq_line("2d", smartlist_get(diff, 6));
+
+  smartlist_free(diff);
+
+  smartlist_clear(cons1);
+  smartlist_clear(cons2);
+  consensus_split_lines(cons1, "B\n", area);
+  consensus_split_lines(cons2, "A\nB\n", area);
+  diff = gen_ed_diff(cons1, cons2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(3, OP_EQ, smartlist_len(diff));
+  tt_str_eq_line("0a", smartlist_get(diff, 0));
+  tt_str_eq_line("A", smartlist_get(diff, 1));
+  tt_str_eq_line(".", smartlist_get(diff, 2));
+
+  /* TODO: small real use-cases, i.e. consensuses. */
+
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_apply_ed_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  memarea_t *area = memarea_new();
+  (void)arg;
+  cons1 = smartlist_new();
+  diff = smartlist_new();
+  setup_capture_of_logs(LOG_WARN);
+
+  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+
+  /* Command without range. */
+  smartlist_add_linecpy(diff, area, "a");
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  smartlist_clear(diff);
+  expect_single_log_msg_containing("an ed command was missing a line number");
+
+  /* Range without command. */
+  smartlist_add_linecpy(diff, area, "1");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("a line with no ed command was found");
+
+  smartlist_clear(diff);
+
+  /* Range without end. */
+  smartlist_add_linecpy(diff, area, "1,");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a range "
+                                   "end line number.");
+
+  smartlist_clear(diff);
+
+  /* Incoherent ranges. */
+  smartlist_add_linecpy(diff, area, "1,1");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an invalid range was found");
+
+  smartlist_clear(diff);
+
+  smartlist_add_linecpy(diff, area, "3,2");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an invalid range was found");
+
+  smartlist_clear(diff);
+
+  /* Unexpected range for add command. */
+  smartlist_add_linecpy(diff, area, "1,2a");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("add lines after a range");
+
+  smartlist_clear(diff);
+
+  /* $ for a non-delete command. */
+  smartlist_add_linecpy(diff, area, "1,$c");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it wanted to use $ with a command "
+                                   "other than delete");
+
+  smartlist_clear(diff);
+
+  /* Script is not in reverse order. */
+  smartlist_add_linecpy(diff, area, "1d");
+  smartlist_add_linecpy(diff, area, "3d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("its commands are not properly sorted");
+
+  smartlist_clear(diff);
+
+  /* Script contains unrecognised commands longer than one char. */
+  smartlist_add_linecpy(diff, area, "1foo");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* Script contains unrecognised commands. */
+  smartlist_add_linecpy(diff, area, "1e");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an unrecognised ed command was found");
+
+  smartlist_clear(diff);
+
+  /* Command that should be followed by at least one line and a ".", but
+   * isn't. */
+  smartlist_add_linecpy(diff, area, "0a");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it has an ed command that tries to "
+                                   "insert zero lines.");
+
+  /* Now it is followed by a ".", but it inserts zero lines. */
+  smartlist_add_linecpy(diff, area, ".");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("it has an ed command that tries to "
+                                   "insert zero lines.");
+
+  smartlist_clear(diff);
+
+  /* Now it it inserts something, but has no terminator. */
+  smartlist_add_linecpy(diff, area, "0a");
+  smartlist_add_linecpy(diff, area, "hello");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("lines to be inserted that don't end with "
+                                   "a \".\".");
+
+  smartlist_clear(diff);
+
+  /* Ranges must be numeric only and cannot contain spaces. */
+  smartlist_add_linecpy(diff, area, "0, 4d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a range "
+                                   "end line number.");
+
+  smartlist_clear(diff);
+
+  /* '+' is not a number. */
+  smartlist_add_linecpy(diff, area, "+0,4d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command was missing a line number");
+
+  smartlist_clear(diff);
+
+  /* range duplication */
+  smartlist_add_linecpy(diff, area, "0,4d,5d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* space before command */
+  smartlist_add_linecpy(diff, area, "0,4 d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* space inside number */
+  smartlist_add_linecpy(diff, area, "0,4 5d");
+  mock_clean_saved_logs();
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("an ed command longer than one char was "
+                                   "found");
+
+  smartlist_clear(diff);
+
+  /* Test appending text, 'a'. */
+  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(8, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("V", smartlist_get(cons2, 0));
+  tt_str_eq_line("A", smartlist_get(cons2, 1));
+  tt_str_eq_line("B", smartlist_get(cons2, 2));
+  tt_str_eq_line("C", smartlist_get(cons2, 3));
+  tt_str_eq_line("U", smartlist_get(cons2, 4));
+  tt_str_eq_line("O", smartlist_get(cons2, 5));
+  tt_str_eq_line("D", smartlist_get(cons2, 6));
+  tt_str_eq_line("E", smartlist_get(cons2, 7));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test deleting text, 'd'. */
+  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(2, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("C", smartlist_get(cons2, 0));
+  tt_str_eq_line("E", smartlist_get(cons2, 1));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test changing text, 'c'. */
+  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(5, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("M", smartlist_get(cons2, 0));
+  tt_str_eq_line("C", smartlist_get(cons2, 1));
+  tt_str_eq_line("T", smartlist_get(cons2, 2));
+  tt_str_eq_line("X", smartlist_get(cons2, 3));
+  tt_str_eq_line("E", smartlist_get(cons2, 4));
+
+  smartlist_clear(diff);
+  smartlist_free(cons2);
+
+  /* Test 'a', 'd' and 'c' together. */
+  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  cons2 = apply_ed_diff(cons1, diff, 0);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_int_op(6, OP_EQ, smartlist_len(cons2));
+  tt_str_eq_line("M", smartlist_get(cons2, 0));
+  tt_str_eq_line("A", smartlist_get(cons2, 1));
+  tt_str_eq_line("C", smartlist_get(cons2, 2));
+  tt_str_eq_line("T", smartlist_get(cons2, 3));
+  tt_str_eq_line("X", smartlist_get(cons2, 4));
+  tt_str_eq_line("E", smartlist_get(cons2, 5));
+
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_gen_diff(void *arg)
+{
+  char *cons1_str=NULL, *cons2_str=NULL;
+  smartlist_t *cons1=NULL, *cons2=NULL, *diff=NULL;
+  consensus_digest_t digests1, digests2;
+  memarea_t *area = memarea_new();
+  (void)arg;
+  cons1 = smartlist_new();
+  cons2 = smartlist_new();
+
+  /* Identity hashes are not sorted properly, return NULL.
+   * Already tested in gen_ed_diff, but see that a NULL ed diff also makes
+   * gen_diff return NULL. */
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name bbbbbbbbbbbbbbbbb etc\nfoo\n"
+      "r name aaaaaaaaaaaaaaaaa etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  cons2_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name aaaaaaaaaaaaaaaaa etc\nfoo\n"
+      "r name ccccccccccccccccc etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest_as_signed(cons1_str, &digests1));
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest(cons2_str, &digests2));
+
+  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines(cons2, cons2_str, area);
+
+  diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
+  tt_ptr_op(NULL, OP_EQ, diff);
+
+  /* Check that the headers are done properly. */
+  tor_free(cons1_str);
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nfoo\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest_as_signed(cons1_str, &digests1));
+  smartlist_clear(cons1);
+  consensus_split_lines(cons1, cons1_str, area);
+  diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
+  tt_ptr_op(NULL, OP_NE, diff);
+  tt_int_op(11, OP_EQ, smartlist_len(diff));
+  tt_assert(line_str_eq(smartlist_get(diff, 0),
+                        "network-status-diff-version 1"));
+  tt_assert(line_str_eq(smartlist_get(diff, 1), "hash "
+      "95D70F5A3CC65F920AA8B44C4563D7781A082674329661884E19E94B79D539C2 "
+      "7AFECEFA4599BA33D603653E3D2368F648DF4AC4723929B0F7CF39281596B0C1"));
+  tt_assert(line_str_eq(smartlist_get(diff, 2), "6,$d"));
+  tt_assert(line_str_eq(smartlist_get(diff, 3), "3,4c"));
+  tt_assert(line_str_eq(smartlist_get(diff, 4), "bar"));
+  tt_assert(line_str_eq(smartlist_get(diff, 5),
+                        "directory-signature foo bar"));
+  tt_assert(line_str_eq(smartlist_get(diff, 6),
+                        "."));
+  tt_assert(line_str_eq(smartlist_get(diff, 7), "1a"));
+  tt_assert(line_str_eq(smartlist_get(diff, 8),
+                        "r name aaaaaaaaaaaaaaaaa etc"));
+  tt_assert(line_str_eq(smartlist_get(diff, 9), "foo"));
+  tt_assert(line_str_eq(smartlist_get(diff, 10), "."));
+
+  /* TODO: small real use-cases, i.e. consensuses. */
+
+ done:
+  tor_free(cons1_str);
+  tor_free(cons2_str);
+  smartlist_free(cons1);
+  smartlist_free(cons2);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+static void
+test_consdiff_apply_diff(void *arg)
+{
+  smartlist_t *cons1=NULL, *diff=NULL;
+  char *cons1_str=NULL, *cons2 = NULL;
+  consensus_digest_t digests1;
+  (void)arg;
+  memarea_t *area = memarea_new();
+  cons1 = smartlist_new();
+  diff = smartlist_new();
+  setup_capture_of_logs(LOG_INFO);
+
+  cons1_str = tor_strdup(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nfoo\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n"
+      );
+  tt_int_op(0, OP_EQ,
+      consensus_compute_digest(cons1_str, &digests1));
+  consensus_split_lines(cons1, cons1_str, area);
+
+  /* diff doesn't have enough lines. */
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("too short")
+
+  /* first line doesn't match format-version string. */
+  smartlist_add_linecpy(diff, area, "foo-bar");
+  smartlist_add_linecpy(diff, area, "header-line");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("format is not known")
+
+  /* The first word of the second header line is not "hash". */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "word a b");
+  smartlist_add_linecpy(diff, area, "x");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("does not include the necessary digests")
+
+  /* Wrong number of words after "hash". */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash a b c");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("does not include the necessary digests")
+
+  /* base16 digests do not have the expected length. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash aaa bbb");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("includes base16-encoded digests of "
+                                   "incorrect size")
+
+  /* base16 digests contain non-base16 characters. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      " ????????????????????????????????????????????????????????????????"
+      " ----------------------------------------------------------------");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("includes malformed digests")
+
+  /* Invalid ed diff.
+   * As tested in apply_ed_diff, but check that apply_diff does return NULL if
+   * the ed diff can't be applied. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* sha256 of cons2. */
+      " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA");
+  smartlist_add_linecpy(diff, area, "foobar");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_single_log_msg_containing("because an ed command was missing a line "
+                                   "number")
+
+  /* Base consensus doesn't match its digest as found in the diff. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* bogus sha256. */
+      " 3333333333333333333333333333333333333333333333333333333333333333"
+      /* sha256 of cons2. */
+      " 635D34593020C08E5ECD865F9986E29D50028EFA62843766A8197AD228A7F6AA");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("base consensus doesn't match the digest "
+                            "as found");
+
+  /* Resulting consensus doesn't match its digest as found in the diff. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* bogus sha3. */
+      " 3333333333333333333333333333333333333333333333333333333333333333");
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("resulting consensus doesn't match the "
+                            "digest as found");
+
+#if 0
+  /* XXXX No longer possible, since we aren't using the other algorithm. */
+  /* Resulting consensus digest cannot be computed */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* bogus sha3. */
+      " 3333333333333333333333333333333333333333333333333333333333333333");
+  smartlist_add_linecpy(diff, area, "1,2d"); // remove starting line
+  mock_clean_saved_logs();
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_EQ, cons2);
+  expect_log_msg_containing("Could not compute digests of the consensus "
+                            "resulting from applying a consensus diff.");
+#endif
+
+  /* Very simple test, only to see that nothing errors. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646D6CF563A41869D3B02E73254372AE3140046C5E7D83C9F71E54976AF9B4"
+      /* sha3 of cons2. */
+      " 90A418881B2FCAB3D9E60EE02E4D666D56CFA38F8A3B7AA3E0ADBA530DDA9353");
+  smartlist_add_linecpy(diff, area, "3c");
+  smartlist_add_linecpy(diff, area, "sample");
+  smartlist_add_linecpy(diff, area, ".");
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_str_op(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nsample\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n", OP_EQ,
+      cons2);
+  tor_free(cons2);
+
+  /* Check that lowercase letters in base16-encoded digests work too. */
+  smartlist_clear(diff);
+  smartlist_add_linecpy(diff, area, "network-status-diff-version 1");
+  smartlist_add_linecpy(diff, area, "hash"
+      /* sha3 of cons1. */
+      " 06646d6cf563a41869d3b02e73254372ae3140046c5e7d83c9f71e54976af9b4"
+      /* sha3 of cons2. */
+      " 90a418881b2fcab3d9e60ee02e4d666d56cfa38f8a3b7aa3e0adba530dda9353");
+  smartlist_add_linecpy(diff, area, "3c");
+  smartlist_add_linecpy(diff, area, "sample");
+  smartlist_add_linecpy(diff, area, ".");
+  cons2 = consdiff_apply_diff(cons1, diff, &digests1);
+  tt_ptr_op(NULL, OP_NE, cons2);
+  tt_str_op(
+      "network-status-version foo\n"
+      "r name ccccccccccccccccc etc\nsample\n"
+      "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+      "directory-signature foo bar\nbar\n", OP_EQ,
+      cons2);
+  tor_free(cons2);
+
+  smartlist_clear(diff);
+
+ done:
+  teardown_capture_of_logs();
+  tor_free(cons1_str);
+  smartlist_free(cons1);
+  smartlist_free(diff);
+  memarea_drop_all(area);
+}
+
+#define CONSDIFF_LEGACY(name)                                          \
+  { #name, test_consdiff_ ## name , 0, NULL, NULL }
+
+struct testcase_t consdiff_tests[] = {
+  CONSDIFF_LEGACY(smartlist_slice),
+  CONSDIFF_LEGACY(smartlist_slice_string_pos),
+  CONSDIFF_LEGACY(lcs_lengths),
+  CONSDIFF_LEGACY(trim_slices),
+  CONSDIFF_LEGACY(set_changed),
+  CONSDIFF_LEGACY(calc_changes),
+  CONSDIFF_LEGACY(get_id_hash),
+  CONSDIFF_LEGACY(is_valid_router_entry),
+  CONSDIFF_LEGACY(next_router),
+  CONSDIFF_LEGACY(base64cmp),
+  CONSDIFF_LEGACY(gen_ed_diff),
+  CONSDIFF_LEGACY(apply_ed_diff),
+  CONSDIFF_LEGACY(gen_diff),
+  CONSDIFF_LEGACY(apply_diff),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
new file mode 100644
index 0000000000..963a6e427a
--- /dev/null
+++ b/src/test/test_consdiffmgr.c
@@ -0,0 +1,896 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONSDIFFMGR_PRIVATE
+
+#include "or.h"
+#include "config.h"
+#include "conscache.h"
+#include "consdiff.h"
+#include "consdiffmgr.h"
+#include "cpuworker.h"
+#include "networkstatus.h"
+#include "routerparse.h"
+#include "workqueue.h"
+
+#include "test.h"
+#include "log_test_helpers.h"
+
+// ============================== Setup/teardown the consdiffmgr
+// These functions get run before/after each test in this module
+
+static void *
+consdiffmgr_test_setup(const struct testcase_t *arg)
+{
+  (void)arg;
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer.
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+
+  consdiff_cfg_t consdiff_cfg = { 300 };
+  consdiffmgr_configure(&consdiff_cfg);
+  return (void *)1; // must return something non-null.
+}
+static int
+consdiffmgr_test_teardown(const struct testcase_t *arg, void *ignore)
+{
+  (void)arg;
+  (void)ignore;
+  consdiffmgr_free_all();
+  return 1;
+}
+static struct testcase_setup_t setup_diffmgr = {
+  consdiffmgr_test_setup,
+  consdiffmgr_test_teardown
+};
+
+// ============================== NS faking functions
+// These functions are for making quick fake consensus objects and
+// strings that are just good enough for consdiff and consdiffmgr.
+
+static networkstatus_t *
+fake_ns_new(consensus_flavor_t flav, time_t valid_after)
+{
+  networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->type = NS_TYPE_CONSENSUS;
+  ns->flavor = flav;
+  ns->valid_after = valid_after;
+  return ns;
+}
+
+static char *
+fake_ns_body_new(consensus_flavor_t flav, time_t valid_after)
+{
+  const char *flavor_string = flav == FLAV_NS ? "" : " microdesc";
+  char valid_after_string[ISO_TIME_LEN+1];
+
+  format_iso_time(valid_after_string, valid_after);
+  char *random_stuff = crypto_random_hostname(3, 25, "junk ", "");
+  char *random_stuff2 = crypto_random_hostname(3, 10, "", "");
+
+  char *consensus;
+  tor_asprintf(&consensus,
+               "network-status-version 3%s\n"
+               "vote-status consensus\n"
+               "valid-after %s\n"
+               "r name ccccccccccccccccc etc\nsample\n"
+               "r name eeeeeeeeeeeeeeeee etc\nbar\n"
+               "%s\n"
+               "directory-signature hello-there\n"
+               "directory-signature %s\n",
+               flavor_string,
+               valid_after_string,
+               random_stuff,
+               random_stuff2);
+  tor_free(random_stuff);
+  tor_free(random_stuff2);
+  return consensus;
+}
+
+// ============================== Cpuworker mocking code
+// These mocking functions and types capture the cpuworker calls
+// so we can inspect them and run them in the main thread.
+static smartlist_t *fake_cpuworker_queue = NULL;
+typedef struct fake_work_queue_ent_t {
+  enum workqueue_reply_t (*fn)(void *, void *);
+  void (*reply_fn)(void *);
+  void *arg;
+} fake_work_queue_ent_t;
+static struct workqueue_entry_s *
+mock_cpuworker_queue_work(workqueue_priority_t prio,
+                          enum workqueue_reply_t (*fn)(void *, void *),
+                          void (*reply_fn)(void *),
+                          void *arg)
+{
+  (void) prio;
+
+  if (! fake_cpuworker_queue)
+    fake_cpuworker_queue = smartlist_new();
+
+  fake_work_queue_ent_t *ent = tor_malloc_zero(sizeof(*ent));
+  ent->fn = fn;
+  ent->reply_fn = reply_fn;
+  ent->arg = arg;
+  smartlist_add(fake_cpuworker_queue, ent);
+  return (struct workqueue_entry_s *)ent;
+}
+static int
+mock_cpuworker_run_work(void)
+{
+  if (! fake_cpuworker_queue)
+    return 0;
+  SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
+      enum workqueue_reply_t r = ent->fn(NULL, ent->arg);
+      if (r != WQ_RPL_REPLY)
+        return -1;
+  });
+  return 0;
+}
+static void
+mock_cpuworker_handle_replies(void)
+{
+  if (! fake_cpuworker_queue)
+    return;
+  SMARTLIST_FOREACH(fake_cpuworker_queue, fake_work_queue_ent_t *, ent, {
+      ent->reply_fn(ent->arg);
+      tor_free(ent);
+  });
+  smartlist_free(fake_cpuworker_queue);
+  fake_cpuworker_queue = NULL;
+}
+
+// ==============================  Other helpers
+
+static consdiff_status_t
+lookup_diff_from(consensus_cache_entry_t **out,
+                 consensus_flavor_t flav,
+                 const char *str1)
+{
+  uint8_t digest[DIGEST256_LEN];
+  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+    TT_FAIL(("Unable to compute sha3-as-signed"));
+    return CONSDIFF_NOT_FOUND;
+  }
+  return consdiffmgr_find_diff_from(out, flav,
+                                    DIGEST_SHA3_256, digest, sizeof(digest),
+                                    NO_METHOD);
+}
+
+static int
+lookup_apply_and_verify_diff(consensus_flavor_t flav,
+                             const char *str1,
+                             const char *str2)
+{
+  consensus_cache_entry_t *ent = NULL;
+  consdiff_status_t status = lookup_diff_from(&ent, flav, str1);
+  if (ent == NULL || status != CONSDIFF_AVAILABLE) {
+    return -1;
+  }
+
+  consensus_cache_entry_incref(ent);
+  size_t size;
+  char *diff_string = NULL;
+  int r = uncompress_or_copy(&diff_string, &size, ent);
+  consensus_cache_entry_decref(ent);
+  if (diff_string == NULL || r < 0)
+    return -1;
+
+  char *applied = consensus_diff_apply(str1, diff_string);
+  tor_free(diff_string);
+  if (applied == NULL)
+    return -1;
+
+  int match = !strcmp(applied, str2);
+  tor_free(applied);
+  return match ? 0 : -1;
+}
+
+static void
+cdm_reload(void)
+{
+  consdiffmgr_free_all();
+  cdm_cache_get();
+  consdiffmgr_rescan();
+}
+
+// ==============================  Beginning of tests
+
+#if 0
+static int got_failure = 0;
+static void
+got_assertion_failure(void)
+{
+  ++got_failure;
+}
+
+/* XXXX This test won't work, because there is currently no way to actually
+ * XXXX capture a real assertion failure. */
+static void
+test_consdiffmgr_init_failure(void *arg)
+{
+  (void)arg;
+  // Capture assertions and bugs.
+
+  /* As in ...test_setup, but do not create the datadir. The missing directory
+   * will cause a failure. */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_cdm"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = ddir_fname; // now owns the pointer.
+
+  consdiff_cfg_t consdiff_cfg = { 7200, 300 };
+
+  tor_set_failed_assertion_callback(got_assertion_failure);
+  tor_capture_bugs_(1);
+  consdiffmgr_configure(&consdiff_cfg); // This should fail.
+  tt_int_op(got_failure, OP_EQ, 1);
+  const smartlist_t *bugs = tor_get_captured_bug_log_();
+  tt_int_op(smartlist_len(bugs), OP_EQ, 1);
+
+ done:
+  tor_end_capture_bugs_();
+}
+#endif
+
+static void
+test_consdiffmgr_sha3_helper(void *arg)
+{
+  (void) arg;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+  config_line_t *lines = NULL;
+  char *mem_op_hex_tmp = NULL;
+  config_line_prepend(&lines, "good-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+  config_line_prepend(&lines, "short-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF0");
+  config_line_prepend(&lines, "long-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00DF00D");
+  config_line_prepend(&lines, "not-sha",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DXXXX");
+  consensus_cache_entry_t *ent =
+    consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+
+  uint8_t buf[DIGEST256_LEN];
+  tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, NULL, "good-sha"));
+  tt_int_op(0, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "good-sha"));
+  test_memeq_hex(buf, "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+
+  tt_int_op(-1, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "missing-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "short-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "long-sha"));
+  tt_int_op(-2, OP_EQ, cdm_entry_get_sha3_value(buf, ent, "not-sha"));
+
+ done:
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_consdiffmgr_add(void *arg)
+{
+  (void) arg;
+  time_t now = approx_time();
+
+  char *body = NULL;
+
+  consensus_cache_entry_t *ent = NULL;
+  networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
+  const char *dummy = "foo";
+  int r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* If we add it again, it won't work */
+  setup_capture_of_logs(LOG_INFO);
+  dummy = "bar";
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, -1);
+  expect_single_log_msg_containing("We already have a copy of that "
+                                   "consensus");
+  mock_clean_saved_logs();
+
+  /* But it will work fine if the flavor is different */
+  dummy = "baz";
+  ns_tmp->flavor = FLAV_MICRODESC;
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* And it will work fine if the time is different */
+  dummy = "quux";
+  ns_tmp->flavor = FLAV_NS;
+  ns_tmp->valid_after = now - 60;
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* If we add one a long long time ago, it will fail. */
+  dummy = "xyzzy";
+  ns_tmp->valid_after = 86400 * 100; /* A few months into 1970 */
+  r = consdiffmgr_add_consensus(dummy, ns_tmp);
+  tt_int_op(r, OP_EQ, -1);
+  expect_log_msg_containing("it's too old.");
+
+  /* Try looking up a consensuses. */
+  ent = cdm_cache_lookup_consensus(FLAV_NS, now-60);
+  tt_assert(ent);
+  consensus_cache_entry_incref(ent);
+  size_t s;
+  r = uncompress_or_copy(&body, &s, ent);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(s, OP_EQ, 4);
+  tt_mem_op(body, OP_EQ, "quux", 4);
+
+  /* Try looking up another entry, but fail */
+  tt_assert(NULL == cdm_cache_lookup_consensus(FLAV_MICRODESC, now-60));
+  tt_assert(NULL == cdm_cache_lookup_consensus(FLAV_NS, now-61));
+
+ done:
+  networkstatus_vote_free(ns_tmp);
+  teardown_capture_of_logs();
+  consensus_cache_entry_decref(ent);
+  tor_free(body);
+}
+
+static void
+test_consdiffmgr_make_diffs(void *arg)
+{
+  (void)arg;
+  networkstatus_t *ns = NULL;
+  char *ns_body = NULL, *md_ns_body = NULL, *md_ns_body_2 = NULL;
+  char *applied = NULL, *diff_text = NULL;
+  time_t now = approx_time();
+  int r;
+  consensus_cache_entry_t *diff = NULL;
+  uint8_t md_ns_sha3[DIGEST256_LEN];
+  consdiff_status_t diff_status;
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  // Try rescan with no consensuses: shouldn't crash or queue work.
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  // Make two consensuses, 1 hour sec ago.
+  ns = fake_ns_new(FLAV_NS, now-3600);
+  ns_body = fake_ns_body_new(FLAV_NS, now-3600);
+  r = consdiffmgr_add_consensus(ns_body, ns);
+  networkstatus_vote_free(ns);
+  tor_free(ns_body);
+  tt_int_op(r, OP_EQ, 0);
+
+  ns = fake_ns_new(FLAV_MICRODESC, now-3600);
+  md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
+  r = consdiffmgr_add_consensus(md_ns_body, ns);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  networkstatus_vote_free(ns);
+  tt_int_op(r, OP_EQ, 0);
+
+  // No diffs will be generated.
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  // Add a MD consensus from 45 minutes ago. This should cause one diff
+  // worth of work to get queued.
+  ns = fake_ns_new(FLAV_MICRODESC, now-45*60);
+  md_ns_body_2 = fake_ns_body_new(FLAV_MICRODESC, now-45*60);
+  r = consdiffmgr_add_consensus(md_ns_body_2, ns);
+  networkstatus_vote_free(ns);
+  tt_int_op(r, OP_EQ, 0);
+
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
+                                           DIGEST_SHA3_256,
+                                           md_ns_sha3, DIGEST256_LEN,
+                                           NO_METHOD);
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
+
+  // Now run that process and get the diff.
+  r = mock_cpuworker_run_work();
+  tt_int_op(r, OP_EQ, 0);
+  mock_cpuworker_handle_replies();
+
+  // At this point we should be able to get that diff.
+  diff_status = consdiffmgr_find_diff_from(&diff, FLAV_MICRODESC,
+                                           DIGEST_SHA3_256,
+                                           md_ns_sha3, DIGEST256_LEN,
+                                           NO_METHOD);
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ, diff_status);
+  tt_assert(diff);
+
+  /* Make sure applying the diff actually works */
+  const uint8_t *diff_body;
+  size_t diff_size;
+  r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
+  tt_int_op(r, OP_EQ, 0);
+  diff_text = tor_memdup_nulterm(diff_body, diff_size);
+  applied = consensus_diff_apply(md_ns_body, diff_text);
+  tt_assert(applied);
+  tt_str_op(applied, OP_EQ, md_ns_body_2);
+
+  /* Rescan again: no more work to do. */
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+ done:
+  tor_free(md_ns_body);
+  tor_free(md_ns_body_2);
+  tor_free(diff_text);
+  tor_free(applied);
+}
+
+static void
+test_consdiffmgr_diff_rules(void *arg)
+{
+  (void)arg;
+#define N 6
+  char *md_body[N], *ns_body[N];
+  networkstatus_t *md_ns[N], *ns_ns[N];
+  int i;
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* Create a bunch of consensus things at 15-second intervals. */
+  time_t start = approx_time() - 120;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 15;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    ns_body[i] = fake_ns_body_new(FLAV_NS, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+    ns_ns[i] = fake_ns_new(FLAV_NS, when);
+  }
+
+  /* For the MD consensuses: add 4 of them, and make sure that
+   * diffs are created to one consensus (the most recent) only. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[4], md_ns[4]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(3, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  /* For the NS consensuses: add 3, generate, and add one older one and
+   * make sure that older one is the only one whose diff is generated */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[0], ns_ns[0]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[1], ns_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[5], ns_ns[5]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  /* At this point, we should actually have working diffs! */
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
+
+  /* Self-to-self diff won't be present */
+  consensus_cache_entry_t *ent;
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+       lookup_diff_from(&ent, FLAV_NS, ns_body[5]));
+  /* No diff from 2 has been added yet */
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+       lookup_diff_from(&ent, FLAV_NS, ns_body[2]));
+  /* No diff arriving at old things. */
+  tt_int_op(-1, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+  /* No backwards diff */
+  tt_int_op(-1, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[4], md_body[3]));
+
+  /* Now, an update: add number 2 and make sure it's the only one whose diff
+   * is regenerated. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(ns_body[2], ns_ns[2]));
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  tt_int_op(0, OP_EQ,
+            lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
+
+  /* Finally: reload, and make sure that the information is still indexed */
+  cdm_reload();
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[0], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[2], ns_body[5]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_NS, ns_body[1], ns_body[5]));
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[2], md_body[4]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[3], md_body[4]));
+
+ done:
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    tor_free(ns_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+    networkstatus_vote_free(ns_ns[i]);
+  }
+  UNMOCK(cpuworker_queue_work);
+#undef N
+}
+
+static void
+test_consdiffmgr_diff_failure(void *arg)
+{
+  (void)arg;
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* We're going to make sure that if we have a bogus request where
+   * we can't actually compute a diff, the world must not end. */
+  networkstatus_t *ns1 = NULL;
+  networkstatus_t *ns2 = NULL;
+  int r;
+
+  ns1 = fake_ns_new(FLAV_NS, approx_time()-100);
+  ns2 = fake_ns_new(FLAV_NS, approx_time()-50);
+  r = consdiffmgr_add_consensus("foo bar baz\n", ns1);
+  tt_int_op(r, OP_EQ, 0);
+  // We refuse to compute a diff to or from a line holding only a single dot.
+  // We can add it here, though.
+  r = consdiffmgr_add_consensus("foo bar baz\n.\n.\n", ns2);
+  tt_int_op(r, OP_EQ, 0);
+
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  setup_capture_of_logs(LOG_WARN);
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  expect_single_log_msg_containing("one of the lines to be added is \".\".");
+  mock_clean_saved_logs();
+  mock_cpuworker_handle_replies();
+  expect_single_log_msg_containing("Worker was unable to compute consensus "
+                                   "diff from ");
+
+  /* Make sure the diff is not present */
+  consensus_cache_entry_t *ent;
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_NS, "foo bar baz\n"));
+
+ done:
+  teardown_capture_of_logs();
+  UNMOCK(cpuworker_queue_work);
+  networkstatus_vote_free(ns1);
+  networkstatus_vote_free(ns2);
+}
+
+static void
+test_consdiffmgr_diff_pending(void *arg)
+{
+#define N 3
+  (void)arg;
+  char *md_body[N];
+  networkstatus_t *md_ns[N];
+  time_t start = approx_time() - 120;
+  int i;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 30;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+  }
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  /* Make a diff */
+  consdiffmgr_rescan();
+  tt_int_op(1, OP_EQ, smartlist_len(fake_cpuworker_queue));
+
+  /* Look it up.  Is it pending? */
+  consensus_cache_entry_t *ent = NULL;
+  consdiff_status_t diff_status;
+  diff_status = lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]);
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ, diff_status);
+  tt_ptr_op(ent, OP_EQ, NULL);
+
+  /* Add another old consensus.  only one new diff should launch! */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
+  consdiffmgr_rescan();
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+
+ done:
+  UNMOCK(cpuworker_queue_work);
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+  }
+#undef N
+}
+
+static void
+test_consdiffmgr_cleanup_old(void *arg)
+{
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  /* This item will be will be cleanable because it has a valid-after
+   * time far in the past. */
+  config_line_prepend(&labels, "document-type", "confribble-blarg");
+  config_line_prepend(&labels, "consensus-valid-after",
+                      "1980-10-10T10:10:10");
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(1, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Deleting entry because its consensus-valid-"
+                            "after value (1980-10-10T10:10:10) was too old");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_bad_valid_after(void *arg)
+{
+  /* This will seem cleanable, but isn't, because its valid-after time is
+   * misformed. */
+
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  config_line_prepend(&labels, "document-type", "consensus");
+  config_line_prepend(&labels, "consensus-valid-after",
+                      "whan that aprille with his shoures soote"); // (~1385?)
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Ignoring entry because its consensus-valid-"
+                            "after value (\"whan that aprille with his "
+                            "shoures soote\") was unparseable");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_no_valid_after(void *arg)
+{
+  (void)arg;
+  config_line_t *labels = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+
+  /* This item will be will be uncleanable because it has no recognized
+   * valid-after. */
+  config_line_prepend(&labels, "document-type", "consensus");
+  config_line_prepend(&labels, "confrooble-voolid-oofter",
+                      "2010-10-10T09:08:07");
+  ent = consensus_cache_add(cache, labels, (const uint8_t*)"Foo", 3);
+  tt_assert(ent);
+  consensus_cache_entry_decref(ent);
+
+  setup_capture_of_logs(LOG_DEBUG);
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  expect_log_msg_containing("Ignoring entry because it had no consensus-"
+                            "valid-after label");
+
+ done:
+  teardown_capture_of_logs();
+  config_free_lines(labels);
+}
+
+static void
+test_consdiffmgr_cleanup_old_diffs(void *arg)
+{
+  (void)arg;
+#define N 4
+  char *md_body[N];
+  networkstatus_t *md_ns[N];
+  int i;
+  consensus_cache_entry_t *hold_ent = NULL, *ent;
+
+  /* Make sure that the cleanup function removes diffs to the not-most-recent
+   * consensus. */
+
+  MOCK(cpuworker_queue_work, mock_cpuworker_queue_work);
+
+  /* Create a bunch of consensus things at 15-second intervals. */
+  time_t start = approx_time() - 120;
+  for (i = 0; i < N; ++i) {
+    time_t when = start + i * 15;
+    md_body[i] = fake_ns_body_new(FLAV_MICRODESC, when);
+    md_ns[i] = fake_ns_new(FLAV_MICRODESC, when);
+  }
+
+  /* add the first 3. */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[0], md_ns[0]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[1], md_ns[1]));
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[2], md_ns[2]));
+  /* Make diffs. */
+  consdiffmgr_rescan();
+  tt_ptr_op(NULL, OP_NE, fake_cpuworker_queue);
+  tt_int_op(2, OP_EQ, smartlist_len(fake_cpuworker_queue));
+  tt_int_op(0, OP_EQ, mock_cpuworker_run_work());
+  mock_cpuworker_handle_replies();
+  tt_ptr_op(NULL, OP_EQ, fake_cpuworker_queue);
+
+  /* Nothing is deletable now */
+  tt_int_op(0, OP_EQ, consdiffmgr_cleanup());
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[0], md_body[2]));
+  tt_int_op(0, OP_EQ,
+       lookup_apply_and_verify_diff(FLAV_MICRODESC, md_body[1], md_body[2]));
+
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
+            lookup_diff_from(&hold_ent, FLAV_MICRODESC, md_body[1]));
+  consensus_cache_entry_incref(hold_ent); // incref, so it is preserved.
+
+  /* Now add an even-more-recent consensus; this should make all previous
+   * diffs deletable, and make delete */
+  tt_int_op(0, OP_EQ, consdiffmgr_add_consensus(md_body[3], md_ns[3]));
+  tt_int_op(2 * n_diff_compression_methods() +
+            (n_consensus_compression_methods() - 1) , OP_EQ,
+            consdiffmgr_cleanup());
+
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
+  /* This one is marked deletable but still in the hashtable */
+  tt_int_op(CONSDIFF_AVAILABLE, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
+  tt_int_op(CONSDIFF_NOT_FOUND, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
+
+  /* Everything should be valid at this point */
+  tt_int_op(0, OP_EQ, consdiffmgr_validate());
+
+  /* And if we recan NOW, we'll purge the hashtable of the entries,
+   * and launch attempts to generate new ones */
+  consdiffmgr_rescan();
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[0]));
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[1]));
+  tt_int_op(CONSDIFF_IN_PROGRESS, OP_EQ,
+            lookup_diff_from(&ent, FLAV_MICRODESC, md_body[2]));
+
+  /* We're still holding on to this, though, so we can still map it! */
+  const uint8_t *t1 = NULL;
+  size_t s;
+  int r = consensus_cache_entry_get_body(hold_ent, &t1, &s);
+  tt_int_op(r, OP_EQ, 0);
+  tt_assert(t1);
+
+ done:
+  for (i = 0; i < N; ++i) {
+    tor_free(md_body[i]);
+    networkstatus_vote_free(md_ns[i]);
+  }
+  consensus_cache_entry_decref(hold_ent);
+  UNMOCK(cpuworker_queue_work);
+#undef N
+}
+
+static void
+test_consdiffmgr_validate(void *arg)
+{
+  (void)arg;
+  config_line_t *lines = NULL;
+  consensus_cache_entry_t *ent = NULL;
+  consensus_cache_t *cache = cdm_cache_get(); // violate abstraction barrier
+  smartlist_t *vals = smartlist_new();
+
+  /* Put these: objects in the cache: one with a good sha3, one with bad sha3,
+   * one with a wrong sha3, and one with no sha3. */
+  config_line_prepend(&lines, "id", "wrong sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "F00DF00DF00DF00DF00DF00DF00DF00D"
+                      "F00DF00DF00DF00DF00DF00DF00DF00D");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "bad sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "now is the winter of our dicotheque");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "no sha3");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  config_line_prepend(&lines, "id", "good sha3");
+  config_line_prepend(&lines, "sha3-digest",
+                      "8d8b1998616cd6b4c4055da8d38728dc"
+                      "93c758d4131a53c7d81aa6337dee1c05");
+  ent = consensus_cache_add(cache, lines, (const uint8_t *)"Hi there", 8);
+  consensus_cache_entry_decref(ent);
+  config_free_lines(lines);
+  lines = NULL;
+
+  cdm_reload();
+  cache = cdm_cache_get();
+  tt_int_op(1, OP_EQ, consdiffmgr_validate());
+
+  consensus_cache_find_all(vals, cache, "id", "good sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 1);
+  smartlist_clear(vals);
+
+  consensus_cache_find_all(vals, cache, "id", "no sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 1);
+  smartlist_clear(vals);
+
+  consensus_cache_find_all(vals, cache, "id", "wrong sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 0);
+  consensus_cache_find_all(vals, cache, "id", "bad sha3");
+  tt_int_op(smartlist_len(vals), OP_EQ, 0);
+
+ done:
+  smartlist_free(vals);
+}
+
+#define TEST(name)                                      \
+  { #name, test_consdiffmgr_ ## name , TT_FORK, &setup_diffmgr, NULL }
+
+struct testcase_t consdiffmgr_tests[] = {
+#if 0
+  { "init_failure", test_consdiffmgr_init_failure, TT_FORK, NULL, NULL },
+#endif
+  TEST(sha3_helper),
+  TEST(add),
+  TEST(make_diffs),
+  TEST(diff_rules),
+  TEST(diff_failure),
+  TEST(diff_pending),
+  TEST(cleanup_old),
+  TEST(cleanup_bad_valid_after),
+  TEST(cleanup_no_valid_after),
+  TEST(cleanup_old_diffs),
+  TEST(validate),
+
+  // XXXX Test: non-cacheing cases of replyfn().
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index d8b82e0661..54484a2a91 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -501,13 +501,13 @@ test_container_smartlist_pos(void *arg)
   (void) arg;
   smartlist_t *sl = smartlist_new();
 
-  smartlist_add(sl, tor_strdup("This"));
-  smartlist_add(sl, tor_strdup("is"));
-  smartlist_add(sl, tor_strdup("a"));
-  smartlist_add(sl, tor_strdup("test"));
-  smartlist_add(sl, tor_strdup("for"));
-  smartlist_add(sl, tor_strdup("a"));
-  smartlist_add(sl, tor_strdup("function"));
+  smartlist_add_strdup(sl, "This");
+  smartlist_add_strdup(sl, "is");
+  smartlist_add_strdup(sl, "a");
+  smartlist_add_strdup(sl, "test");
+  smartlist_add_strdup(sl, "for");
+  smartlist_add_strdup(sl, "a");
+  smartlist_add_strdup(sl, "function");
 
   /* Test string_pos */
   tt_int_op(smartlist_string_pos(NULL, "Fred"), ==, -1);
@@ -830,7 +830,7 @@ test_container_strmap(void *arg)
   found_keys = smartlist_new();
   while (!strmap_iter_done(iter)) {
     strmap_iter_get(iter,&k,&v);
-    smartlist_add(found_keys, tor_strdup(k));
+    smartlist_add_strdup(found_keys, k);
     tt_ptr_op(v,OP_EQ, strmap_get(map, k));
 
     if (!strcmp(k, "K2")) {
@@ -882,6 +882,46 @@ test_container_strmap(void *arg)
   tor_free(v105);
 }
 
+static void
+test_container_smartlist_remove(void *arg)
+{
+  (void) arg;
+  int array[5];
+  smartlist_t *sl = smartlist_new();
+  int i,j;
+
+  for (j=0; j < 2; ++j)
+    for (i=0; i < 5; ++i)
+      smartlist_add(sl, &array[i]);
+
+  smartlist_remove(sl, &array[0]);
+  smartlist_remove(sl, &array[3]);
+  smartlist_remove(sl, &array[4]);
+  tt_assert(! smartlist_contains(sl, &array[0]));
+  tt_assert(smartlist_contains(sl, &array[1]));
+  tt_assert(smartlist_contains(sl, &array[2]));
+  tt_assert(! smartlist_contains(sl, &array[3]));
+  tt_assert(! smartlist_contains(sl, &array[4]));
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+
+  smartlist_clear(sl);
+  for (j=0; j < 2; ++j)
+    for (i=0; i < 5; ++i)
+      smartlist_add(sl, &array[i]);
+
+  smartlist_remove_keeporder(sl, &array[0]);
+  smartlist_remove_keeporder(sl, &array[3]);
+  smartlist_remove_keeporder(sl, &array[4]);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  tt_ptr_op(smartlist_get(sl, 0), OP_EQ, &array[1]);
+  tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
+  tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for getting the median of a list. */
 static void
 test_container_order_functions(void *arg)
@@ -1239,6 +1279,7 @@ struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
+  CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index f19c846144..592f91a988 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,14 +1,16 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONTROL_PRIVATE
 #include "or.h"
+#include "bridges.h"
 #include "control.h"
 #include "entrynodes.h"
 #include "networkstatus.h"
 #include "rendservice.h"
 #include "routerlist.h"
 #include "test.h"
+#include "test_helpers.h"
 
 static void
 test_add_onion_helper_keyarg(void *arg)
@@ -107,6 +109,45 @@ test_add_onion_helper_keyarg(void *arg)
 }
 
 static void
+test_getinfo_helper_onion(void *arg)
+{
+  (void)arg;
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+  char *service_id = NULL;
+  int rt = 0;
+
+  dummy.ephemeral_onion_services = NULL;
+
+  /* successfully get an empty answer */
+  rt = getinfo_helper_onions(&dummy, "onions/current", &answer, &errmsg);
+  tt_assert(rt == 0);
+  tt_str_op(answer, OP_EQ, "");
+  tor_free(answer);
+
+  /* successfully get an empty answer */
+  rt = getinfo_helper_onions(&dummy, "onions/detached", &answer, &errmsg);
+  tt_assert(rt == 0);
+  tt_str_op(answer, OP_EQ, "");
+  tor_free(answer);
+
+  /* get an answer for one onion service */
+  service_id = tor_strdup("dummy_onion_id");
+  dummy.ephemeral_onion_services = smartlist_new();
+  smartlist_add(dummy.ephemeral_onion_services, service_id);
+  rt = getinfo_helper_onions(&dummy, "onions/current", &answer, &errmsg);
+  tt_assert(rt == 0);
+  tt_str_op(answer, OP_EQ, "dummy_onion_id");
+
+ done:
+  tor_free(answer);
+  tor_free(service_id);
+  smartlist_free(dummy.ephemeral_onion_services);
+}
+
+static void
 test_rend_service_parse_port_config(void *arg)
 {
   const char *sep = ",";
@@ -185,8 +226,10 @@ test_rend_service_parse_port_config(void *arg)
   tor_free(err_msg);
 
   /* bogus IP address */
-  cfg = rend_service_parse_port_config("100 1.2.3.4.5:9000",
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  cfg = rend_service_parse_port_config("100 foo!!.example.com:9000",
                                        " ", &err_msg);
+  UNMOCK(tor_addr_lookup);
   tt_assert(!cfg);
   tt_str_op(err_msg, OP_EQ, "Unparseable address in hidden service port "
             "configuration.");
@@ -1328,6 +1371,7 @@ test_download_status_bridge(void *arg)
 
 struct testcase_t controller_tests[] = {
   { "add_onion_helper_keyarg", test_add_onion_helper_keyarg, 0, NULL, NULL },
+  { "getinfo_helper_onion", test_getinfo_helper_onion, 0, NULL, NULL },
   { "rend_service_parse_port_config", test_rend_service_parse_port_config, 0,
     NULL, NULL },
   { "add_onion_helper_clientauth", test_add_onion_helper_clientauth, 0, NULL,
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 11e1e3dc8f..901ad7ab3d 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 8fd9ca7671..75cd30ebb5 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -15,9 +15,6 @@
 #include "crypto_ed25519.h"
 #include "ed25519_vectors.inc"
 
-#include <openssl/evp.h>
-#include <openssl/rand.h>
-
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
 test_crypto_dh(void *arg)
@@ -331,38 +328,6 @@ test_crypto_rng_strongest(void *arg)
 #undef N
 }
 
-/* Test for rectifying openssl RAND engine. */
-static void
-test_crypto_rng_engine(void *arg)
-{
-  (void)arg;
-  RAND_METHOD dummy_method;
-  memset(&dummy_method, 0, sizeof(dummy_method));
-
-  /* We should be a no-op if we're already on RAND_OpenSSL */
-  tt_int_op(0, ==, crypto_force_rand_ssleay());
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-
-  /* We should correct the method if it's a dummy. */
-  RAND_set_rand_method(&dummy_method);
-#ifdef LIBRESSL_VERSION_NUMBER
-  /* On libressl, you can't override the RNG. */
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-  tt_int_op(0, ==, crypto_force_rand_ssleay());
-#else
-  tt_assert(RAND_get_rand_method() == &dummy_method);
-  tt_int_op(1, ==, crypto_force_rand_ssleay());
-#endif
-  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
-
-  /* Make sure we aren't calling dummy_method */
-  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
-  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
-
- done:
-  ;
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -1135,6 +1100,54 @@ test_crypto_sha3_xof(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+/* Test our MAC-SHA3 function. There are not actually any MAC-SHA3 test
+ * vectors out there for our H(len(k) || k || m) construction. Hence what we
+ * are gonna do is test our crypto_mac_sha3_256() function against manually
+ * doing H(len(k) || k||m).  If in the future the Keccak group decides to
+ * standarize an MAC construction and make test vectors, we should
+ * incorporate them here. */
+static void
+test_crypto_mac_sha3(void *arg)
+{
+  const char msg[] = "i am in a library somewhere using my computer";
+  const char key[] = "i'm from the past talking to the future.";
+
+  uint8_t hmac_test[DIGEST256_LEN];
+  char hmac_manual[DIGEST256_LEN];
+
+  (void) arg;
+
+  /* First let's use our nice HMAC-SHA3 function */
+  crypto_mac_sha3_256(hmac_test, sizeof(hmac_test),
+                      (uint8_t *) key, strlen(key),
+                      (uint8_t *) msg, strlen(msg));
+
+  /* Now let's try a manual H(len(k) || k || m) construction */
+  {
+    char *key_msg_concat = NULL, *all = NULL;
+    int result;
+    const uint64_t key_len_netorder = tor_htonll(strlen(key));
+    size_t all_len;
+
+    tor_asprintf(&key_msg_concat, "%s%s", key, msg);
+    all_len = sizeof(key_len_netorder) + strlen(key_msg_concat);
+    all = tor_malloc_zero(all_len);
+    memcpy(all, &key_len_netorder, sizeof(key_len_netorder));
+    memcpy(all + sizeof(key_len_netorder), key_msg_concat,
+           strlen(key_msg_concat));
+
+    result = crypto_digest256(hmac_manual, all, all_len, DIGEST_SHA3_256);
+    tor_free(key_msg_concat);
+    tor_free(all);
+    tt_int_op(result, ==, 0);
+  }
+
+  /* Now compare the two results */
+  tt_mem_op(hmac_test, OP_EQ, hmac_manual, DIGEST256_LEN);
+
+ done: ;
+}
+
 /** Run unit tests for our public key crypto functions */
 static void
 test_crypto_pk(void *arg)
@@ -1469,28 +1482,6 @@ test_crypto_digest_names(void *arg)
   ;
 }
 
-#ifndef OPENSSL_1_1_API
-#define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX))
-#define EVP_ENCODE_CTX_free(ctx) tor_free(ctx)
-#endif
-
-/** Encode src into dest with OpenSSL's EVP Encode interface, returning the
- * length of the encoded data in bytes.
- */
-static int
-base64_encode_evp(char *dest, char *src, size_t srclen)
-{
-  const unsigned char *s = (unsigned char*)src;
-  EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
-  int len, ret;
-
-  EVP_EncodeInit(ctx);
-  EVP_EncodeUpdate(ctx, (unsigned char *)dest, &len, s, (int)srclen);
-  EVP_EncodeFinal(ctx, (unsigned char *)(dest + len), &ret);
-  EVP_ENCODE_CTX_free(ctx);
-  return ret+ len;
-}
-
 /** Run unit tests for misc crypto formatting functionality (base64, base32,
  * fingerprints, etc) */
 static void
@@ -1519,7 +1510,7 @@ test_crypto_formats(void *arg)
     tt_int_op(i, OP_GE, 0);
     tt_int_op(i, OP_EQ, strlen(data2));
     tt_assert(! strchr(data2, '='));
-    j = base64_decode_nopad((uint8_t*)data3, 1024, data2, i);
+    j = base64_decode(data3, 1024, data2, i);
     tt_int_op(j, OP_EQ, idx);
     tt_mem_op(data3,OP_EQ, data1, idx);
   }
@@ -1546,20 +1537,6 @@ test_crypto_formats(void *arg)
 
   tt_assert(digest_from_base64(data3, "###") < 0);
 
-  for (i = 0; i < 256; i++) {
-    /* Test the multiline format Base64 encoder with 0 .. 256 bytes of
-     * output against OpenSSL.
-     */
-    const size_t enclen = base64_encode_size(i, BASE64_ENCODE_MULTILINE);
-    data1[i] = i;
-    j = base64_encode(data2, 1024, data1, i, BASE64_ENCODE_MULTILINE);
-    tt_int_op(j, OP_EQ, enclen);
-    j = base64_encode_evp(data3, data1, i);
-    tt_int_op(j, OP_EQ, enclen);
-    tt_mem_op(data2, OP_EQ, data3, enclen);
-    tt_int_op(j, OP_EQ, strlen(data2));
-  }
-
   /* Encoding SHA256 */
   crypto_rand(data2, DIGEST256_LEN);
   memset(data2, 100, 1024);
@@ -2933,7 +2910,6 @@ struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
   CRYPTO_LEGACY(rng),
   { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL },
   { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
   { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
     &passthrough_setup, (void*)"nosyscall" },
@@ -2959,6 +2935,7 @@ struct testcase_t crypto_tests[] = {
   { "digest_names", test_crypto_digest_names, 0, NULL, NULL },
   { "sha3", test_crypto_sha3, TT_FORK, NULL, NULL},
   { "sha3_xof", test_crypto_sha3_xof, TT_FORK, NULL, NULL},
+  { "mac_sha3", test_crypto_mac_sha3, TT_FORK, NULL, NULL},
   CRYPTO_LEGACY(dh),
   { "aes_iv_AES", test_crypto_aes_iv, TT_FORK, &passthrough_setup,
     (void*)"aes" },
diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c
new file mode 100644
index 0000000000..3d7d2b4639
--- /dev/null
+++ b/src/test/test_crypto_openssl.c
@@ -0,0 +1,107 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define CRYPTO_PRIVATE
+
+#include "crypto.h"
+#include "util.h"
+#include "util_format.h"
+#include "compat.h"
+#include "test.h"
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+#include "compat_openssl.h"
+
+/* Test for rectifying openssl RAND engine. */
+static void
+test_crypto_rng_engine(void *arg)
+{
+  (void)arg;
+  RAND_METHOD dummy_method;
+  memset(&dummy_method, 0, sizeof(dummy_method));
+
+  /* We should be a no-op if we're already on RAND_OpenSSL */
+  tt_int_op(0, ==, crypto_force_rand_ssleay());
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+
+  /* We should correct the method if it's a dummy. */
+  RAND_set_rand_method(&dummy_method);
+#ifdef LIBRESSL_VERSION_NUMBER
+  /* On libressl, you can't override the RNG. */
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+  tt_int_op(0, ==, crypto_force_rand_ssleay());
+#else
+  tt_assert(RAND_get_rand_method() == &dummy_method);
+  tt_int_op(1, ==, crypto_force_rand_ssleay());
+#endif
+  tt_assert(RAND_get_rand_method() == RAND_OpenSSL());
+
+  /* Make sure we aren't calling dummy_method */
+  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
+  crypto_rand((void *) &dummy_method, sizeof(dummy_method));
+
+ done:
+  ;
+}
+
+#ifndef OPENSSL_1_1_API
+#define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX))
+#define EVP_ENCODE_CTX_free(ctx) tor_free(ctx)
+#endif
+
+/** Encode src into dest with OpenSSL's EVP Encode interface, returning the
+ * length of the encoded data in bytes.
+ */
+static int
+base64_encode_evp(char *dest, char *src, size_t srclen)
+{
+  const unsigned char *s = (unsigned char*)src;
+  EVP_ENCODE_CTX *ctx = EVP_ENCODE_CTX_new();
+  int len, ret;
+
+  EVP_EncodeInit(ctx);
+  EVP_EncodeUpdate(ctx, (unsigned char *)dest, &len, s, (int)srclen);
+  EVP_EncodeFinal(ctx, (unsigned char *)(dest + len), &ret);
+  EVP_ENCODE_CTX_free(ctx);
+  return ret+ len;
+}
+
+static void
+test_crypto_base64_encode_matches(void *arg)
+{
+  (void)arg;
+  int i, j;
+  char data1[1024];
+  char data2[1024];
+  char data3[1024];
+
+  for (i = 0; i < 256; i++) {
+    /* Test the multiline format Base64 encoder with 0 .. 256 bytes of
+     * output against OpenSSL.
+     */
+    const size_t enclen = base64_encode_size(i, BASE64_ENCODE_MULTILINE);
+    data1[i] = i;
+    j = base64_encode(data2, 1024, data1, i, BASE64_ENCODE_MULTILINE);
+    tt_int_op(j, OP_EQ, enclen);
+    j = base64_encode_evp(data3, data1, i);
+    tt_int_op(j, OP_EQ, enclen);
+    tt_mem_op(data2, OP_EQ, data3, enclen);
+    tt_int_op(j, OP_EQ, strlen(data2));
+  }
+
+ done:
+  ;
+}
+
+struct testcase_t crypto_openssl_tests[] = {
+  { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL },
+  { "base64_encode_match", test_crypto_base64_encode_matches,
+    TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index 0be58c9389..75c6ba9aaa 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_data.c b/src/test/test_data.c
index 788489a097..ce6c3394f6 100644
--- a/src/test/test_data.c
+++ b/src/test/test_data.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "test.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index ad5f086434..1ead6d78c2 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -1,12 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <math.h>
 
 #define CONFIG_PRIVATE
+#define CONTROL_PRIVATE
 #define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
 #define ROUTER_PRIVATE
@@ -19,10 +20,12 @@
 #include "or.h"
 #include "confparse.h"
 #include "config.h"
+#include "control.h"
 #include "crypto_ed25519.h"
 #include "directory.h"
 #include "dirserv.h"
 #include "dirvote.h"
+#include "entrynodes.h"
 #include "hibernate.h"
 #include "memarea.h"
 #include "networkstatus.h"
@@ -328,7 +331,7 @@ test_dir_formats(void *arg)
     ntor_cc = make_ntor_onion_key_crosscert(&r2_onion_keypair,
                                           &kp1.pubkey,
                                           r2->cache_info.published_on,
-                                          MIN_ONION_KEY_LIFETIME,
+                                          get_onion_key_lifetime(),
                                           &ntor_cc_sign);
     tt_assert(ntor_cc);
     base64_encode(cert_buf, sizeof(cert_buf),
@@ -678,16 +681,16 @@ test_dir_parse_router_list(void *arg)
   routerinfo_t *ri = NULL;
   char d[DIGEST_LEN];
 
-  smartlist_add(chunks, tor_strdup(EX_RI_MINIMAL));     // ri 0
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_PORTS));   // bad ri 0
-  smartlist_add(chunks, tor_strdup(EX_EI_MAXIMAL));     // ei 0
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_SIG2));    // bad ei --
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_NICKNAME));// bad ei 0
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_SIG1));    // bad ri --
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_PUBLISHED));  // bad ei 1
-  smartlist_add(chunks, tor_strdup(EX_RI_MAXIMAL));     // ri 1
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_FAMILY));  // bad ri 1
-  smartlist_add(chunks, tor_strdup(EX_EI_MINIMAL));     // ei 1
+  smartlist_add_strdup(chunks, EX_RI_MINIMAL);     // ri 0
+  smartlist_add_strdup(chunks, EX_RI_BAD_PORTS);   // bad ri 0
+  smartlist_add_strdup(chunks, EX_EI_MAXIMAL);     // ei 0
+  smartlist_add_strdup(chunks, EX_EI_BAD_SIG2);    // bad ei --
+  smartlist_add_strdup(chunks, EX_EI_BAD_NICKNAME);// bad ei 0
+  smartlist_add_strdup(chunks, EX_RI_BAD_SIG1);    // bad ri --
+  smartlist_add_strdup(chunks, EX_EI_BAD_PUBLISHED);  // bad ei 1
+  smartlist_add_strdup(chunks, EX_RI_MAXIMAL);     // ri 1
+  smartlist_add_strdup(chunks, EX_RI_BAD_FAMILY);  // bad ri 1
+  smartlist_add_strdup(chunks, EX_EI_MINIMAL);     // ei 1
 
   list = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -812,19 +815,19 @@ test_dir_load_routers(void *arg)
 #define ADD(str)                                                        \
   do {                                                                  \
     tt_int_op(0,OP_EQ,router_get_router_hash(str, strlen(str), buf));      \
-    smartlist_add(wanted, tor_strdup(hex_str(buf, DIGEST_LEN)));        \
+    smartlist_add_strdup(wanted, hex_str(buf, DIGEST_LEN));        \
   } while (0)
 
   MOCK(router_get_dl_status_by_descriptor_digest, mock_router_get_dl_status);
 
   update_approx_time(1412510400);
 
-  smartlist_add(chunks, tor_strdup(EX_RI_MINIMAL));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_FINGERPRINT));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_SIG2));
-  smartlist_add(chunks, tor_strdup(EX_RI_MAXIMAL));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_PORTS));
-  smartlist_add(chunks, tor_strdup(EX_RI_BAD_TOKENS));
+  smartlist_add_strdup(chunks, EX_RI_MINIMAL);
+  smartlist_add_strdup(chunks, EX_RI_BAD_FINGERPRINT);
+  smartlist_add_strdup(chunks, EX_RI_BAD_SIG2);
+  smartlist_add_strdup(chunks, EX_RI_MAXIMAL);
+  smartlist_add_strdup(chunks, EX_RI_BAD_PORTS);
+  smartlist_add_strdup(chunks, EX_RI_BAD_TOKENS);
 
   /* not ADDing MINIMIAL */
   ADD(EX_RI_MAXIMAL);
@@ -909,6 +912,23 @@ mock_get_by_ei_desc_digest(const char *d)
   }
 }
 
+static signed_descriptor_t *
+mock_ei_get_by_ei_digest(const char *d)
+{
+  char hex[HEX_DIGEST_LEN+1];
+  base16_encode(hex, sizeof(hex), d, DIGEST_LEN);
+  signed_descriptor_t *sd = &sd_ei_minimal;
+
+  if (!strcmp(hex, "11E0EDF526950739F7769810FCACAB8C882FAEEE")) {
+    sd->signed_descriptor_body = (char *)EX_EI_MINIMAL;
+    sd->signed_descriptor_len = sizeof(EX_EI_MINIMAL);
+    sd->annotations_len = 0;
+    sd->saved_location = SAVED_NOWHERE;
+    return sd;
+  }
+  return NULL;
+}
+
 static smartlist_t *mock_ei_insert_list = NULL;
 static was_router_added_t
 mock_ei_insert(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)
@@ -932,18 +952,18 @@ test_dir_load_extrainfo(void *arg)
 #define ADD(str)                                                        \
   do {                                                                  \
     tt_int_op(0,OP_EQ,router_get_extrainfo_hash(str, strlen(str), buf));   \
-    smartlist_add(wanted, tor_strdup(hex_str(buf, DIGEST_LEN)));        \
+    smartlist_add_strdup(wanted, hex_str(buf, DIGEST_LEN));        \
   } while (0)
 
   mock_ei_insert_list = smartlist_new();
   MOCK(router_get_by_extrainfo_digest, mock_get_by_ei_desc_digest);
   MOCK(extrainfo_insert, mock_ei_insert);
 
-  smartlist_add(chunks, tor_strdup(EX_EI_MINIMAL));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_NICKNAME));
-  smartlist_add(chunks, tor_strdup(EX_EI_MAXIMAL));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_PUBLISHED));
-  smartlist_add(chunks, tor_strdup(EX_EI_BAD_TOKENS));
+  smartlist_add_strdup(chunks, EX_EI_MINIMAL);
+  smartlist_add_strdup(chunks, EX_EI_BAD_NICKNAME);
+  smartlist_add_strdup(chunks, EX_EI_MAXIMAL);
+  smartlist_add_strdup(chunks, EX_EI_BAD_PUBLISHED);
+  smartlist_add_strdup(chunks, EX_EI_BAD_TOKENS);
 
   /* not ADDing MINIMIAL */
   ADD(EX_EI_MAXIMAL);
@@ -998,6 +1018,37 @@ test_dir_load_extrainfo(void *arg)
 }
 
 static void
+test_dir_getinfo_extra(void *arg)
+{
+  int r;
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+  MOCK(extrainfo_get_by_descriptor_digest, mock_ei_get_by_ei_digest);
+  r = getinfo_helper_dir(NULL, "extra-info/digest/"
+                         "11E0EDF526950739F7769810FCACAB8C882FAEEE", &answer,
+                         &errmsg);
+  tt_int_op(0, OP_EQ, r);
+  tt_ptr_op(NULL, OP_EQ, errmsg);
+  tt_str_op(answer, OP_EQ, EX_EI_MINIMAL);
+  tor_free(answer);
+
+  answer = NULL;
+  r = getinfo_helper_dir(NULL, "extra-info/digest/"
+                         "NOTAVALIDHEXSTRINGNOTAVALIDHEXSTRINGNOTA", &answer,
+                         &errmsg);
+  tt_int_op(0, OP_EQ, r);
+  /* getinfo_helper_dir() should maybe return an error here but doesn't */
+  tt_ptr_op(NULL, OP_EQ, errmsg);
+  /* In any case, there should be no answer for an invalid hex string. */
+  tt_ptr_op(NULL, OP_EQ, answer);
+
+ done:
+  UNMOCK(extrainfo_get_by_descriptor_digest);
+}
+
+static void
 test_dir_versions(void *arg)
 {
   tor_version_t ver1;
@@ -1064,6 +1115,7 @@ test_dir_versions(void *arg)
   tt_int_op(0, OP_EQ, ver1.patchlevel);
   tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
   tt_str_op("alpha", OP_EQ, ver1.status_tag);
+  /* Go through the full set of status tags */
   tt_int_op(0, OP_EQ, tor_version_parse("2.1.700-alpha", &ver1));
   tt_int_op(2, OP_EQ, ver1.major);
   tt_int_op(1, OP_EQ, ver1.minor);
@@ -1078,6 +1130,60 @@ test_dir_versions(void *arg)
   tt_int_op(0, OP_EQ, ver1.patchlevel);
   tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
   tt_str_op("alpha-dev", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.5-rc", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(5, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("rc", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.6-rc-dev", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(6, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("rc-dev", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.8", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(8, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("", OP_EQ, ver1.status_tag);
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2.9.9-dev", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2, OP_EQ, ver1.minor);
+  tt_int_op(9, OP_EQ, ver1.micro);
+  tt_int_op(9, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("dev", OP_EQ, ver1.status_tag);
+  /* In #21450, we fixed an inconsistency in parsing versions > INT32_MAX
+   * between i386 and x86_64, as we used tor_parse_long, and then cast to int
+   */
+  tt_int_op(0, OP_EQ, tor_version_parse("0.2147483647.0", &ver1));
+  tt_int_op(0, OP_EQ, ver1.major);
+  tt_int_op(2147483647, OP_EQ, ver1.minor);
+  tt_int_op(0, OP_EQ, ver1.micro);
+  tt_int_op(0, OP_EQ, ver1.patchlevel);
+  tt_int_op(VER_RELEASE, OP_EQ, ver1.status);
+  tt_str_op("", OP_EQ, ver1.status_tag);
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.2147483648.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.4294967295.0", &ver1));
+  /* In #21278, we reject negative version components */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-1.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-2147483648.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-4294967295.0", &ver1));
+  /* In #21507, we reject version components with non-numeric prefixes */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.-0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("+1.0.0", &ver1));
+  /* use the list in isspace() */
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\t0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\n0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\v0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\f0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0.\r0.0", &ver1));
+  tt_int_op(-1, OP_EQ, tor_version_parse("0. 0.0", &ver1));
 
 #define tt_versionstatus_op(vs1, op, vs2)                               \
   tt_assert_test_type(vs1,vs2,#vs1" "#op" "#vs2,version_status_t,       \
@@ -1097,6 +1203,7 @@ test_dir_versions(void *arg)
   test_v_i_o(VS_RECOMMENDED, "0.0.7rc2", "0.0.7,Tor 0.0.7rc2,Tor 0.0.8");
   test_v_i_o(VS_OLD, "0.0.5.0", "0.0.5.1-cvs");
   test_v_i_o(VS_NEW_IN_SERIES, "0.0.5.1-cvs", "0.0.5, 0.0.6");
+  test_v_i_o(VS_NEW, "0.2.9.9-dev", "0.2.9.9");
   /* Not on list, but newer than any in same series. */
   test_v_i_o(VS_NEW_IN_SERIES, "0.1.0.3",
              "Tor 0.1.0.2,Tor 0.0.9.5,Tor 0.1.1.0");
@@ -1135,6 +1242,70 @@ test_dir_versions(void *arg)
                                    "Tor 0.2.1.0-dev (r99)"));
   tt_int_op(1,OP_EQ, tor_version_as_new_as("Tor 0.2.1.1",
                                    "Tor 0.2.1.0-dev (r99)"));
+  /* And git revisions */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-56788a2489127072)",
+                                        "Tor 0.2.9.9 (git-56788a2489127072)"));
+  /* a git revision is newer than no git revision */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-56788a2489127072)",
+                                        "Tor 0.2.9.9"));
+  /* a longer git revision is newer than a shorter git revision
+   * this should be true if they prefix-match, but if they don't, they are
+   * incomparable, because hashes aren't ordered (but we compare their bytes
+   * anyway) */
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                  "Tor 0.2.9.9 (git-56788a2489127072d513cf4baf35a8ff475f3c7b)",
+                  "Tor 0.2.9.9 (git-56788a2489127072)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-0102)",
+                                        "Tor 0.2.9.9 (git-03)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                        "Tor 0.2.9.9 (git-0102)",
+                                        "Tor 0.2.9.9 (git-00)"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2.9.9 (git-01)",
+                                           "Tor 0.2.9.9 (git-00)"));
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2.9.9 (git-00)",
+                                           "Tor 0.2.9.9 (git-01)"));
+  /* In #21278, we comapre without integer overflows.
+   * But since #21450 limits version components to [0, INT32_MAX], it is no
+   * longer possible to cause an integer overflow in tor_version_compare() */
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.0.0.0",
+                                           "Tor 2147483647.0.0.0"));
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 2147483647.0.0.0",
+                                           "Tor 0.0.0.0"));
+  /* These versions used to cause an overflow, now they don't parse
+   * (and authorities reject their descriptors), and log a BUG message */
+  setup_full_capture_of_logs(LOG_WARN);
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.0.0.0",
+                                           "Tor 0.-2147483648.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2147483647.0.0",
+                                           "Tor 0.-1.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.2147483647.0.0",
+                                           "Tor 0.-2147483648.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  tt_int_op(1,OP_EQ, tor_version_as_new_as(
+                                           "Tor 4294967295.0.0.0",
+                                           "Tor 0.0.0.0"));
+  expect_no_log_entry();
+  tt_int_op(0,OP_EQ, tor_version_as_new_as(
+                                           "Tor 0.4294967295.0.0",
+                                           "Tor 0.-4294967295.0.0"));
+  expect_single_log_msg_containing("unparseable");
+  mock_clean_saved_logs();
+  teardown_capture_of_logs();
 
   /* Now try git revisions */
   tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00ff)", &ver1));
@@ -1144,11 +1315,24 @@ test_dir_versions(void *arg)
   tt_int_op(7,OP_EQ, ver1.patchlevel);
   tt_int_op(3,OP_EQ, ver1.git_tag_len);
   tt_mem_op(ver1.git_tag,OP_EQ, "\xff\x00\xff", 3);
+  /* reject bad hex digits */
   tt_int_op(-1,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00xx)", &ver1));
+  /* reject odd hex digit count */
   tt_int_op(-1,OP_EQ, tor_version_parse("0.5.6.7 (git-ff00fff)", &ver1));
+  /* ignore "git " */
   tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git ff00fff)", &ver1));
+  /* standard length is 16 hex digits */
+  tt_int_op(0,OP_EQ, tor_version_parse("0.5.6.7 (git-0010203040506070)",
+                                       &ver1));
+  /* length limit is 40 hex digits */
+  tt_int_op(0,OP_EQ, tor_version_parse(
+                     "0.5.6.7 (git-000102030405060708090a0b0c0d0e0f10111213)",
+                     &ver1));
+  tt_int_op(-1,OP_EQ, tor_version_parse(
+                    "0.5.6.7 (git-000102030405060708090a0b0c0d0e0f1011121314)",
+                    &ver1));
  done:
-  ;
+  teardown_capture_of_logs();
 }
 
 /** Run unit tests for directory fp_pair functions. */
@@ -1512,6 +1696,15 @@ test_dir_param_voting(void *arg)
   tt_int_op(-8,OP_EQ, networkstatus_get_param(&vote4, "ab", -12, -100, -8));
   tt_int_op(0,OP_EQ, networkstatus_get_param(&vote4, "foobar", 0, -100, 8));
 
+  tt_int_op(100,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -1, "x-yz", 50, 0, 300));
+  tt_int_op(30,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, 30, "x-yz", 50, 0, 300));
+  tt_int_op(0,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -101, "foobar", 0, -100, 8));
+  tt_int_op(-99,OP_EQ, networkstatus_get_overridable_param(
+                                        &vote4, -99, "foobar", 0, -100, 8));
+
   smartlist_add(votes, &vote1);
 
   /* Do the first tests without adding all the other votes, for
@@ -1893,6 +2086,249 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now)
   return;
 }
 
+static void
+test_dir_networkstatus_compute_bw_weights_v10(void *arg)
+{
+  (void) arg;
+  smartlist_t *chunks = smartlist_new();
+  int64_t G, M, E, D, T, weight_scale;
+  int ret;
+  weight_scale = 10000;
+
+  /* no case. one or more of the values is 0 */
+  G = M = E = D = 0;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(smartlist_len(chunks), OP_EQ, 0);
+
+  /* case 1 */
+  /* XXX dir-spec not followed? See #20272. If it isn't closed, then this is
+   * testing current behavior, not spec. */
+  G = E = 10;
+  M = D = 1;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_int_op(smartlist_len(chunks), OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=3333 "
+    "Wbe=3000 Wbg=3000 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=7000 "
+    "Weg=3333 Wem=7000 Wgb=10000 Wgd=3333 Wgg=7000 Wgm=7000 Wmb=10000 "
+    "Wmd=3333 Wme=3000 Wmg=3000 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2a E scarce */
+  M = 100;
+  G = 20;
+  E = D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2a G scarce */
+  M = 100;
+  E = 20;
+  G = D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=10000 Weg=0 Wem=10000 "
+    "Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 Wme=0 Wmg=0 "
+    "Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b1 (Wgg=1, Wmd=Wgd) */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 100;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=4000 "
+    "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=2000 Wee=10000 Weg=2000 "
+    "Wem=10000 Wgb=10000 Wgd=4000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=4000 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b2 */
+  M = 60;
+  E = 30;
+  G = 10;
+  D = 100;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=666 Wbe=0 "
+    "Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3666 Wee=10000 Weg=3666 "
+    "Wem=10000 Wgb=10000 Wgd=5668 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=666 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 2b3 */
+  /* XXX I can't get a combination of values that hits this case without error,
+   * so this just tests that it fails. See #20285. Also see #20284 as 2b3 does
+   * not follow dir-spec. */
+  /* (E < T/3 && G < T/3) && (E+D>=G || G+D>=E) && (M > T/3) */
+  M = 80;
+  E = 30;
+  G = 30;
+  D = 30;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 0);
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3a G scarce */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 "
+    "Wbe=3333 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=6667 Weg=0 "
+    "Wem=6667 Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 "
+    "Wme=3333 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3a E scarce */
+  M = 10;
+  E = 10;
+  G = 30;
+  D = 5;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=3333 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=6667 Wgm=6667 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=3333 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3bg */
+  M = 10;
+  E = 30;
+  G = 10;
+  D = 10;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 "
+    "Wbe=3334 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=0 Wee=6666 Weg=0 "
+    "Wem=6666 Wgb=10000 Wgd=10000 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=0 "
+    "Wme=3334 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case 3be */
+  M = 10;
+  E = 10;
+  G = 30;
+  D = 10;
+  T = G + M + E + D;
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_int_op(ret, OP_EQ, 1);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=3334 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=6666 Wgm=6666 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=3334 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 21 Jul 2013 (3be) */
+  G = 5483409;
+  M = 1455379;
+  E = 980834;
+  D = 3385803;
+  T = 11305425;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=883 Wbe=0 "
+    "Wbg=3673 Wbm=10000 Wdb=10000 Web=10000 Wed=8233 Wee=10000 Weg=8233 "
+    "Wem=10000 Wgb=10000 Wgd=883 Wgg=6327 Wgm=6327 Wmb=10000 Wmd=883 Wme=0 "
+    "Wmg=3673 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 04 Oct 2016 (3a E scarce) */
+  G=29322240;
+  M=4721546;
+  E=1522058;
+  D=9273571;
+  T=44839415;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=0 Wbe=0 "
+    "Wbg=4194 Wbm=10000 Wdb=10000 Web=10000 Wed=10000 Wee=10000 Weg=10000 "
+    "Wem=10000 Wgb=10000 Wgd=0 Wgg=5806 Wgm=5806 Wmb=10000 Wmd=0 Wme=0 "
+    "Wmg=4194 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* case from 04 Sep 2013 (2b1) */
+  G=3091352;
+  M=1838837;
+  E=2109300;
+  D=2469369;
+  T=9508858;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=317 "
+    "Wbe=5938 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=9366 Wee=4061 "
+    "Weg=9366 Wem=4061 Wgb=10000 Wgd=317 Wgg=10000 Wgm=10000 Wmb=10000 "
+    "Wmd=317 Wme=5938 Wmg=0 Wmm=10000\n");
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_clear(chunks);
+
+  /* explicitly test initializing weights to 1*/
+  G=1;
+  M=1;
+  E=1;
+  D=1;
+  T=4;
+  tt_i64_op(G+M+E+D, OP_EQ, T);
+  ret = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T,
+                                             weight_scale);
+  tt_str_op(smartlist_get(chunks, 0), OP_EQ, "bandwidth-weights Wbd=3333 "
+    "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=3333 Wee=10000 Weg=3333 "
+    "Wem=10000 Wgb=10000 Wgd=3333 Wgg=10000 Wgm=10000 Wmb=10000 Wmd=3333 "
+    "Wme=0 Wmg=0 Wmm=10000\n");
+
+ done:
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+}
+
 static authority_cert_t *mock_cert;
 
 static authority_cert_t *
@@ -3272,17 +3708,88 @@ test_dir_http_handling(void *args)
 }
 
 static void
-test_dir_purpose_needs_anonymity(void *arg)
+test_dir_purpose_needs_anonymity_returns_true_by_default(void *arg)
+{
+  (void)arg;
+
+  tor_capture_bugs_(1);
+  setup_full_capture_of_logs(LOG_WARN);
+  tt_int_op(1, ==, purpose_needs_anonymity(0, 0, NULL));
+  tt_int_op(1, ==, smartlist_len(tor_get_captured_bug_log_()));
+  expect_single_log_msg_containing("Called with dir_purpose=0");
+
+  tor_end_capture_bugs_();
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_true_for_bridges(void *arg)
+{
+  (void)arg;
+
+  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE, NULL));
+  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE,
+                                           "foobar"));
+  tt_int_op(1, ==, purpose_needs_anonymity(DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2,
+                                           ROUTER_PURPOSE_BRIDGE, NULL));
+ done: ;
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_false_for_own_bridge_desc(void *arg)
+{
+  (void)arg;
+  tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_FETCH_SERVERDESC,
+                                           ROUTER_PURPOSE_BRIDGE,
+                                           "authority.z"));
+ done: ;
+}
+
+static void
+test_dir_purpose_needs_anonymity_returns_true_for_sensitive_purpose(void *arg)
 {
   (void)arg;
-  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE));
-  tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_GENERAL));
-  tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC,
-                                            ROUTER_PURPOSE_GENERAL));
+
+  tt_int_op(1, ==, purpose_needs_anonymity(
+                    DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2,
+                    ROUTER_PURPOSE_GENERAL, NULL));
+  tt_int_op(1, ==, purpose_needs_anonymity(
+                    DIR_PURPOSE_UPLOAD_RENDDESC_V2, 0,  NULL));
+  tt_int_op(1, ==, purpose_needs_anonymity(
+                    DIR_PURPOSE_FETCH_RENDDESC_V2, 0, NULL));
  done: ;
 }
 
 static void
+test_dir_purpose_needs_anonymity_ret_false_for_non_sensitive_conn(void *arg)
+{
+  (void)arg;
+
+  tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_DIR,
+                                           ROUTER_PURPOSE_GENERAL, NULL));
+  tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_VOTE, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_UPLOAD_SIGNATURES, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL));
+  tt_int_op(0, ==, purpose_needs_anonymity(
+                    DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_CONSENSUS, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_SERVERDESC, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_EXTRAINFO, 0, NULL));
+  tt_int_op(0, ==,
+            purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC, 0, NULL));
+  done: ;
+}
+
+static void
 test_dir_fetch_type(void *arg)
 {
   (void)arg;
@@ -4054,7 +4561,6 @@ test_dir_should_use_directory_guards(void *data)
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
   tt_int_op(CALLED(public_server_mode), OP_EQ, 1);
 
-  options->UseEntryGuardsAsDirGuards = 1;
   options->UseEntryGuards = 1;
   options->DownloadExtraInfo = 0;
   options->FetchDirInfoEarly = 0;
@@ -4068,29 +4574,24 @@ test_dir_should_use_directory_guards(void *data)
   tt_int_op(CALLED(public_server_mode), OP_EQ, 3);
   options->UseEntryGuards = 1;
 
-  options->UseEntryGuardsAsDirGuards = 0;
-  tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
-  options->UseEntryGuardsAsDirGuards = 1;
-
   options->DownloadExtraInfo = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
   options->DownloadExtraInfo = 0;
 
   options->FetchDirInfoEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
   options->FetchDirInfoEarly = 0;
 
   options->FetchDirInfoExtraEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
   options->FetchDirInfoExtraEarly = 0;
 
   options->FetchUselessDescriptors = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 8);
+  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
   options->FetchUselessDescriptors = 0;
 
   done:
@@ -4100,14 +4601,7 @@ test_dir_should_use_directory_guards(void *data)
 }
 
 NS_DECL(void,
-directory_initiate_command_routerstatus, (const routerstatus_t *status,
-                                          uint8_t dir_purpose,
-                                          uint8_t router_purpose,
-                                          dir_indirection_t indirection,
-                                          const char *resource,
-                                          const char *payload,
-                                          size_t payload_len,
-                                          time_t if_modified_since));
+directory_initiate_request, (directory_request_t *req));
 
 static void
 test_dir_should_not_init_request_to_ourselves(void *data)
@@ -4117,7 +4611,7 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   crypto_pk_t *key = pk_generate(2);
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4132,15 +4626,15 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   dir_server_add(ourself);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
 
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
     crypto_pk_free(key);
@@ -4154,7 +4648,7 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
                                 | MICRODESC_DIRINFO;
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4165,14 +4659,14 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 0);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
@@ -4183,7 +4677,7 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_t *ds = NULL;
   (void) data;
 
-  NS_MOCK(directory_initiate_command_routerstatus);
+  NS_MOCK(directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4194,37 +4688,23 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 1);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 1);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_command_routerstatus), OP_EQ, 2);
+  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 2);
 
   done:
-    NS_UNMOCK(directory_initiate_command_routerstatus);
+    NS_UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
 
 void
-NS(directory_initiate_command_routerstatus)(const routerstatus_t *status,
-                                            uint8_t dir_purpose,
-                                            uint8_t router_purpose,
-                                            dir_indirection_t indirection,
-                                            const char *resource,
-                                            const char *payload,
-                                            size_t payload_len,
-                                            time_t if_modified_since)
+NS(directory_initiate_request)(directory_request_t *req)
 {
-  (void)status;
-  (void)dir_purpose;
-  (void)router_purpose;
-  (void)indirection;
-  (void)resource;
-  (void)payload;
-  (void)payload_len;
-  (void)if_modified_since;
-  CALLED(directory_initiate_command_routerstatus)++;
+  (void)req;
+  CALLED(directory_initiate_request)++;
 }
 
 static void
@@ -5167,9 +5647,9 @@ listdir_mock(const char *dname)
   (void)dname;
 
   l = smartlist_new();
-  smartlist_add(l, tor_strdup("foo"));
-  smartlist_add(l, tor_strdup("bar"));
-  smartlist_add(l, tor_strdup("baz"));
+  smartlist_add_strdup(l, "foo");
+  smartlist_add_strdup(l, "bar");
+  smartlist_add_strdup(l, "baz");
 
   return l;
 }
@@ -5456,6 +5936,67 @@ test_dir_assumed_flags(void *arg)
   routerstatus_free(rs);
 }
 
+static void
+test_dir_post_parsing(void *arg)
+{
+  (void) arg;
+
+  /* Test the version parsing from an HS descriptor publish request. */
+  {
+    const char *end;
+    const char *prefix = "/tor/hs/";
+    int version = parse_hs_version_from_post("/tor/hs//publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/a/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, 3);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("/tor/hs/42/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, 42);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("/tor/hs/18163/publish",prefix, &end);
+    tt_int_op(version, OP_EQ, 18163);
+    tt_str_op(end, OP_EQ, "/publish");
+    version = parse_hs_version_from_post("JUNKJUNKJUNK", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish", "blah", &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    /* Missing the '/' at the end of the prefix. */
+    version = parse_hs_version_from_post("/tor/hs/3/publish", "/tor/hs", &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/random/blah/tor/hs/3/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    version = parse_hs_version_from_post("/tor/hs/3/publish/random/junk",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, 3);
+    tt_str_op(end, OP_EQ, "/publish/random/junk");
+    version = parse_hs_version_from_post("/tor/hs/-1/publish", prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+    /* INT_MAX */
+    version = parse_hs_version_from_post("/tor/hs/2147483647/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, INT_MAX);
+    tt_str_op(end, OP_EQ, "/publish");
+    /* INT_MAX + 1*/
+    version = parse_hs_version_from_post("/tor/hs/2147483648/publish",
+                                         prefix, &end);
+    tt_int_op(version, OP_EQ, -1);
+    tt_ptr_op(end, OP_EQ, NULL);
+  }
+
+ done:
+  ;
+}
+
 #define DIR_LEGACY(name)                             \
   { #name, test_dir_ ## name , TT_FORK, NULL, NULL }
 
@@ -5474,6 +6015,7 @@ struct testcase_t dir_tests[] = {
   DIR(parse_router_list, TT_FORK),
   DIR(load_routers, TT_FORK),
   DIR(load_extrainfo, TT_FORK),
+  DIR(getinfo_extra, 0),
   DIR_LEGACY(versions),
   DIR_LEGACY(fp_pairs),
   DIR(split_fps, 0),
@@ -5490,7 +6032,12 @@ struct testcase_t dir_tests[] = {
   DIR(fmt_control_ns, 0),
   DIR(dirserv_set_routerstatus_testing, 0),
   DIR(http_handling, 0),
-  DIR(purpose_needs_anonymity, 0),
+  DIR(purpose_needs_anonymity_returns_true_for_bridges, 0),
+  DIR(purpose_needs_anonymity_returns_false_for_own_bridge_desc, 0),
+  DIR(purpose_needs_anonymity_returns_true_by_default, 0),
+  DIR(purpose_needs_anonymity_returns_true_for_sensitive_purpose, 0),
+  DIR(purpose_needs_anonymity_ret_false_for_non_sensitive_conn, 0),
+  DIR(post_parsing, 0),
   DIR(fetch_type, 0),
   DIR(packages, 0),
   DIR(download_status_schedule, 0),
@@ -5511,6 +6058,7 @@ struct testcase_t dir_tests[] = {
   DIR_ARG(find_dl_schedule, TT_FORK, "cf"),
   DIR_ARG(find_dl_schedule, TT_FORK, "ca"),
   DIR(assumed_flags, 0),
+  DIR(networkstatus_compute_bw_weights_v10, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index ca43dd4c04..fca70249bd 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index 9682b0db49..65b9cf6436 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index a0f22f1f0c..75fe6249ad 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define RENDCOMMON_PRIVATE
@@ -12,8 +12,10 @@
 #include "or.h"
 #include "config.h"
 #include "connection.h"
+#include "consdiffmgr.h"
 #include "directory.h"
 #include "test.h"
+#include "compress.h"
 #include "connection.h"
 #include "rendcommon.h"
 #include "rendcache.h"
@@ -28,8 +30,8 @@
 #include "networkstatus.h"
 #include "geoip.h"
 #include "dirserv.h"
-#include "torgzip.h"
 #include "dirvote.h"
+#include "log_test_helpers.h"
 
 #ifdef _WIN32
 /* For mkdir() */
@@ -50,22 +52,10 @@ ENABLE_GCC_WARNING(overlength-strings)
 
 #define NS_MODULE dir_handle_get
 
-static void
-connection_write_to_buf_mock(const char *string, size_t len,
-                             connection_t *conn, int zlib)
-{
-  (void) zlib;
-
-  tor_assert(string);
-  tor_assert(conn);
-
-  write_to_buf(string, len, conn->outbuf);
-}
-
-#define GET(path) "GET " path " HTTP/1.0\r\n\r\n"
 #define NOT_FOUND "HTTP/1.0 404 Not found\r\n\r\n"
 #define BAD_REQUEST "HTTP/1.0 400 Bad request\r\n\r\n"
 #define SERVER_BUSY "HTTP/1.0 503 Directory busy, try again later\r\n\r\n"
+#define TOO_OLD "HTTP/1.0 404 Consensus is too old\r\n\r\n"
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
@@ -74,6 +64,7 @@ new_dir_conn(void)
 {
   dir_connection_t *conn = dir_connection_new(AF_INET);
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
   return conn;
 }
 
@@ -476,6 +467,8 @@ init_mock_options(void)
   mock_options = tor_malloc(sizeof(or_options_t));
   memset(mock_options, 0, sizeof(or_options_t));
   mock_options->TestingTorNetwork = 1;
+  mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
+  check_private_dir(mock_options->DataDirectory, CPD_CREATE, NULL);
 }
 
 static const or_options_t *
@@ -512,14 +505,6 @@ test_dir_handle_get_micro_d(void *data)
 
   /* SETUP */
   init_mock_options();
-  const char *fn = get_fname("dir_handle_datadir_test1");
-  mock_options->DataDirectory = tor_strdup(fn);
-
-#ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory));
-#else
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700));
-#endif
 
   /* Add microdesc to cache */
   crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256);
@@ -579,14 +564,6 @@ test_dir_handle_get_micro_d_server_busy(void *data)
 
   /* SETUP */
   init_mock_options();
-  const char *fn = get_fname("dir_handle_datadir_test2");
-  mock_options->DataDirectory = tor_strdup(fn);
-
-#ifdef _WIN32
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory));
-#else
-  tt_int_op(0, OP_EQ, mkdir(mock_options->DataDirectory, 0700));
-#endif
 
   /* Add microdesc to cache */
   crypto_digest256(digest, microdesc, strlen(microdesc), DIGEST_SHA256);
@@ -754,7 +731,7 @@ test_dir_handle_get_server_descriptors_not_found(void* data)
                       NULL, NULL, 1, 0);
 
   tt_str_op(NOT_FOUND, OP_EQ, header);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_SERVER_BY_FP);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
@@ -784,6 +761,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   tt_int_op(smartlist_len(our_routerlist->routers), OP_GE, 1);
   mock_routerinfo = smartlist_get(our_routerlist->routers, 0);
   set_server_identity_key(mock_routerinfo->identity_pkey);
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   /* Treat "all" requests as if they were unencrypted */
   mock_routerinfo->cache_info.send_unencrypted = 1;
@@ -798,7 +776,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   //which is smaller than that by annotation_len bytes
   fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
                       &body, &body_used,
-                      mock_routerinfo->cache_info.signed_descriptor_len+1, 0);
+                      1024*1024, 0);
 
   tt_assert(header);
   tt_assert(body);
@@ -814,7 +792,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
 
   tt_str_op(body, OP_EQ, mock_routerinfo->cache_info.signed_descriptor_body +
                          mock_routerinfo->cache_info.annotations_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
@@ -893,6 +871,7 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   mock_routerinfo->cache_info.signed_descriptor_len =
     strlen(TEST_DESCRIPTOR) - annotation_len;;
   mock_routerinfo->cache_info.annotations_len = annotation_len;
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   conn = new_dir_conn();
 
@@ -915,7 +894,7 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   tt_int_op(body_used, OP_EQ, strlen(body));
 
   tt_str_op(body, OP_EQ, TEST_DESCRIPTOR + annotation_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
@@ -957,6 +936,7 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   mock_routerinfo->cache_info.signed_descriptor_len =
     strlen(TEST_DESCRIPTOR) - annotation_len;
   mock_routerinfo->cache_info.annotations_len = annotation_len;
+  mock_routerinfo->cache_info.published_on = time(NULL);
 
   conn = new_dir_conn();
 
@@ -986,7 +966,7 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   tt_int_op(body_used, OP_EQ, strlen(body));
 
   tt_str_op(body, OP_EQ, TEST_DESCRIPTOR + annotation_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     NS_UNMOCK(router_get_my_routerinfo);
@@ -1052,7 +1032,7 @@ test_dir_handle_get_server_descriptors_d(void* data)
 
   tt_str_op(body, OP_EQ, router->cache_info.signed_descriptor_body +
                          router->cache_info.annotations_len);
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
@@ -1107,7 +1087,7 @@ test_dir_handle_get_server_descriptors_busy(void* data)
   tt_assert(header);
   tt_str_op(SERVER_BUSY, OP_EQ, header);
 
-  tt_int_op(conn->dir_spool_src, OP_EQ, DIR_SPOOL_NONE);
+  tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
     UNMOCK(get_options);
@@ -1629,7 +1609,13 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_enough_sigs(void* d)
   /* init mock */
   mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
   mock_ns_val->flavor = FLAV_NS;
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
   mock_ns_val->voters = smartlist_new();
+  mock_ns_val->valid_after = time(NULL) - 1800;
+  mock_ns_val->valid_until = time(NULL) - 60;
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
 
   /* init mock */
   init_mock_options();
@@ -1709,6 +1695,80 @@ test_dir_handle_get_status_vote_current_consensus_ns_not_found(void* data)
     or_options_free(mock_options); mock_options = NULL;
 }
 
+static void
+test_dir_handle_get_status_vote_current_consensus_too_old(void *data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void)data;
+
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
+  mock_ns_val->flavor = FLAV_MICRODESC;
+  mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800);
+  mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900);
+  mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20);
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
+
+  init_mock_options();
+
+  MOCK(get_options, mock_get_options);
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+  MOCK(networkstatus_get_latest_consensus_by_flavor, mock_ns_get_by_flavor);
+
+  conn = new_dir_conn();
+
+  setup_capture_of_logs(LOG_WARN);
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/consensus-microdesc"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(TOO_OLD, OP_EQ, header);
+
+  expect_log_msg_containing("too old");
+
+  tor_free(header);
+  teardown_capture_of_logs();
+  tor_free(mock_ns_val);
+
+  mock_ns_val = tor_malloc_zero(sizeof(networkstatus_t));
+  mock_ns_val->type = NS_TYPE_CONSENSUS;
+  mock_ns_val->flavor = FLAV_NS;
+  mock_ns_val->valid_after = time(NULL) - (24 * 60 * 60 + 1800);
+  mock_ns_val->fresh_until = time(NULL) - (24 * 60 * 60 + 900);
+  mock_ns_val->valid_until = time(NULL) - (24 * 60 * 60 + 20);
+
+  #define NETWORK_STATUS "some network status string"
+  consdiffmgr_add_consensus(NETWORK_STATUS, mock_ns_val);
+
+  setup_capture_of_logs(LOG_WARN);
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/consensus"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(TOO_OLD, OP_EQ, header);
+
+  expect_no_log_entry();
+
+  done:
+    teardown_capture_of_logs();
+    UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+    UNMOCK(connection_write_to_buf_impl_);
+    UNMOCK(get_options);
+    connection_free_(TO_CONN(conn));
+    tor_free(header);
+    tor_free(mock_ns_val);
+    or_options_free(mock_options); mock_options = NULL;
+}
+
 NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
 
 int
@@ -1723,12 +1783,26 @@ static void
 status_vote_current_consensus_ns_test(char **header, char **body,
                                       size_t *body_len)
 {
-  common_digests_t digests;
   dir_connection_t *conn = NULL;
 
   #define NETWORK_STATUS "some network status string"
+#if 0
+  common_digests_t digests;
+  uint8_t sha3[DIGEST256_LEN];
+  memset(&digests, 0x60, sizeof(digests));
+  memset(sha3, 0x06, sizeof(sha3));
   dirserv_set_cached_consensus_networkstatus(NETWORK_STATUS, "ns", &digests,
+                                             sha3,
                                              time(NULL));
+#endif
+  networkstatus_t *ns = tor_malloc_zero(sizeof(networkstatus_t));
+  ns->type = NS_TYPE_CONSENSUS;
+  ns->flavor = FLAV_NS;
+  ns->valid_after = time(NULL) - 1800;
+  ns->fresh_until = time(NULL) - 900;
+  ns->valid_until = time(NULL) - 60;
+  consdiffmgr_add_consensus(NETWORK_STATUS, ns);
+  networkstatus_vote_free(ns);
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -1741,7 +1815,6 @@ status_vote_current_consensus_ns_test(char **header, char **body,
   tt_str_op("ab", OP_EQ, geoip_get_country_name(1));
 
   conn = new_dir_conn();
-  TO_CONN(conn)->address = tor_strdup("127.0.0.1");
 
   tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
     GET("/tor/status-vote/current/consensus-ns"), NULL, 0));
@@ -1783,8 +1856,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
                                                             comp_body_used);
   tt_int_op(ZLIB_METHOD, OP_EQ, compression);
 
-  tor_gzip_uncompress(&body, &body_used, comp_body, comp_body_used,
-                      compression, 0, LOG_PROTOCOL_WARN);
+  tor_uncompress(&body, &body_used, comp_body, comp_body_used,
+                 compression, 0, LOG_PROTOCOL_WARN);
 
   tt_str_op(NETWORK_STATUS, OP_EQ, body);
   tt_int_op(strlen(NETWORK_STATUS), OP_EQ, body_used);
@@ -2448,6 +2521,53 @@ test_dir_handle_get_status_vote_current_authority(void* data)
     dirvote_free_all();
 }
 
+static void
+test_dir_handle_get_parse_accept_encoding(void *arg)
+{
+  (void)arg;
+  const unsigned B_NONE = 1u << NO_METHOD;
+  const unsigned B_ZLIB = 1u << ZLIB_METHOD;
+  const unsigned B_GZIP = 1u << GZIP_METHOD;
+  const unsigned B_LZMA = 1u << LZMA_METHOD;
+  const unsigned B_ZSTD = 1u << ZSTD_METHOD;
+
+  unsigned encodings;
+
+  encodings = parse_accept_encoding_header("");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("  ");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and howe ");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and gzip");
+  tt_uint_op(B_NONE, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("dewey, cheatham, and, gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header(" gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("gzip");
+  tt_uint_op(B_NONE|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("x-zstd, deflate, x-tor-lzma");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header(
+                                        "x-zstd, deflate, x-tor-lzma, gzip");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA|B_GZIP, OP_EQ, encodings);
+
+  encodings = parse_accept_encoding_header("x-zstd,deflate,x-tor-lzma,gzip");
+  tt_uint_op(B_NONE|B_ZLIB|B_ZSTD|B_LZMA|B_GZIP, OP_EQ, encodings);
+
+ done:
+  ;
+}
+
 #define DIR_HANDLE_CMD(name,flags) \
   { #name, test_dir_handle_get_##name, (flags), NULL, NULL }
 
@@ -2492,10 +2612,11 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, 0),
-  DIR_HANDLE_CMD(status_vote_current_consensus_ns, 0),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns_busy, TT_FORK),
+  DIR_HANDLE_CMD(status_vote_current_consensus_ns, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_d_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_d_not_found, 0),
   DIR_HANDLE_CMD(status_vote_d, 0),
@@ -2505,6 +2626,7 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures_busy, 0),
   DIR_HANDLE_CMD(status_vote_next_consensus_signatures, 0),
+  DIR_HANDLE_CMD(parse_accept_encoding, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index 9580a1fd3f..12a631630b 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -100,7 +100,7 @@ test_entryconn_rewrite_automap_ipv4(void *arg)
   ec3 = entry_connection_new(CONN_TYPE_AP, AF_INET);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes, tor_strdup("."));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes, ".");
   parse_virtual_addr_network("127.202.0.0/16", AF_INET, 0, &msg);
 
   /* Automap this on resolve. */
@@ -173,7 +173,7 @@ test_entryconn_rewrite_automap_ipv6(void *arg)
   ec3 = entry_connection_new(CONN_TYPE_AP, AF_INET6);
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes, tor_strdup("."));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes, ".");
   parse_virtual_addr_network("FE80::/32", AF_INET6, 0, &msg);
 
   /* Automap this on resolve. */
@@ -489,8 +489,8 @@ test_entryconn_rewrite_automap_exit(void *arg)
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
   get_options_mutable()->AllowDotExit = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".EXIT"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".EXIT");
   parse_virtual_addr_network("127.1.0.0/16", AF_INET, 0, &msg);
 
   /* Automap this on resolve. */
@@ -574,8 +574,8 @@ test_entryconn_rewrite_mapaddress_automap_onion(void *arg)
 
   get_options_mutable()->AutomapHostsOnResolve = 1;
   get_options_mutable()->AllowDotExit = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
   config_line_append(&get_options_mutable()->AddressMap,
                      "MapAddress", "foo.onion abcdefghijklmnop.onion");
@@ -709,8 +709,8 @@ test_entryconn_rewrite_mapaddress_automap_onion2(void *arg)
 {
   char *msg = NULL;
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
   config_line_append(&get_options_mutable()->AddressMap,
                      "MapAddress", "irc.example.com abcdefghijklmnop.onion");
@@ -736,8 +736,8 @@ test_entryconn_rewrite_mapaddress_automap_onion4(void *arg)
 {
   char *msg = NULL;
   get_options_mutable()->AutomapHostsOnResolve = 1;
-  smartlist_add(get_options_mutable()->AutomapHostsSuffixes,
-                tor_strdup(".onion"));
+  smartlist_add_strdup(get_options_mutable()->AutomapHostsSuffixes,
+                ".onion");
   parse_virtual_addr_network("192.168.0.0/16", AF_INET, 0, &msg);
 
   test_entryconn_rewrite_mapaddress_automap_onion_common(arg, 0, 1);
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index b1c3accfab..1f008d93b3 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -1,8 +1,9 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
+#define CIRCUITLIST_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
@@ -10,9 +11,13 @@
 #include "or.h"
 #include "test.h"
 
+#include "bridges.h"
+#include "circuitlist.h"
 #include "config.h"
+#include "confparse.h"
 #include "entrynodes.h"
 #include "nodelist.h"
+#include "networkstatus.h"
 #include "policies.h"
 #include "routerlist.h"
 #include "routerparse.h"
@@ -21,6 +26,7 @@
 #include "util.h"
 
 #include "test_helpers.h"
+#include "log_test_helpers.h"
 
 /* TODO:
  * choose_random_entry() test with state set.
@@ -39,37 +45,120 @@ get_or_state_replacement(void)
   return dummy_state;
 }
 
+static networkstatus_t *dummy_consensus = NULL;
+
+static smartlist_t *big_fake_net_nodes = NULL;
+
+static smartlist_t *
+bfn_mock_nodelist_get_list(void)
+{
+  return big_fake_net_nodes;
+}
+
+static networkstatus_t *
+bfn_mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void)now;
+  return dummy_consensus;
+}
+
+static const node_t *
+bfn_mock_node_get_by_id(const char *id)
+{
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
+                    if (fast_memeq(n->identity, id, 20))
+                      return n);
+
+  return NULL;
+}
+
 /* Unittest cleanup function: Cleanup the fake network. */
 static int
-fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
+big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
 {
   (void) testcase;
   (void) ptr;
 
-  routerlist_free_all();
-  nodelist_free_all();
-  entry_guards_free_all();
+  if (big_fake_net_nodes) {
+    SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+      tor_free(n->rs);
+      tor_free(n->md);
+      tor_free(n);
+    });
+    smartlist_free(big_fake_net_nodes);
+  }
+
+  UNMOCK(nodelist_get_list);
+  UNMOCK(node_get_by_id);
+  UNMOCK(get_or_state);
+  UNMOCK(networkstatus_get_live_consensus);
   or_state_free(dummy_state);
+  dummy_state = NULL;
+  tor_free(dummy_consensus);
 
   return 1; /* NOP */
 }
 
 /* Unittest setup function: Setup a fake network. */
 static void *
-fake_network_setup(const struct testcase_t *testcase)
+big_fake_network_setup(const struct testcase_t *testcase)
 {
-  (void) testcase;
+  int i;
+
+  /* These are minimal node_t objects that only contain the aspects of node_t
+   * that we need for entrynodes.c. */
+  const int N_NODES = 271;
+
+  big_fake_net_nodes = smartlist_new();
+  for (i = 0; i < N_NODES; ++i) {
+    node_t *n = tor_malloc_zero(sizeof(node_t));
+    n->md = tor_malloc_zero(sizeof(microdesc_t));
+
+    crypto_rand(n->identity, sizeof(n->identity));
+    n->rs = tor_malloc_zero(sizeof(routerstatus_t));
+
+    memcpy(n->rs->identity_digest, n->identity, DIGEST_LEN);
+
+    n->is_running = n->is_valid = n->is_fast = n->is_stable = 1;
+
+    /* Note: all these guards have the same address, so you'll need to
+     * disable EnforceDistinctSubnets when a restriction is applied. */
+    n->rs->addr = 0x04020202;
+    n->rs->or_port = 1234;
+    n->rs->is_v2_dir = 1;
+    n->rs->has_bandwidth = 1;
+    n->rs->bandwidth_kb = 30;
+
+    /* Call half of the nodes a possible guard. */
+    if (i % 2 == 0) {
+      n->is_possible_guard = 1;
+      n->rs->guardfraction_percentage = 100;
+      n->rs->has_guardfraction = 1;
+    }
+
+    smartlist_add(big_fake_net_nodes, n);
+  }
 
-  /* Setup fake state */
   dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  dummy_consensus->valid_after = approx_time() - 3600;
+  dummy_consensus->valid_until = approx_time() + 3600;
+
+  MOCK(nodelist_get_list, bfn_mock_nodelist_get_list);
+  MOCK(node_get_by_id, bfn_mock_node_get_by_id);
   MOCK(get_or_state,
        get_or_state_replacement);
-
-  /* Setup fake routerlist. */
-  helper_setup_fake_routerlist();
-
+  MOCK(networkstatus_get_live_consensus,
+       bfn_mock_networkstatus_get_live_consensus);
   /* Return anything but NULL (it's interpreted as test fail) */
-  return dummy_state;
+  return (void*)testcase;
+}
+
+static time_t
+mock_randomize_time_no_randomization(time_t a, time_t b)
+{
+  (void) b;
+  return a;
 }
 
 static or_options_t mocked_options;
@@ -80,796 +169,2563 @@ mock_get_options(void)
   return &mocked_options;
 }
 
-/** Test choose_random_entry() with none of our routers being guard nodes. */
+#define TEST_IPV4_ADDR "123.45.67.89"
+#define TEST_IPV6_ADDR "[1234:5678:90ab:cdef::]"
+
 static void
-test_choose_random_entry_no_guards(void *arg)
+test_node_preferred_orport(void *arg)
 {
-  const node_t *chosen_entry = NULL;
-
-  (void) arg;
-
-  MOCK(get_options, mock_get_options);
+  (void)arg;
+  tor_addr_t ipv4_addr;
+  const uint16_t ipv4_port = 4444;
+  tor_addr_t ipv6_addr;
+  const uint16_t ipv6_port = 6666;
+  routerinfo_t node_ri;
+  node_t node;
+  tor_addr_port_t ap;
 
-  /* Check that we get a guard if it passes preferred
-   * address settings */
+  /* Setup options */
   memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 0;
-
-  /* Try to pick an entry even though none of our routers are guards. */
-  chosen_entry = choose_random_entry(NULL);
-
-  /* Unintuitively, we actually pick a random node as our entry,
-     because router_choose_random_node() relaxes its constraints if it
-     can't find a proper entry guard. */
-  tt_assert(chosen_entry);
+  /* We don't test ClientPreferIPv6ORPort here, because it's used in
+   * nodelist_set_consensus to setup node.ipv6_preferred, which we set
+   * directly. */
+  MOCK(get_options, mock_get_options);
 
-  /* And with the other IP version active */
-  mocked_options.ClientUseIPv6 = 1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  /* Setup IP addresses */
+  tor_addr_parse(&ipv4_addr, TEST_IPV4_ADDR);
+  tor_addr_parse(&ipv6_addr, TEST_IPV6_ADDR);
 
-  /* And with the preference on auto */
-  mocked_options.ClientPreferIPv6ORPort = -1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  /* Setup node_ri */
+  memset(&node_ri, 0, sizeof(node_ri));
+  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
+  node_ri.or_port = ipv4_port;
+  tor_addr_copy(&node_ri.ipv6_addr, &ipv6_addr);
+  node_ri.ipv6_orport = ipv6_port;
 
-  /* Check that we don't get a guard if it doesn't pass mandatory address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+  /* Setup node */
+  memset(&node, 0, sizeof(node));
+  node.ri = &node_ri;
 
-  chosen_entry = choose_random_entry(NULL);
+  /* Check the preferred address is IPv4 if we're only using IPv4, regardless
+   * of whether we prefer it or not */
+  mocked_options.ClientUseIPv4 = 1;
+  mocked_options.ClientUseIPv6 = 0;
+  node.ipv6_preferred = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* If we don't allow IPv4 at all, we don't get a guard*/
-  tt_assert(!chosen_entry);
+  node.ipv6_preferred = 1;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* Check that we get a guard if it passes allowed but not preferred address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
+   * don't prefer the IPv6 address */
   mocked_options.ClientUseIPv4 = 1;
   mocked_options.ClientUseIPv6 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 1;
-
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  node.ipv6_preferred = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
+  tt_assert(ap.port == ipv4_port);
 
-  /* Check that we get a guard if it passes preferred address settings when
-   * they're auto */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  /* Check the preferred address is IPv6 if we prefer it and
+   * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
   mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = -1;
+  mocked_options.ClientUseIPv6 = 1;
+  node.ipv6_preferred = 1;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  mocked_options.ClientUseIPv4 = 0;
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
-  /* And with IPv6 active */
+  /* Check the preferred address is IPv6 if we don't prefer it, but
+   * ClientUseIPv4 is 0 */
+  mocked_options.ClientUseIPv4 = 0;
   mocked_options.ClientUseIPv6 = 1;
-
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  node_get_pref_orport(&node, &ap);
+  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
+  tt_assert(ap.port == ipv6_port);
 
  done:
-  memset(&mocked_options, 0, sizeof(mocked_options));
   UNMOCK(get_options);
 }
 
-/** Test choose_random_entry() with only one of our routers being a
-    guard node. */
 static void
-test_choose_random_entry_one_possible_guard(void *arg)
+test_entry_guard_describe(void *arg)
 {
-  const node_t *chosen_entry = NULL;
-  node_t *the_guard = NULL;
-  smartlist_t *our_nodelist = NULL;
+  (void)arg;
+  entry_guard_t g;
+  memset(&g, 0, sizeof(g));
+  strlcpy(g.nickname, "okefenokee", sizeof(g.nickname));
+  memcpy(g.identity, "theforestprimeval---", DIGEST_LEN);
 
-  (void) arg;
+  tt_str_op(entry_guard_describe(&g), OP_EQ,
+            "okefenokee ($746865666F726573747072696D6576616C2D2D2D)");
 
-  MOCK(get_options, mock_get_options);
-
-  /* Set one of the nodes to be a guard. */
-  our_nodelist = nodelist_get_list();
-  the_guard = smartlist_get(our_nodelist, 4); /* chosen by fair dice roll */
-  the_guard->is_possible_guard = 1;
+ done:
+  ;
+}
 
-  /* Check that we get the guard if it passes preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+static void
+test_entry_guard_randomize_time(void *arg)
+{
+  const time_t now = 1479153573;
+  const int delay = 86400;
+  const int N = 1000;
+  (void)arg;
 
-  /* Pick an entry. Make sure we pick the node we marked as guard. */
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+  time_t t;
+  int i;
+  for (i = 0; i < N; ++i) {
+    t = randomize_time(now, delay);
+    tt_int_op(t, OP_LE, now);
+    tt_int_op(t, OP_GE, now-delay);
+  }
 
-  /* And with the other IP version active */
-  mocked_options.ClientUseIPv6 = 1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+  /* now try the corner cases */
+  for (i = 0; i < N; ++i) {
+    t = randomize_time(100, delay);
+    tt_int_op(t, OP_GE, 1);
+    tt_int_op(t, OP_LE, 100);
 
-  /* And with the preference on auto */
-  mocked_options.ClientPreferIPv6ORPort = -1;
-  chosen_entry = choose_random_entry(NULL);
-  tt_ptr_op(chosen_entry, OP_EQ, the_guard);
+    t = randomize_time(0, delay);
+    tt_int_op(t, OP_EQ, 1);
+  }
 
-  /* Check that we don't get a guard if it doesn't pass mandatory address
-   * settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientPreferIPv6ORPort = 0;
+ done:
+  ;
+}
 
-  chosen_entry = choose_random_entry(NULL);
+static void
+test_entry_guard_encode_for_state_minimal(void *arg)
+{
+  (void) arg;
+  entry_guard_t *eg = tor_malloc_zero(sizeof(entry_guard_t));
 
-  /* If we don't allow IPv4 at all, we don't get a guard*/
-  tt_assert(!chosen_entry);
+  eg->selection_name = tor_strdup("wubwub");
+  memcpy(eg->identity, "plurpyflurpyslurpydo", DIGEST_LEN);
+  eg->sampled_on_date = 1479081600;
+  eg->confirmed_idx = -1;
 
-  /* Check that we get a node if it passes allowed but not preferred
-   * address settings */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  mocked_options.ClientPreferIPv6ORPort = 1;
+  char *s = NULL;
+  s = entry_guard_encode_for_state(eg);
 
-  chosen_entry = choose_random_entry(NULL);
+  tt_str_op(s, OP_EQ,
+            "in=wubwub "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "sampled_on=2016-11-14T00:00:00 "
+            "listed=0");
 
-  /* We disable the guard check and the preferred address check at the same
-   * time, so we can't be sure we get the guard */
-  tt_assert(chosen_entry);
+ done:
+  entry_guard_free(eg);
+  tor_free(s);
+}
 
-  /* Check that we get a node if it is allowed but not preferred when settings
-   * are auto */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientPreferIPv6ORPort = -1;
+static void
+test_entry_guard_encode_for_state_maximal(void *arg)
+{
+  (void) arg;
+  entry_guard_t *eg = tor_malloc_zero(sizeof(entry_guard_t));
+
+  strlcpy(eg->nickname, "Fred", sizeof(eg->nickname));
+  eg->selection_name = tor_strdup("default");
+  memcpy(eg->identity, "plurpyflurpyslurpydo", DIGEST_LEN);
+  eg->bridge_addr = tor_malloc_zero(sizeof(tor_addr_port_t));
+  tor_addr_from_ipv4h(&eg->bridge_addr->addr, 0x08080404);
+  eg->bridge_addr->port = 9999;
+  eg->sampled_on_date = 1479081600;
+  eg->sampled_by_version = tor_strdup("1.2.3");
+  eg->unlisted_since_date = 1479081645;
+  eg->currently_listed = 1;
+  eg->confirmed_on_date = 1479081690;
+  eg->confirmed_idx = 333;
+  eg->extra_state_fields = tor_strdup("and the green grass grew all around");
+
+  char *s = NULL;
+  s = entry_guard_encode_for_state(eg);
+
+  tt_str_op(s, OP_EQ,
+            "in=default "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=8.8.4.4:9999 "
+            "nickname=Fred "
+            "sampled_on=2016-11-14T00:00:00 "
+            "sampled_by=1.2.3 "
+            "unlisted_since=2016-11-14T00:00:45 "
+            "listed=1 "
+            "confirmed_on=2016-11-14T00:01:30 "
+            "confirmed_idx=333 "
+            "and the green grass grew all around");
 
-  chosen_entry = choose_random_entry(NULL);
+ done:
+  entry_guard_free(eg);
+  tor_free(s);
+}
 
-  /* We disable the guard check and the preferred address check at the same
-   * time, so we can't be sure we get the guard */
-  tt_assert(chosen_entry);
+static void
+test_entry_guard_parse_from_state_minimal(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+  time_t t = approx_time();
+
+  eg = entry_guard_parse_from_state(
+                 "in=default_plus "
+                 "rsa_id=596f75206d6179206e656564206120686f626279");
+  tt_assert(eg);
+
+  tt_str_op(eg->selection_name, OP_EQ, "default_plus");
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "596f75206d6179206e656564206120686f626279");
+  tt_str_op(eg->nickname, OP_EQ, "$596F75206D6179206E656564206120686F626279");
+  tt_ptr_op(eg->bridge_addr, OP_EQ, NULL);
+  tt_i64_op(eg->sampled_on_date, OP_GE, t);
+  tt_i64_op(eg->sampled_on_date, OP_LE, t+86400);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 0);
+  tt_ptr_op(eg->sampled_by_version, OP_EQ, NULL);
+  tt_int_op(eg->currently_listed, OP_EQ, 0);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(eg->confirmed_idx, OP_EQ, -1);
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
-  /* and with IPv6 active */
-  mocked_options.ClientUseIPv6 = 1;
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
 
-  chosen_entry = choose_random_entry(NULL);
-  tt_assert(chosen_entry);
+static void
+test_entry_guard_parse_from_state_maximal(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+
+  eg = entry_guard_parse_from_state(
+            "in=fred "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=[1::3]:9999 "
+            "nickname=Fred "
+            "sampled_on=2016-11-14T00:00:00 "
+            "sampled_by=1.2.3 "
+            "unlisted_since=2016-11-14T00:00:45 "
+            "listed=1 "
+            "confirmed_on=2016-11-14T00:01:30 "
+            "confirmed_idx=333 "
+            "and the green grass grew all around "
+            "rsa_id=all,around");
+  tt_assert(eg);
+
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "706C75727079666C75727079736C75727079646F");
+  tt_str_op(fmt_addr(&eg->bridge_addr->addr), OP_EQ, "1::3");
+  tt_int_op(eg->bridge_addr->port, OP_EQ, 9999);
+  tt_str_op(eg->nickname, OP_EQ, "Fred");
+  tt_i64_op(eg->sampled_on_date, OP_EQ, 1479081600);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 1479081645);
+  tt_str_op(eg->sampled_by_version, OP_EQ, "1.2.3");
+  tt_int_op(eg->currently_listed, OP_EQ, 1);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 1479081690);
+  tt_int_op(eg->confirmed_idx, OP_EQ, 333);
+  tt_str_op(eg->extra_state_fields, OP_EQ,
+            "and the green grass grew all around rsa_id=all,around");
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
 
  done:
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  UNMOCK(get_options);
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
 }
 
-/** Helper to conduct tests for populate_live_entry_guards().
+static void
+test_entry_guard_parse_from_state_failure(void *arg)
+{
+  (void)arg;
+  entry_guard_t *eg = NULL;
+
+  /* no selection */
+  eg = entry_guard_parse_from_state(
+                 "rsa_id=596f75206d6179206e656564206120686f626270");
+  tt_assert(! eg);
+
+  /* no RSA ID. */
+  eg = entry_guard_parse_from_state("in=default nickname=Fred");
+  tt_assert(! eg);
+
+  /* Bad RSA ID: bad character. */
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e656564206120686f62627q");
+  tt_assert(! eg);
+
+  /* Bad RSA ID: too long.*/
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e656564206120686f6262703");
+  tt_assert(! eg);
+
+  /* Bad RSA ID: too short.*/
+  eg = entry_guard_parse_from_state(
+                 "in=default "
+                 "rsa_id=596f75206d6179206e65656420612");
+  tt_assert(! eg);
 
-   This test adds some entry guards to our list, and then tests
-   populate_live_entry_guards() to mke sure it filters them correctly.
+ done:
+  entry_guard_free(eg);
+}
 
-   <b>num_needed</b> is the number of guard nodes we support. It's
-   configurable to make sure we function properly with 1 or 3 guard
-   nodes configured.
-*/
 static void
-populate_live_entry_guards_test_helper(int num_needed)
+test_entry_guard_parse_from_state_partial_failure(void *arg)
+{
+  (void)arg;
+  char *mem_op_hex_tmp = NULL;
+  entry_guard_t *eg = NULL;
+  time_t t = approx_time();
+
+  eg = entry_guard_parse_from_state(
+            "in=default "
+            "rsa_id=706C75727079666C75727079736C75727079646F "
+            "bridge_addr=1.2.3.3.4:5 "
+            "nickname=FredIsANodeWithAStrangeNicknameThatIsTooLong "
+            "sampled_on=2016-11-14T00:00:99 "
+            "sampled_by=1.2.3 stuff in the middle "
+            "unlisted_since=2016-xx-14T00:00:45 "
+            "listed=0 "
+            "confirmed_on=2016-11-14T00:01:30zz "
+            "confirmed_idx=idx "
+            "and the green grass grew all around "
+            "rsa_id=all,around");
+  tt_assert(eg);
+
+  test_mem_op_hex(eg->identity, OP_EQ,
+                  "706C75727079666C75727079736C75727079646F");
+  tt_str_op(eg->nickname, OP_EQ, "FredIsANodeWithAStrangeNicknameThatIsTooL");
+  tt_ptr_op(eg->bridge_addr, OP_EQ, NULL);
+  tt_i64_op(eg->sampled_on_date, OP_EQ, t);
+  tt_i64_op(eg->unlisted_since_date, OP_EQ, 0);
+  tt_str_op(eg->sampled_by_version, OP_EQ, "1.2.3");
+  tt_int_op(eg->currently_listed, OP_EQ, 0);
+  tt_i64_op(eg->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(eg->confirmed_idx, OP_EQ, -1);
+  tt_str_op(eg->extra_state_fields, OP_EQ,
+            "stuff in the middle and the green grass grew all around "
+            "rsa_id=all,around");
+
+  tt_int_op(eg->last_tried_to_connect, OP_EQ, 0);
+  tt_int_op(eg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+ done:
+  entry_guard_free(eg);
+  tor_free(mem_op_hex_tmp);
+}
+
+static int
+mock_entry_guard_is_listed(guard_selection_t *gs, const entry_guard_t *guard)
 {
-  smartlist_t *our_nodelist = NULL;
-  smartlist_t *live_entry_guards = smartlist_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  or_options_t *options = get_options_mutable();
-  int retval;
+  (void)gs;
+  (void)guard;
+  return 1;
+}
 
-  /* Set NumEntryGuards to the provided number. */
-  options->NumEntryGuards = num_needed;
-  tt_int_op(num_needed, OP_EQ, decide_num_guards(options, 0));
+static void
+test_entry_guard_parse_from_state_full(void *arg)
+{
+  (void)arg;
+  /* Here's a state I made while testing.  The identities and locations for
+   * the bridges are redacted. */
+  const char STATE[] =
+  "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
+    "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n"
+  "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
+    "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
+    "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
+    "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1 "
+    "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
+    "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
+    "pb_successful_circuits_closed=13.000000\n"
+  "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
+    "bridge_addr=37.218.246.143:28366 "
+    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n";
+
+  config_line_t *lines = NULL;
+  or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
+  int r = config_get_lines(STATE, &lines, 0);
+  char *msg = NULL;
+  smartlist_t *text = smartlist_new();
+  char *joined = NULL;
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  // So nodes aren't expired. This is Tue, 13 Dec 2016 09:37:14 GMT
+  update_approx_time(1481621834);
 
-  /* Walk the nodelist and add all nodes as entry guards. */
-  our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
+  MOCK(entry_guard_is_listed, mock_entry_guard_is_listed);
 
-  SMARTLIST_FOREACH_BEGIN(our_nodelist, const node_t *, node) {
-    const node_t *node_tmp;
-    node_tmp = add_an_entry_guard(node, 0, 1, 0, 0);
-    tt_assert(node_tmp);
-  } SMARTLIST_FOREACH_END(node);
+  dummy_state = state;
+  MOCK(get_or_state,
+       get_or_state_replacement);
 
-  /* Make sure the nodes were added as entry guards. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ,
-            HELPER_NUMBER_OF_DESCRIPTORS);
+  tt_assert(r == 0);
+  tt_assert(lines);
+
+  state->Guard = lines;
+
+  /* Try it first without setting the result. */
+  r = entry_guards_parse_state(state, 0, &msg);
+  tt_assert(r == 0);
+  guard_selection_t *gs_br =
+    get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  tt_assert(!gs_br);
+
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_assert(r == 0);
+  gs_br = get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  guard_selection_t *gs_df =
+    get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  guard_selection_t *gs_wb =
+    get_guard_selection_by_name("wobblesome", GS_TYPE_NORMAL, 0);
+
+  tt_assert(gs_br);
+  tt_assert(gs_df);
+  tt_assert(gs_wb);
+
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 5);
+  tt_int_op(smartlist_len(gs_br->sampled_entry_guards), OP_EQ, 2);
+  tt_int_op(smartlist_len(gs_wb->sampled_entry_guards), OP_EQ, 1);
+
+  /* Try again; make sure it doesn't double-add the guards. */
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_assert(r == 0);
+  gs_br = get_guard_selection_by_name("bridges", GS_TYPE_BRIDGE, 0);
+  gs_df = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_assert(gs_br);
+  tt_assert(gs_df);
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 5);
+  tt_int_op(smartlist_len(gs_br->sampled_entry_guards), OP_EQ, 2);
+
+  /* Re-encode; it should be the same... almost. */
+  {
+    /* (Make a guard nonpersistent first) */
+    entry_guard_t *g = smartlist_get(gs_df->sampled_entry_guards, 0);
+    g->is_persistent = 0;
+  }
+  config_free_lines(lines);
+  lines = state->Guard = NULL; // to prevent double-free.
+  entry_guards_update_state(state);
+  tt_assert(state->Guard);
+  lines = state->Guard;
+
+  config_line_t *ln;
+  for (ln = lines; ln; ln = ln->next) {
+    smartlist_add_asprintf(text, "%s %s\n",ln->key, ln->value);
+  }
+  joined = smartlist_join_strings(text, "", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+  "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=1 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n"
+  "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
+    "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
+    "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n"
+  "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
+    "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1 "
+    "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
+    "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
+    "pb_successful_circuits_closed=13.000000\n"
+  "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
+    "bridge_addr=37.218.246.143:28366 "
+    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n");
 
-  /* Ensure that all the possible entry guards are enough to satisfy us. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_GE, num_needed);
+ done:
+  config_free_lines(lines);
+  tor_free(state);
+  tor_free(msg);
+  UNMOCK(get_or_state);
+  UNMOCK(entry_guard_is_listed);
+  SMARTLIST_FOREACH(text, char *, cp, tor_free(cp));
+  smartlist_free(text);
+  tor_free(joined);
+}
 
-  /* Walk the entry guard list for some sanity checking */
-  SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) {
-    /* Since we called add_an_entry_guard() with 'for_discovery' being
-       False, all guards should have made_contact enabled. */
-    tt_int_op(entry->made_contact, OP_EQ, 1);
+static void
+test_entry_guard_parse_from_state_broken(void *arg)
+{
+  (void)arg;
+  /* Here's a variation on the previous state. Every line but the first is
+   * busted somehow. */
+  const char STATE[] =
+  /* Okay. */
+  "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
+    "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1\n"
+  /* No selection listed. */
+  "Guard rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
+    "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
+    "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
+    "pb_successful_circuits_closed=2.000000\n"
+  /* Selection is "legacy"!! */
+  "Guard in=legacy rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
+    "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_by=0.3.0.0-alpha-dev "
+    "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
+    "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
+    "pb_successful_circuits_closed=5.000000\n";
+
+  config_line_t *lines = NULL;
+  or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
+  int r = config_get_lines(STATE, &lines, 0);
+  char *msg = NULL;
 
-  } SMARTLIST_FOREACH_END(entry);
+  dummy_state = state;
+  MOCK(get_or_state,
+       get_or_state_replacement);
 
-  /* First, try to get some fast guards. This should fail. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO, /* Don't care about DIRINFO*/
-                                      0, 0,
-                                      1); /* We want fast guard! */
-  tt_int_op(retval, OP_EQ, 0);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, 0);
+  tt_assert(r == 0);
+  tt_assert(lines);
 
-  /* Now try to get some stable guards. This should fail too. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO,
-                                      0,
-                                      1, /* We want stable guard! */
-                                      0);
-  tt_int_op(retval, OP_EQ, 0);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, 0);
+  state->Guard = lines;
 
-  /* Now try to get any guard we can find. This should succeed. */
-  retval = populate_live_entry_guards(live_entry_guards,
-                                      all_entry_guards,
-                                      NULL,
-                                      NO_DIRINFO,
-                                      0, 0, 0); /* No restrictions! */
+  /* First, no-set case. we should get an error. */
+  r = entry_guards_parse_state(state, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  /* And we shouldn't have made anything. */
+  guard_selection_t *gs_df =
+    get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_assert(gs_df == NULL);
+  tor_free(msg);
 
-  /* Since we had more than enough guards in 'all_entry_guards', we
-     should have added 'num_needed' of them to live_entry_guards.
-     'retval' should be 1 since we now have enough live entry guards
-     to pick one.  */
-  tt_int_op(retval, OP_EQ, 1);
-  tt_int_op(smartlist_len(live_entry_guards), OP_EQ, num_needed);
+  /* Now see about the set case (which shouldn't happen IRL) */
+  r = entry_guards_parse_state(state, 1, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  gs_df = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_assert(gs_df != NULL);
+  tt_int_op(smartlist_len(gs_df->sampled_entry_guards), OP_EQ, 1);
 
  done:
-  smartlist_free(live_entry_guards);
+  config_free_lines(lines);
+  tor_free(state);
+  tor_free(msg);
+  UNMOCK(get_or_state);
 }
 
-/* Test populate_live_entry_guards() for 1 guard node. */
 static void
-test_populate_live_entry_guards_1guard(void *arg)
+test_entry_guard_get_guard_selection_by_name(void *arg)
 {
-  (void) arg;
+  (void)arg;
+  guard_selection_t *gs1, *gs2, *gs3;
+
+  gs1 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 0);
+  tt_assert(gs1 == NULL);
+  gs1 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 1);
+  tt_assert(gs1 != NULL);
+  gs2 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 1);
+  tt_assert(gs2 == gs1);
+  gs2 = get_guard_selection_by_name("unlikely", GS_TYPE_NORMAL, 0);
+  tt_assert(gs2 == gs1);
+
+  gs2 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 0);
+  tt_assert(gs2 == NULL);
+  gs2 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 1);
+  tt_assert(gs2 != NULL);
+  tt_assert(gs2 != gs1);
+  gs3 = get_guard_selection_by_name("implausible", GS_TYPE_NORMAL, 0);
+  tt_assert(gs3 == gs2);
+
+  gs3 = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 0);
+  tt_assert(gs3 == NULL);
+  gs3 = get_guard_selection_by_name("default", GS_TYPE_NORMAL, 1);
+  tt_assert(gs3 != NULL);
+  tt_assert(gs3 != gs2);
+  tt_assert(gs3 != gs1);
+  tt_assert(gs3 == get_guard_selection_info());
 
-  populate_live_entry_guards_test_helper(1);
+ done:
+  entry_guards_free_all();
 }
 
-/* Test populate_live_entry_guards() for 3 guard nodes. */
 static void
-test_populate_live_entry_guards_3guards(void *arg)
+test_entry_guard_choose_selection_initial(void *arg)
 {
-  (void) arg;
+  /* Tests for picking our initial guard selection (based on having had
+   * no previous selection */
+  (void)arg;
+  guard_selection_type_t type = GS_TYPE_INFER;
+  const char *name = choose_guard_selection(get_options(),
+                                            dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "default");
+  tt_int_op(type, OP_EQ, GS_TYPE_NORMAL);
+
+  /* If we're using bridges, we get the bridge selection. */
+  get_options_mutable()->UseBridges = 1;
+  name = choose_guard_selection(get_options(),
+                                dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "bridges");
+  tt_int_op(type, OP_EQ, GS_TYPE_BRIDGE);
+  get_options_mutable()->UseBridges = 0;
+
+  /* If we discard >99% of our guards, though, we should be in the restricted
+   * set. */
+  tt_assert(get_options_mutable()->EntryNodes == NULL);
+  get_options_mutable()->EntryNodes = routerset_new();
+  routerset_parse(get_options_mutable()->EntryNodes, "1.0.0.0/8", "foo");
+  name = choose_guard_selection(get_options(),
+                                dummy_consensus, NULL, &type);
+  tt_str_op(name, OP_EQ, "restricted");
+  tt_int_op(type, OP_EQ, GS_TYPE_RESTRICTED);
 
-  populate_live_entry_guards_test_helper(3);
+ done:
+  ;
 }
 
-/** Append some EntryGuard lines to the Tor state at <b>state</b>.
-
-   <b>entry_guard_lines</b> is a smartlist containing 2-tuple
-   smartlists that carry the key and values of the statefile.
-   As an example:
-   entry_guard_lines =
-     (("EntryGuard", "name 67E72FF33D7D41BF11C569646A0A7B4B188340DF DirCache"),
-      ("EntryGuardDownSince", "2014-06-07 16:02:46 2014-06-07 16:02:46"))
-*/
 static void
-state_insert_entry_guard_helper(or_state_t *state,
-                                smartlist_t *entry_guard_lines)
+test_entry_guard_add_single_guard(void *arg)
 {
-  config_line_t **next, *line;
-
-  next = &state->EntryGuards;
-  *next = NULL;
-
-  /* Loop over all the state lines in the smartlist */
-  SMARTLIST_FOREACH_BEGIN(entry_guard_lines, const smartlist_t *,state_lines) {
-    /* Get key and value for each line */
-    const char *state_key = smartlist_get(state_lines, 0);
-    const char *state_value = smartlist_get(state_lines, 1);
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* 1: Add a single guard to the sample. */
+  node_t *n1 = smartlist_get(big_fake_net_nodes, 0);
+  time_t now = approx_time();
+  tt_assert(n1->is_possible_guard == 1);
+  entry_guard_t *g1 = entry_guard_add_to_sample(gs, n1);
+  tt_assert(g1);
+
+  /* Make sure its fields look right. */
+  tt_mem_op(n1->identity, OP_EQ, g1->identity, DIGEST_LEN);
+  tt_i64_op(g1->sampled_on_date, OP_GE, now - 12*86400);
+  tt_i64_op(g1->sampled_on_date, OP_LE, now);
+  tt_str_op(g1->sampled_by_version, OP_EQ, VERSION);
+  tt_assert(g1->currently_listed == 1);
+  tt_i64_op(g1->confirmed_on_date, OP_EQ, 0);
+  tt_int_op(g1->confirmed_idx, OP_EQ, -1);
+  tt_int_op(g1->last_tried_to_connect, OP_EQ, 0);
+  tt_uint_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_i64_op(g1->failing_since, OP_EQ, 0);
+  tt_assert(g1->is_filtered_guard == 1);
+  tt_assert(g1->is_usable_filtered_guard == 1);
+  tt_assert(g1->is_primary == 0);
+  tt_assert(g1->extra_state_fields == NULL);
+
+  /* Make sure it got added. */
+  tt_int_op(1, OP_EQ, smartlist_len(gs->sampled_entry_guards));
+  tt_ptr_op(g1, OP_EQ, smartlist_get(gs->sampled_entry_guards, 0));
+  tt_ptr_op(g1, OP_EQ, get_sampled_guard_with_id(gs, (uint8_t*)n1->identity));
+  const uint8_t bad_id[20] = {0};
+  tt_ptr_op(NULL, OP_EQ, get_sampled_guard_with_id(gs, bad_id));
 
-    *next = line = tor_malloc_zero(sizeof(config_line_t));
-    line->key = tor_strdup(state_key);
-    tor_asprintf(&line->value, "%s", state_value);
-    next = &(line->next);
-  } SMARTLIST_FOREACH_END(state_lines);
+ done:
+  guard_selection_free(gs);
 }
 
-/** Free memory occupied by <b>entry_guard_lines</b>. */
 static void
-state_lines_free(smartlist_t *entry_guard_lines)
+test_entry_guard_node_filter(void *arg)
 {
-  SMARTLIST_FOREACH_BEGIN(entry_guard_lines, smartlist_t *, state_lines) {
-    char *state_key = smartlist_get(state_lines, 0);
-    char *state_value = smartlist_get(state_lines, 1);
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  bridge_line_t *bl = NULL;
+
+  /* Initialize a bunch of node objects that are all guards. */
+#define NUM 7
+  node_t *n[NUM];
+  entry_guard_t *g[NUM];
+  int i;
+  for (i=0; i < NUM; ++i) {
+    n[i] = smartlist_get(big_fake_net_nodes, i*2); // even ones are guards.
+    g[i] = entry_guard_add_to_sample(gs, n[i]);
+
+    // everything starts out filtered-in
+    tt_assert(g[i]->is_filtered_guard == 1);
+    tt_assert(g[i]->is_usable_filtered_guard == 1);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, NUM);
 
-    tor_free(state_key);
-    tor_free(state_value);
-    smartlist_free(state_lines);
-  } SMARTLIST_FOREACH_END(state_lines);
+  /* Make sure refiltering doesn't hurt */
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_assert(g[i]->is_filtered_guard == 1);
+    tt_assert(g[i]->is_usable_filtered_guard == 1);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, NUM);
 
-  smartlist_free(entry_guard_lines);
-}
+  /* Now start doing things to make the guards get filtered out, 1 by 1. */
 
-/* Tests entry_guards_parse_state(). It creates a fake Tor state with
-   a saved entry guard and makes sure that Tor can parse it and
-   creates the right entry node out of it.
-*/
-static void
-test_entry_guards_parse_state_simple(void *arg)
-{
-  or_state_t *state = or_state_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  smartlist_t *entry_state_lines = smartlist_new();
-  char *msg = NULL;
-  int retval;
+  /* 0: Not listed. */
+  g[0]->currently_listed = 0;
 
-  /* Details of our fake guard node */
-  const char *nickname = "hagbard";
-  const char *fpr = "B29D536DD1752D542E1FBB3C9CE4449D51298212";
-  const char *tor_version = "0.2.5.3-alpha-dev";
-  const char *added_at = get_yesterday_date_str();
-  const char *unlisted_since = "2014-06-08 16:16:50";
+  /* 1: path bias says this guard is maybe eeeevil. */
+  g[1]->pb.path_bias_disabled = 1;
 
-  (void) arg;
+  /* 2: Unreachable address. */
+  n[2]->rs->addr = 0;
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  /* 3: ExcludeNodes */
+  n[3]->rs->addr = 0x90902020;
+  routerset_free(get_options_mutable()->ExcludeNodes);
+  get_options_mutable()->ExcludeNodes = routerset_new();
+  routerset_parse(get_options_mutable()->ExcludeNodes, "144.144.0.0/16", "");
 
-  { /* Prepare the state entry */
+  /* 4: Bridge. */
+  sweep_bridge_list();
+  bl = tor_malloc_zero(sizeof(bridge_line_t));
+  tor_addr_from_ipv4h(&bl->addr, n[4]->rs->addr);
+  bl->port = n[4]->rs->or_port;
+  memcpy(bl->digest, n[4]->identity, 20);
+  bridge_add_from_config(bl);
+  bl = NULL; // prevent free.
 
-    /* Prepare the smartlist to hold the key/value of each line */
-    smartlist_t *state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuard");
-    smartlist_add_asprintf(state_line, "%s %s %s", nickname, fpr, "DirCache");
-    smartlist_add(entry_state_lines, state_line);
+  /* 5: Unreachable. This stays in the filter, but isn't in usable-filtered */
+  g[5]->last_tried_to_connect = approx_time(); // prevent retry.
+  g[5]->is_reachable = GUARD_REACHABLE_NO;
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardAddedBy");
-    smartlist_add_asprintf(state_line, "%s %s %s", fpr, tor_version, added_at);
-    smartlist_add(entry_state_lines, state_line);
+  /* 6: no change. */
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardUnlistedSince");
-    smartlist_add_asprintf(state_line, "%s", unlisted_since);
-    smartlist_add(entry_state_lines, state_line);
+  /* Now refilter and inspect. */
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_assert(g[i]->is_filtered_guard == (i == 5 || i == 6));
+    tt_assert(g[i]->is_usable_filtered_guard == (i == 6));
   }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 1);
+
+  /* Now make sure we have no live consensus, and no nodes.  Nothing should
+   * pass the filter any more. */
+  tor_free(dummy_consensus);
+  dummy_consensus = NULL;
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, node, {
+    memset(node->identity, 0xff, 20);
+  });
+  entry_guards_update_filtered_sets(gs);
+  for (i = 0; i < NUM; ++i) {
+    tt_assert(g[i]->is_filtered_guard == 0);
+    tt_assert(g[i]->is_usable_filtered_guard == 0);
+  }
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 0);
 
-  /* Inject our lines in the state */
-  state_insert_entry_guard_helper(state, entry_state_lines);
+ done:
+  guard_selection_free(gs);
+  tor_free(bl);
+#undef NUM
+}
 
-  /* Parse state */
-  retval = entry_guards_parse_state(state, 1, &msg);
-  tt_int_op(retval, OP_GE, 0);
+static void
+test_entry_guard_expand_sample(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  digestmap_t *node_by_id = digestmap_new();
+
+  entry_guard_t *guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // the last guard returned.
+
+  // Every sampled guard here should be filtered and reachable for now.
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  /* Make sure we got the right number. */
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Make sure everything we got was from our fake node list, and everything
+  // was unique.
+  SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, g) {
+    const node_t *n = bfn_mock_node_get_by_id(g->identity);
+    tt_assert(n);
+    tt_ptr_op(NULL, OP_EQ, digestmap_get(node_by_id, g->identity));
+    digestmap_set(node_by_id, g->identity, (void*) n);
+    int idx = smartlist_pos(big_fake_net_nodes, n);
+    // The even ones are the guards; make sure we got guards.
+    tt_int_op(idx & 1, OP_EQ, 0);
+  } SMARTLIST_FOREACH_END(g);
+
+  // Nothing became unusable/unfiltered, so a subsequent expand should
+  // make no changes.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(! guard); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Make a few guards unreachable.
+  guard = smartlist_get(gs->sampled_entry_guards, 0);
+  guard->is_usable_filtered_guard = 0;
+  guard = smartlist_get(gs->sampled_entry_guards, 1);
+  guard->is_usable_filtered_guard = 0;
+  guard = smartlist_get(gs->sampled_entry_guards, 2);
+  guard->is_usable_filtered_guard = 0;
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE - 3, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // This time, expanding the sample will add some more guards.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ,
+            num_reachable_filtered_guards(gs, NULL)+3);
+
+  // Still idempotent.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(! guard); // no guard was added.
+  tt_int_op(DFLT_MIN_FILTERED_SAMPLE_SIZE, OP_EQ,
+            num_reachable_filtered_guards(gs, NULL));
+
+  // Now, do a nasty trick: tell the filter to exclude 31/32 of the guards.
+  // This will cause the sample size to get reeeeally huge, while the
+  // filtered sample size grows only slowly.
+  routerset_free(get_options_mutable()->ExcludeNodes);
+  get_options_mutable()->ExcludeNodes = routerset_new();
+  routerset_parse(get_options_mutable()->ExcludeNodes, "144.144.0.0/16", "");
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+    if (n_sl_idx % 64 != 0) {
+      n->rs->addr = 0x90903030;
+    }
+  });
+  entry_guards_update_filtered_sets(gs);
+
+  // Surely (p ~ 1-2**-60), one of our guards has been excluded.
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_LT,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+
+  // Try to regenerate the guards.
+  guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // no guard was added.
+
+  /* this time, it's possible that we didn't add enough sampled guards. */
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_LE,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+  /* but we definitely didn't exceed the sample maximum. */
+  const int n_guards = 271 / 2;
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_LE,
+            (int)(n_guards * .3));
 
-  /* Test that the guard was registered.
-     We need to re-get the entry guard list since its pointer was
-     overwritten in entry_guards_parse_state(). */
-  all_entry_guards = get_entry_guards();
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+ done:
+  guard_selection_free(gs);
+  digestmap_free(node_by_id, NULL);
+}
+
+static void
+test_entry_guard_expand_sample_small_net(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* Fun corner case: not enough guards to make up our whole sample size. */
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, {
+    if (n_sl_idx >= 15) {
+      tor_free(n->rs);
+      tor_free(n->md);
+      tor_free(n);
+      SMARTLIST_DEL_CURRENT(big_fake_net_nodes, n);
+    } else {
+      n->rs->addr = 0; // make the filter reject this.
+    }
+  });
+
+  entry_guard_t *guard = entry_guards_expand_sample(gs);
+  tt_assert(guard); // the last guard returned -- some guard was added.
+  // half the nodes are guards, so we have 8 guards left.  The set
+  // is small, so we sampled everything.
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, 8);
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, 0);
+ done:
+  guard_selection_free(gs);
+}
 
-  { /* Test the entry guard structure */
-    char hex_digest[1024];
-    char str_time[1024];
+static void
+test_entry_guard_update_from_consensus_status(void *arg)
+{
+  /* Here we're going to have some nodes become un-guardy, and say we got a
+   * new consensus. This should cause those nodes to get detected as
+   * unreachable. */
 
-    const entry_guard_t *e = smartlist_get(all_entry_guards, 0);
-    tt_str_op(e->nickname, OP_EQ, nickname); /* Verify nickname */
+  (void)arg;
+  int i;
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  networkstatus_t *ns_tmp = NULL;
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  /* At this point, it should be a no-op to do this: */
+  sampled_guards_update_from_consensus(gs);
+
+  /* Now let's make some of our guards become unlisted.  The easiest way to
+   * do that would be to take away their guard flag. */
+  for (i = 0; i < 5; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    n->is_possible_guard = 0;
+  }
 
-    base16_encode(hex_digest, sizeof(hex_digest),
-                  e->identity, DIGEST_LEN);
-    tt_str_op(hex_digest, OP_EQ, fpr); /* Verify fingerprint */
+  update_approx_time(start + 30);
+  {
+    /* try this with no live networkstatus. Nothing should happen! */
+    ns_tmp = dummy_consensus;
+    dummy_consensus = NULL;
+    sampled_guards_update_from_consensus(gs);
+    tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+    tt_i64_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre);
+    /* put the networkstatus back. */
+    dummy_consensus = ns_tmp;
+    ns_tmp = NULL;
+  }
 
-    tt_assert(e->is_dir_cache); /* Verify dirness */
+  /* Now those guards should become unlisted, and drop off the filter, but
+   * stay in the sample. */
+  update_approx_time(start + 60);
+  sampled_guards_update_from_consensus(gs);
+
+  tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  tt_i64_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre-5);
+  for (i = 0; i < 5; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(! g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, start+60);
+  }
+  for (i = 5; i < n_sampled_pre; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+  }
 
-    tt_str_op(e->chosen_by_version, OP_EQ, tor_version); /* Verify version */
+  /* Now re-list one, and remove one completely. */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 0);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    n->is_possible_guard = 1;
+  }
+  {
+    /* try removing the node, to make sure we don't crash on an absent node
+     */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 5);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    smartlist_remove(big_fake_net_nodes, n);
+    tor_free(n->rs);
+    tor_free(n->md);
+    tor_free(n);
+  }
+  update_approx_time(start + 300);
+  sampled_guards_update_from_consensus(gs);
+
+  /* guards 1..5 are now unlisted; 0,6,7.. are listed. */
+  tt_i64_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  for (i = 1; i < 6; ++i) {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(! g->currently_listed);
+    if (i == 5)
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+300);
+    else
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+60);
+  }
+  for (i = 0; i < n_sampled_pre; i = (!i) ? 6 : i+1) { /* 0,6,7,8, ... */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    tt_assert(g->currently_listed);
+    tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+  }
 
-    tt_assert(e->made_contact); /* All saved guards have been contacted */
+ done:
+  tor_free(ns_tmp); /* in case we couldn't put it back */
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
+}
 
-    tt_assert(e->bad_since); /* Verify bad_since timestamp */
-    format_iso_time(str_time, e->bad_since);
-    tt_str_op(str_time, OP_EQ, unlisted_since);
+static void
+test_entry_guard_update_from_consensus_repair(void *arg)
+{
+  /* Here we'll make sure that our code to repair the unlisted-since
+   * times is correct. */
 
-    /* The rest should be unset */
-    tt_assert(!e->unreachable_since);
-    tt_assert(!e->can_retry);
-    tt_assert(!e->path_bias_noticed);
-    tt_assert(!e->path_bias_warned);
-    tt_assert(!e->path_bias_extreme);
-    tt_assert(!e->path_bias_disabled);
-    tt_assert(!e->path_bias_use_noticed);
-    tt_assert(!e->path_bias_use_extreme);
-    tt_assert(!e->last_attempted);
+  (void)arg;
+  int i;
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  /* Now corrupt the list a bit.  Call some unlisted-since-never, and some
+   * listed-and-unlisted-since-a-time. */
+  update_approx_time(start + 300);
+  for (i = 0; i < 3; ++i) {
+    /* these will get a date. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+  }
+  for (i = 3; i < 6; ++i) {
+    /* these will become listed. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    g->unlisted_since_date = start+100;
+  }
+  setup_full_capture_of_logs(LOG_WARN);
+  sampled_guards_update_from_consensus(gs);
+  expect_log_msg_containing(
+             "was listed, but with unlisted_since_date set");
+  expect_log_msg_containing(
+             "was unlisted, but with unlisted_since_date unset");
+  teardown_capture_of_logs();
+
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre);
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_filtered_pre-3);
+  for (i = 3; i < n_sampled_pre; ++i) {
+    /* these will become listed. */
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, i);
+    if (i < 3) {
+      tt_assert(! g->currently_listed);
+      tt_i64_op(g->unlisted_since_date, OP_EQ, start+300);
+    } else {
+      tt_assert(g->currently_listed);
+      tt_i64_op(g->unlisted_since_date, OP_EQ, 0);
+    }
   }
 
  done:
-  state_lines_free(entry_state_lines);
-  or_state_free(state);
-  tor_free(msg);
+  teardown_capture_of_logs();
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
 }
 
-/** Similar to test_entry_guards_parse_state_simple() but aims to test
-    the PathBias-related details of the entry guard. */
 static void
-test_entry_guards_parse_state_pathbias(void *arg)
+test_entry_guard_update_from_consensus_remove(void *arg)
 {
-  or_state_t *state = or_state_new();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  char *msg = NULL;
-  int retval;
-  smartlist_t *entry_state_lines = smartlist_new();
+  /* Now let's check the logic responsible for removing guards from the
+   * sample entirely. */
 
-  /* Path bias details of the fake guard */
-  const double circ_attempts = 9;
-  const double circ_successes = 8;
-  const double successful_closed = 4;
-  const double collapsed = 2;
-  const double unusable = 0;
-  const double timeouts = 1;
+  (void)arg;
+  //int i;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  smartlist_t *keep_ids = smartlist_new();
+  smartlist_t *remove_ids = smartlist_new();
+
+  /* Don't randomly backdate stuff; it will make correctness harder to check.*/
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* First, sample some guards. */
+  entry_guards_expand_sample(gs);
+  int n_sampled_pre = smartlist_len(gs->sampled_entry_guards);
+  int n_filtered_pre = num_reachable_filtered_guards(gs, NULL);
+  tt_i64_op(n_sampled_pre, OP_EQ, n_filtered_pre);
+  tt_i64_op(n_sampled_pre, OP_GT, 10);
+
+  const time_t one_day_ago = approx_time() - 1*24*60*60;
+  const time_t one_year_ago = approx_time() - 365*24*60*60;
+  const time_t two_years_ago = approx_time() - 2*365*24*60*60;
+  /* 0: unlisted for a day. (keep this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 0);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+    g->unlisted_since_date = one_day_ago;
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 1: unlisted for a year. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 1);
+    node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity);
+    n->is_possible_guard = 0;
+    g->currently_listed = 0;
+    g->unlisted_since_date = one_year_ago;
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
+  /* 2: added a day ago, never confirmed. (keep this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 2);
+    g->sampled_on_date = one_day_ago;
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 3: added a year ago, never confirmed. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 3);
+    g->sampled_on_date = one_year_ago;
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
+  /* 4: added two year ago, confirmed yesterday, primary. (keep this.) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 4);
+    g->sampled_on_date = one_year_ago;
+    g->confirmed_on_date = one_day_ago;
+    g->confirmed_idx = 0;
+    g->is_primary = 1;
+    smartlist_add(gs->confirmed_entry_guards, g);
+    smartlist_add(gs->primary_entry_guards, g);
+    smartlist_add(keep_ids, tor_memdup(g->identity, 20));
+  }
+  /* 5: added two years ago, confirmed a year ago, primary. (remove this) */
+  {
+    entry_guard_t *g = smartlist_get(gs->sampled_entry_guards, 5);
+    g->sampled_on_date = two_years_ago;
+    g->confirmed_on_date = one_year_ago;
+    g->confirmed_idx = 1;
+    g->is_primary = 1;
+    smartlist_add(gs->confirmed_entry_guards, g);
+    smartlist_add(gs->primary_entry_guards, g);
+    smartlist_add(remove_ids, tor_memdup(g->identity, 20));
+  }
 
-  (void) arg;
+  sampled_guards_update_from_consensus(gs);
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+  /* Did we remove the right ones? */
+  SMARTLIST_FOREACH(keep_ids, uint8_t *, id, {
+      tt_assert(get_sampled_guard_with_id(gs, id) != NULL);
+  });
+  SMARTLIST_FOREACH(remove_ids, uint8_t *, id, {
+    tt_want(get_sampled_guard_with_id(gs, id) == NULL);
+  });
 
-  { /* Prepare the state entry */
+  /* Did we remove the right number? */
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_sampled_pre - 3);
 
-    /* Prepare the smartlist to hold the key/value of each line */
-    smartlist_t *state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuard");
-    smartlist_add_asprintf(state_line,
-             "givethanks B29D536DD1752D542E1FBB3C9CE4449D51298212 NoDirCache");
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  guard_selection_free(gs);
+  UNMOCK(randomize_time);
+  SMARTLIST_FOREACH(keep_ids, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(remove_ids, char *, cp, tor_free(cp));
+  smartlist_free(keep_ids);
+  smartlist_free(remove_ids);
+}
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardAddedBy");
-    smartlist_add_asprintf(state_line,
-      "B29D536DD1752D542E1FBB3C9CE4449D51298212 0.2.5.3-alpha-dev "
-                           "%s", get_yesterday_date_str());
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_confirming_guards(void *arg)
+{
+  (void)arg;
+  /* Now let's check the logic responsible for manipulating the list
+   * of confirmed guards */
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  MOCK(randomize_time, mock_randomize_time_no_randomization);
+
+  /* Create the sample. */
+  entry_guards_expand_sample(gs);
+
+  /* Confirm a few  guards. */
+  time_t start = approx_time();
+  entry_guard_t *g1 = smartlist_get(gs->sampled_entry_guards, 0);
+  entry_guard_t *g2 = smartlist_get(gs->sampled_entry_guards, 1);
+  entry_guard_t *g3 = smartlist_get(gs->sampled_entry_guards, 8);
+  make_guard_confirmed(gs, g2);
+  update_approx_time(start + 10);
+  make_guard_confirmed(gs, g1);
+  make_guard_confirmed(gs, g3);
+
+  /* Were the correct dates and indices fed in? */
+  tt_int_op(g1->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_i64_op(g1->confirmed_on_date, OP_EQ, start+10);
+  tt_i64_op(g2->confirmed_on_date, OP_EQ, start);
+  tt_i64_op(g3->confirmed_on_date, OP_EQ, start+10);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
+
+  /* Now make sure we can regenerate the confirmed_entry_guards list. */
+  smartlist_clear(gs->confirmed_entry_guards);
+  g2->confirmed_idx = 0;
+  g1->confirmed_idx = 10;
+  g3->confirmed_idx = 100;
+  entry_guards_update_confirmed(gs);
+  tt_int_op(g1->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
+
+  /* Now make sure we can regenerate the confirmed_entry_guards list if
+   * the indices are messed up. */
+  g1->confirmed_idx = g2->confirmed_idx = g3->confirmed_idx = 999;
+  smartlist_clear(gs->confirmed_entry_guards);
+  entry_guards_update_confirmed(gs);
+  tt_int_op(g1->confirmed_idx, OP_GE, 0);
+  tt_int_op(g2->confirmed_idx, OP_GE, 0);
+  tt_int_op(g3->confirmed_idx, OP_GE, 0);
+  tt_int_op(g1->confirmed_idx, OP_LE, 2);
+  tt_int_op(g2->confirmed_idx, OP_LE, 2);
+  tt_int_op(g3->confirmed_idx, OP_LE, 2);
+  g1 = smartlist_get(gs->confirmed_entry_guards, 0);
+  g2 = smartlist_get(gs->confirmed_entry_guards, 1);
+  g3 = smartlist_get(gs->confirmed_entry_guards, 2);
+  tt_int_op(g1->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g2->confirmed_idx, OP_EQ, 1);
+  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_assert(g1 != g2);
+  tt_assert(g1 != g3);
+  tt_assert(g2 != g3);
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardUnlistedSince");
-    smartlist_add_asprintf(state_line, "2014-06-08 16:16:50");
-    smartlist_add(entry_state_lines, state_line);
+ done:
+  UNMOCK(randomize_time);
+  guard_selection_free(gs);
+}
 
-    state_line = smartlist_new();
-    smartlist_add_asprintf(state_line, "EntryGuardPathBias");
-    smartlist_add_asprintf(state_line, "%f %f %f %f %f %f",
-                           circ_attempts, circ_successes, successful_closed,
-                           collapsed, unusable, timeouts);
-    smartlist_add(entry_state_lines, state_line);
+static void
+test_entry_guard_sample_reachable_filtered(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guards_expand_sample(gs);
+  const int N = 10000;
+  bitarray_t *selected = NULL;
+  int i, j;
+
+  /* We've got a sampled list now; let's make one non-usable-filtered; some
+   * confirmed, some primary, some pending.
+   */
+  int n_guards = smartlist_len(gs->sampled_entry_guards);
+  tt_int_op(n_guards, OP_GT, 10);
+  entry_guard_t *g;
+  g = smartlist_get(gs->sampled_entry_guards, 0);
+  g->is_pending = 1;
+  g = smartlist_get(gs->sampled_entry_guards, 1);
+  make_guard_confirmed(gs, g);
+  g = smartlist_get(gs->sampled_entry_guards, 2);
+  g->is_primary = 1;
+  g = smartlist_get(gs->sampled_entry_guards, 3);
+  g->pb.path_bias_disabled = 1;
+
+  entry_guards_update_filtered_sets(gs);
+  gs->primary_guards_up_to_date = 1;
+  tt_int_op(num_reachable_filtered_guards(gs, NULL), OP_EQ, n_guards - 1);
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+
+  // +1 since the one we made disabled will make  another one get added.
+  ++n_guards;
+
+  /* Try a bunch of selections. */
+  const struct {
+    int flag; int idx;
+  } tests[] = {
+    { 0, -1 },
+    { SAMPLE_EXCLUDE_CONFIRMED, 1 },
+    { SAMPLE_EXCLUDE_PRIMARY|SAMPLE_NO_UPDATE_PRIMARY, 2 },
+    { SAMPLE_EXCLUDE_PENDING, 0 },
+    { -1, -1},
+  };
+
+  for (j = 0; tests[j].flag >= 0; ++j) {
+    selected = bitarray_init_zero(n_guards);
+    const int excluded_flags = tests[j].flag;
+    const int excluded_idx = tests[j].idx;
+    for (i = 0; i < N; ++i) {
+      g = sample_reachable_filtered_entry_guards(gs, NULL, excluded_flags);
+      tor_assert(g);
+      int pos = smartlist_pos(gs->sampled_entry_guards, g);
+      tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+      tt_int_op(pos, OP_GE, 0);
+      tt_int_op(pos, OP_LT, n_guards);
+      bitarray_set(selected, pos);
+    }
+    for (i = 0; i < n_guards; ++i) {
+      const int should_be_set = (i != excluded_idx &&
+                                 i != 3); // filtered out.
+      tt_int_op(!!bitarray_is_set(selected, i), OP_EQ, should_be_set);
+    }
+    bitarray_free(selected);
+    selected = NULL;
   }
 
-  /* Inject our lines in the state */
-  state_insert_entry_guard_helper(state, entry_state_lines);
+ done:
+  guard_selection_free(gs);
+  bitarray_free(selected);
+}
+
+static void
+test_entry_guard_sample_reachable_filtered_empty(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  /* What if we try to sample from a set of 0? */
+  SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
+                    n->is_possible_guard = 0);
 
-  /* Parse state */
-  retval = entry_guards_parse_state(state, 1, &msg);
-  tt_int_op(retval, OP_GE, 0);
+  entry_guard_t *g = sample_reachable_filtered_entry_guards(gs, NULL, 0);
+  tt_ptr_op(g, OP_EQ, NULL);
 
-  /* Test that the guard was registered */
-  all_entry_guards = get_entry_guards();
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+ done:
+  guard_selection_free(gs);
+}
 
-  { /* Test the path bias of this guard */
-    const entry_guard_t *e = smartlist_get(all_entry_guards, 0);
+static void
+test_entry_guard_retry_unreachable(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  entry_guards_expand_sample(gs);
+  /* Let's say that we have two guards, and they're down.
+   */
+  time_t start = approx_time();;
+  entry_guard_t *g1 = smartlist_get(gs->sampled_entry_guards, 0);
+  entry_guard_t *g2 = smartlist_get(gs->sampled_entry_guards, 1);
+  entry_guard_t *g3 = smartlist_get(gs->sampled_entry_guards, 2);
+  g1->is_reachable = GUARD_REACHABLE_NO;
+  g2->is_reachable = GUARD_REACHABLE_NO;
+  g1->is_primary = 1;
+  g1->failing_since = g2->failing_since = start;
+  g1->last_tried_to_connect = g2->last_tried_to_connect = start;
+
+  /* Wait 5 minutes.  Nothing will get retried. */
+  update_approx_time(start + 5 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  entry_guard_consider_retry(g3); // just to make sure this doesn't crash.
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g3->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  /* After 30 min, the primary one gets retried */
+  update_approx_time(start + 35 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+
+  g1->is_reachable = GUARD_REACHABLE_NO;
+  g1->last_tried_to_connect = start + 55*60;
+
+  /* After 1 hour, we'll retry the nonprimary one. */
+  update_approx_time(start + 61 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  g2->is_reachable = GUARD_REACHABLE_NO;
+  g2->last_tried_to_connect = start + 61*60;
+
+  /* And then the primary one again. */
+  update_approx_time(start + 66 * 60);
+  entry_guard_consider_retry(g1);
+  entry_guard_consider_retry(g2);
+  tt_int_op(g1->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
 
-    tt_assert(!e->is_dir_cache);
-    tt_assert(!e->can_retry);
+ done:
+  guard_selection_free(gs);
+}
 
-    /* XXX tt_double_op doesn't support equality. Cast to int for now. */
-    tt_int_op((int)e->circ_attempts, OP_EQ, (int)circ_attempts);
-    tt_int_op((int)e->circ_successes, OP_EQ, (int)circ_successes);
-    tt_int_op((int)e->successful_circuits_closed, OP_EQ,
-              (int)successful_closed);
-    tt_int_op((int)e->timeouts, OP_EQ, (int)timeouts);
-    tt_int_op((int)e->collapsed_circuits, OP_EQ, (int)collapsed);
-    tt_int_op((int)e->unusable_circuits, OP_EQ, (int)unusable);
+static void
+test_entry_guard_manage_primary(void *arg)
+{
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  smartlist_t *prev_guards = smartlist_new();
+
+  /* If no guards are confirmed, we should pick a few reachable guards and
+   * call them all primary. But not confirmed.*/
+  entry_guards_update_primary(gs);
+  int n_primary = smartlist_len(gs->primary_entry_guards);
+  tt_int_op(n_primary, OP_GE, 1);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_assert(g->is_primary);
+    tt_assert(g->confirmed_idx == -1);
+  });
+
+  /* Calling it a second time should leave the guards unchanged. */
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
+  });
+
+  /* If we have one confirmed guard, that guards becomes the first primary
+   * guard, and the other primary guards get kept. */
+
+  /* find a non-primary guard... */
+  entry_guard_t *confirmed = NULL;
+  SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, g, {
+    if (! g->is_primary) {
+      confirmed = g;
+      break;
+    }
+  });
+  tt_assert(confirmed);
+  /* make it confirmed. */
+  make_guard_confirmed(gs, confirmed);
+  /* update the list... */
+  smartlist_clear(prev_guards);
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+
+  /*  and see what's primary now! */
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  tt_ptr_op(smartlist_get(gs->primary_entry_guards, 0), OP_EQ, confirmed);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_assert(g->is_primary);
+    if (g_sl_idx == 0)
+      continue;
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx - 1));
+  });
+  {
+    entry_guard_t *prev_last_guard = smartlist_get(prev_guards, n_primary-1);
+    tt_assert(! prev_last_guard->is_primary);
   }
 
+  /* Calling it a fourth time should leave the guards unchanged. */
+  smartlist_clear(prev_guards);
+  smartlist_add_all(prev_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
+    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
+  });
+
  done:
-  or_state_free(state);
-  state_lines_free(entry_state_lines);
-  tor_free(msg);
+  guard_selection_free(gs);
+  smartlist_free(prev_guards);
 }
 
-/* Simple test of entry_guards_set_from_config() by specifying a
-   particular EntryNode and making sure it gets picked. */
 static void
-test_entry_guards_set_from_config(void *arg)
+test_entry_guard_guard_preferred(void *arg)
 {
-  or_options_t *options = get_options_mutable();
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  const char *entrynodes_str = "test003r";
-  const node_t *chosen_entry = NULL;
-  int retval;
-
   (void) arg;
+  entry_guard_t *g1 = tor_malloc_zero(sizeof(entry_guard_t));
+  entry_guard_t *g2 = tor_malloc_zero(sizeof(entry_guard_t));
+
+  g1->confirmed_idx = g2->confirmed_idx = -1;
+  g1->last_tried_to_connect = approx_time();
+  g2->last_tried_to_connect = approx_time();
+
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g1));
 
-  /* Prase EntryNodes as a routerset. */
-  options->EntryNodes = routerset_new();
-  retval = routerset_parse(options->EntryNodes,
-                           entrynodes_str,
-                           "test_entrynodes");
-  tt_int_op(retval, OP_GE, 0);
+  /* Neither is pending; priorities equal. */
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-  /* Read nodes from EntryNodes */
-  entry_guards_set_from_config(options);
+  /* If one is pending, the pending one has higher priority */
+  g1->is_pending = 1;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
 
-  /* Test that only one guard was added. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 1);
+  /* If both are pending, and last_tried_to_connect is equal:
+     priorities equal */
+  g2->is_pending = 1;
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
-  /* Make sure it was the guard we specified. */
-  chosen_entry = choose_random_entry(NULL);
-  tt_str_op(chosen_entry->ri->nickname, OP_EQ, entrynodes_str);
+  /* One had a connection that startied earlier: it has higher priority. */
+  g2->last_tried_to_connect -= 10;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+
+  /* Now, say that g1 is confirmed. It will get higher priority. */
+  g1->confirmed_idx = 5;
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g1, g2));
+
+  /* But if g2 was confirmed first, it will get priority */
+  g2->confirmed_idx = 2;
+  tt_int_op(1, OP_EQ, entry_guard_has_higher_priority(g2, g1));
+  tt_int_op(0, OP_EQ, entry_guard_has_higher_priority(g1, g2));
 
  done:
-  routerset_free(options->EntryNodes);
+  tor_free(g1);
+  tor_free(g2);
 }
 
 static void
-test_entry_is_time_to_retry(void *arg)
+test_entry_guard_select_for_circuit_no_confirmed(void *arg)
 {
-  entry_guard_t *test_guard;
-  time_t now;
-  int retval;
+  /* Simpler cases: no gaurds are confirmed yet. */
   (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  /* simple starting configuration */
+  entry_guards_update_primary(gs);
+  unsigned state = 9999;
+
+  entry_guard_t *g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                    NULL, &state);
+
+  tt_assert(g);
+  tt_assert(g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+  tt_assert(g->is_pending == 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we do that again, we should get the same guard.
+  entry_guard_t *g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                     NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+
+  // if we mark that guard down, we should get a different primary guard.
+  // auto-retry it.
+  g->is_reachable = GUARD_REACHABLE_NO;
+  g->failing_since = approx_time() - 10;
+  g->last_tried_to_connect = approx_time() - 10;
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_assert(g2);
+  tt_assert(g2->is_primary);
+  tt_int_op(g2->confirmed_idx, OP_EQ, -1);
+  tt_assert(g2->is_pending == 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we say that the first primary guard was last tried a long time ago, we
+  // should get an automatic retry on it.
+  g->failing_since = approx_time() - 72*60*60;
+  g->last_tried_to_connect = approx_time() - 72*60*60;
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+  tt_assert(g2);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  // And if we mark ALL the primary guards down, we should get another guard
+  // at random.
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    guard->is_reachable = GUARD_REACHABLE_NO;
+    guard->last_tried_to_connect = approx_time() - 5;
+    guard->failing_since = approx_time() - 30;
+  });
+  state = 9999;
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g2);
+  tt_assert(!g2->is_primary);
+  tt_int_op(g2->confirmed_idx, OP_EQ, -1);
+  tt_assert(g2->is_pending == 1);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+  tt_int_op(g2->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+
+  // As a bonus, maybe we should be retrying the primary guards. Let's say so.
+  mark_primary_guards_maybe_reachable(gs);
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    tt_int_op(guard->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+    tt_assert(guard->is_usable_filtered_guard == 1);
+    // no change to these fields.
+    tt_i64_op(guard->last_tried_to_connect, OP_EQ, approx_time() - 5);
+    tt_i64_op(guard->failing_since, OP_EQ, approx_time() - 30);
+  });
+
+  /* Let's try again and we should get the first primary guard again */
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g2, OP_EQ, g);
+
+  /* But if we impose a restriction, we don't get the same guard */
+  entry_guard_restriction_t rst;
+  memset(&rst, 0, sizeof(rst));
+  memcpy(rst.exclude_id, g->identity, DIGEST_LEN);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, &rst, &state);
+  tt_ptr_op(g2, OP_NE, g);
 
-  now = time(NULL);
+ done:
+  guard_selection_free(gs);
+}
 
-  test_guard = tor_malloc_zero(sizeof(entry_guard_t));
+static void
+test_entry_guard_select_for_circuit_confirmed(void *arg)
+{
+  /* Case 2: if all the primary guards are down, and there are more confirmed
+     guards, we use a confirmed guard. */
+  (void)arg;
+  int i;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  const int N_CONFIRMED = 10;
+
+  /* slightly more complicated simple starting configuration */
+  entry_guards_update_primary(gs);
+  for (i = 0; i < N_CONFIRMED; ++i) {
+    entry_guard_t *guard = smartlist_get(gs->sampled_entry_guards, i);
+    make_guard_confirmed(gs, guard);
+  }
+  entry_guards_update_primary(gs); // rebuild the primary list.
+
+  unsigned state = 9999;
+
+  // As above, this gives us a primary guard.
+  entry_guard_t *g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                    NULL, &state);
+  tt_assert(g);
+  tt_assert(g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+  tt_assert(g->is_pending == 0); // primary implies non-pending.
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+
+  // But if we mark all the primary guards down...
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, guard, {
+    guard->last_tried_to_connect = approx_time();
+    entry_guards_note_guard_failure(gs, guard);
+  });
+
+  // ... we should get a confirmed guard.
+  state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g);
+  tt_assert(! g->is_primary);
+  tt_int_op(g->confirmed_idx, OP_EQ, smartlist_len(gs->primary_entry_guards));
+  tt_assert(g->is_pending);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, approx_time());
+
+  // And if we try again, we should get a different confirmed guard, since
+  // that one is pending.
+  state = 9999;
+  entry_guard_t *g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC,
+                                                     NULL, &state);
+  tt_assert(g2);
+  tt_assert(! g2->is_primary);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_int_op(g2->confirmed_idx, OP_EQ,
+            smartlist_len(gs->primary_entry_guards)+1);
+  tt_assert(g2->is_pending);
+  tt_uint_op(state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_i64_op(g2->last_tried_to_connect, OP_EQ, approx_time());
+
+  // If we say that the next confirmed guard in order is excluded, and
+  // we disable EnforceDistinctSubnets, we get the guard AFTER the
+  // one we excluded.
+  get_options_mutable()->EnforceDistinctSubnets = 0;
+  g = smartlist_get(gs->confirmed_entry_guards,
+                     smartlist_len(gs->primary_entry_guards)+2);
+  entry_guard_restriction_t rst;
+  memset(&rst, 0, sizeof(rst));
+  memcpy(rst.exclude_id, g->identity, DIGEST_LEN);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, &rst, &state);
+  tt_ptr_op(g2, OP_NE, NULL);
+  tt_ptr_op(g2, OP_NE, g);
+  tt_int_op(g2->confirmed_idx, OP_EQ,
+            smartlist_len(gs->primary_entry_guards)+3);
+
+  // If we make every confirmed guard become pending then we start poking
+  // other guards.
+  const int n_remaining_confirmed =
+    N_CONFIRMED - 3 - smartlist_len(gs->primary_entry_guards);
+  for (i = 0; i < n_remaining_confirmed; ++i) {
+    g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+    tt_int_op(g->confirmed_idx, OP_GE, 0);
+    tt_assert(g);
+  }
+  state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_assert(g);
+  tt_assert(g->is_pending);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+
+  // If we EnforceDistinctSubnets and apply a restriction, we get
+  // nothing, since we put all of the nodes in the same /16.
+  // Regression test for bug 22753/TROVE-2017-006.
+  get_options_mutable()->EnforceDistinctSubnets = 1;
+  g = smartlist_get(gs->confirmed_entry_guards, 0);
+  memset(&rst, 0, sizeof(rst));
+  memcpy(rst.exclude_id, g->identity, DIGEST_LEN);
+  g2 = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, &rst, &state);
+  tt_ptr_op(g2, OP_EQ, NULL);
 
-  test_guard->last_attempted = now - 10;
-  test_guard->unreachable_since = now - 1;
+ done:
+  guard_selection_free(gs);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_select_for_circuit_highlevel_primary(void *arg)
+{
+  /* Play around with selecting primary guards for circuits and markign
+   * them up and down */
+  (void)arg;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+
+  time_t start = approx_time();
+
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard = NULL;
+  entry_guard_t *g;
+  guard_usable_t u;
+  /*
+   * Make sure that the pick-for-circuit API basically works.  We'll get
+   * a primary guard, so it'll be usable on completion.
+   */
+  int r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                       &node, &guard);
+
+  tt_assert(r == 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+
+  /* Call that circuit successful. */
+  update_approx_time(start+15);
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_USABLE_NOW); /* We can use it now. */
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_YES);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+
+  circuit_guard_state_free(guard);
+  guard = NULL;
+  node = NULL;
+  g = NULL;
+
+  /* Try again. We'll also get a primary guard this time. (The same one,
+     in fact.)  But this time, we'll say the connection has failed. */
+  update_approx_time(start+35);
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(r == 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  tt_i64_op(guard->state_set_at, OP_EQ, start+35);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start+35);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0); // same one.
+
+  /* It's failed!  What will happen to our poor guard? */
+  update_approx_time(start+45);
+  entry_guard_failed(&guard);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_DEAD);
+  tt_i64_op(guard->state_set_at, OP_EQ, start+45);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+  tt_i64_op(g->failing_since, OP_EQ, start+45);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0); // still confirmed.
+
+  circuit_guard_state_free(guard);
+  guard = NULL;
+  node = NULL;
+  entry_guard_t *g_prev = g;
+  g = NULL;
+
+  /* Now try a third time. Since the other one is down, we'll get a different
+   * (still primary) guard.
+   */
+  update_approx_time(start+60);
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(r == 0);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_ptr_op(g, OP_NE, g_prev);
+  tt_mem_op(g->identity, OP_EQ, node->identity, DIGEST_LEN);
+  tt_mem_op(g->identity, OP_NE, g_prev->identity, DIGEST_LEN);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_i64_op(g->last_tried_to_connect, OP_EQ, start+60);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1); // not confirmd now.
+
+  /* Call this one up; watch it get confirmed. */
+  update_approx_time(start+90);
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_USABLE_NOW);
+  tt_assert(guard);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_YES);
+  tt_int_op(g->confirmed_idx, OP_EQ, 1);
 
-  test_guard->unreachable_since = now - (6*60*60 - 1);
-  test_guard->last_attempted = now - (60*60 + 1);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_select_for_circuit_highlevel_confirm_other(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+
+  /* At the start, we have no confirmed guards.  We'll mark the primary guards
+   * down, then confirm something else.  As soon as we do, it should become
+   * primary, and we should get it next time. */
+
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard = NULL;
+  int i, r;
+  const node_t *node = NULL;
+  guard_usable_t u;
+
+  /* Declare that we're on the internet. */
+  entry_guards_note_internet_connectivity(gs);
+
+  /* Primary guards are down! */
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_assert(node);
+    tt_assert(guard);
+    tt_assert(r == 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    guard = NULL;
+    node = NULL;
+  }
 
-  test_guard->last_attempted = now - (60*60 - 1);
+  /* Next guard should be non-primary. */
+  node = NULL;
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_assert(r == 0);
+  entry_guard_t *g = entry_guard_handle_get(guard->guard);
+  tt_assert(g);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tt_int_op(g->confirmed_idx, OP_EQ, -1);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 1);
+  (void)start;
+
+  u = entry_guard_succeeded(&guard);
+  /* We're on the internet (by fiat), so this guard will get called "confirmed"
+   * and should immediately become primary.
+   */
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+  tt_assert(u == GUARD_USABLE_NOW);
+  tt_int_op(g->confirmed_idx, OP_EQ, 0);
+  tt_int_op(g->is_primary, OP_EQ, 1);
+  tt_int_op(g->is_pending, OP_EQ, 0);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,0);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-  test_guard->unreachable_since = now - (6*60*60 + 1);
-  test_guard->last_attempted = now - (4*60*60 + 1);
+static void
+test_entry_guard_select_for_circuit_highlevel_primary_retry(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+
+  /* At the start, we have no confirmed guards.  We'll mark the primary guards
+   * down, then confirm something else.  As soon as we do, it should become
+   * primary, and we should get it next time. */
+
+  time_t start = approx_time();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard = NULL, *guard2 = NULL;
+  int i, r;
+  const node_t *node = NULL;
+  entry_guard_t *g;
+  guard_usable_t u;
+
+  /* Declare that we're on the internet. */
+  entry_guards_note_internet_connectivity(gs);
+
+  /* Make primary guards confirmed (so they won't be superseded by a later
+   * guard), then mark them down. */
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_assert(node);
+    tt_assert(guard);
+    tt_assert(r == 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    g = entry_guard_handle_get(guard->guard);
+    make_guard_confirmed(gs, g);
+    tt_int_op(g->is_primary, OP_EQ, 1);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    tt_int_op(g->is_reachable, OP_EQ, GUARD_REACHABLE_NO);
+    guard = NULL;
+    node = NULL;
+  }
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  /* Get another guard that we might try. */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_assert(r == 0);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+
+  tt_assert(entry_guards_all_primary_guards_are_down(gs));
+
+  /* And an hour has passed ... */
+  update_approx_time(start + 3600);
+
+  /* Say that guard has succeeded! */
+  u = entry_guard_succeeded(&guard);
+  tt_int_op(u, OP_EQ, GUARD_MAYBE_USABLE_LATER);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+
+  /* The primary guards should have been marked up! */
+  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, pg, {
+    tt_int_op(pg->is_primary, OP_EQ, 1);
+    tt_ptr_op(g, OP_NE, pg);
+    tt_int_op(pg->is_reachable, OP_EQ, GUARD_REACHABLE_MAYBE);
+  });
+
+  /* Have a circuit to a primary guard succeed. */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard2);
+  tt_assert(r == 0);
+  tt_int_op(guard2->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+  u = entry_guard_succeeded(&guard2);
+  tt_assert(u == GUARD_USABLE_NOW);
+  tt_int_op(guard2->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  tt_assert(! entry_guards_all_primary_guards_are_down(gs));
 
-  test_guard->unreachable_since = now - (3*24*60*60 - 1);
-  test_guard->last_attempted = now - (4*60*60 + 1);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+  circuit_guard_state_free(guard2);
+}
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+static void
+test_entry_guard_select_and_cancel(void *arg)
+{
+  (void) arg;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+  int i,r;
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard;
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guard_t *g;
+
+  /* Once more, we mark all the primary guards down. */
+  entry_guards_note_internet_connectivity(gs);
+  for (i = 0; i < N_PRIMARY; ++i) {
+    r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                     &node, &guard);
+    tt_int_op(r, OP_EQ, 0);
+    tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_ON_COMPLETION);
+    g = entry_guard_handle_get(guard->guard);
+    tt_int_op(g->is_primary, OP_EQ, 1);
+    tt_int_op(g->is_pending, OP_EQ, 0);
+    make_guard_confirmed(gs, g);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+    guard = NULL;
+    node = NULL;
+  }
 
-  test_guard->unreachable_since = now - (3*24*60*60 + 1);
-  test_guard->last_attempted = now - (18*60*60 + 1);
+  tt_assert(entry_guards_all_primary_guards_are_down(gs));
+
+  /* Now get another guard we could try... */
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_assert(node);
+  tt_assert(guard);
+  tt_assert(r == 0);
+  tt_int_op(guard->state, OP_EQ, GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  g = entry_guard_handle_get(guard->guard);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 1);
+
+  /* Whoops! We should never have asked for this guard. Cancel the request! */
+  entry_guard_cancel(&guard);
+  tt_assert(guard == NULL);
+  tt_int_op(g->is_primary, OP_EQ, 0);
+  tt_int_op(g->is_pending, OP_EQ, 0);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  guard_selection_free(gs);
+  circuit_guard_state_free(guard);
+}
 
-  test_guard->unreachable_since = now - (7*24*60*60 - 1);
-  test_guard->last_attempted = now - (18*60*60 + 1);
+static void
+test_entry_guard_drop_guards(void *arg)
+{
+  (void) arg;
+  int r;
+  const node_t *node = NULL;
+  circuit_guard_state_t *guard;
+  guard_selection_t *gs = get_guard_selection_info();
+
+  // Pick a guard, to get things set up.
+  r = entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                                   &node, &guard);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_GE,
+            DFLT_MIN_FILTERED_SAMPLE_SIZE);
+  tt_ptr_op(gs, OP_EQ, get_guard_selection_info());
+
+  // Drop all the guards!  (This is a bad idea....)
+  remove_all_entry_guards_for_guard_selection(gs);
+  gs = get_guard_selection_info();
+  tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, 0);
+  tt_int_op(smartlist_len(gs->confirmed_entry_guards), OP_EQ, 0);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+ done:
+  circuit_guard_state_free(guard);
+  guard_selection_free(gs);
+}
 
-  test_guard->last_attempted = now - (18*60*60 - 1);
+/* Unit test setup function: Create a fake network, and set everything up
+ * for testing the upgrade-a-waiting-circuit code. */
+typedef struct {
+  guard_selection_t *gs;
+  time_t start;
+  circuit_guard_state_t *guard1_state;
+  circuit_guard_state_t *guard2_state;
+  entry_guard_t *guard1;
+  entry_guard_t *guard2;
+  origin_circuit_t *circ1;
+  origin_circuit_t *circ2;
+  smartlist_t *all_origin_circuits;
+} upgrade_circuits_data_t;
+static void *
+upgrade_circuits_setup(const struct testcase_t *testcase)
+{
+  upgrade_circuits_data_t *data = tor_malloc_zero(sizeof(*data));
+  guard_selection_t *gs = data->gs =
+    guard_selection_new("default", GS_TYPE_NORMAL);
+  circuit_guard_state_t *guard;
+  const node_t *node;
+  entry_guard_t *g;
+  int i;
+  const int N_PRIMARY = DFLT_N_PRIMARY_GUARDS;
+  const char *argument = testcase->setup_data;
+  const int make_circ1_succeed = strstr(argument, "c1-done") != NULL;
+  const int make_circ2_succeed = strstr(argument, "c2-done") != NULL;
+
+  big_fake_network_setup(testcase);
+
+  /* We're going to set things up in a state where a circuit will be ready to
+   * be upgraded.  Each test can make a single change (or not) that should
+   * block the upgrade.
+   */
+
+  /* First, make all the primary guards confirmed, and down. */
+  data->start = approx_time();
+  entry_guards_note_internet_connectivity(gs);
+  for (i = 0; i < N_PRIMARY; ++i) {
+    entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &node, &guard);
+    g = entry_guard_handle_get(guard->guard);
+    make_guard_confirmed(gs, g);
+    entry_guard_failed(&guard);
+    circuit_guard_state_free(guard);
+  }
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,0);
+  /* Grab another couple of guards */
+  data->all_origin_circuits = smartlist_new();
+
+  update_approx_time(data->start + 27);
+  entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                               &node, &data->guard1_state);
+  origin_circuit_t *circ;
+  data->circ1 = circ = origin_circuit_new();
+  circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circ->guard_state = data->guard1_state;
+  smartlist_add(data->all_origin_circuits, circ);
+
+  update_approx_time(data->start + 30);
+  entry_guard_pick_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL,
+                               &node, &data->guard2_state);
+  data->circ2 = circ = origin_circuit_new();
+  circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circ->guard_state = data->guard2_state;
+  smartlist_add(data->all_origin_circuits, circ);
+
+  data->guard1 = entry_guard_handle_get(data->guard1_state->guard);
+  data->guard2 = entry_guard_handle_get(data->guard2_state->guard);
+  tor_assert(data->guard1 != data->guard2);
+  tor_assert(data->guard1_state->state ==
+             GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+  tor_assert(data->guard2_state->state ==
+             GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD);
+
+  guard_usable_t r;
+  update_approx_time(data->start + 32);
+  if (make_circ1_succeed) {
+    r = entry_guard_succeeded(&data->guard1_state);
+    tor_assert(r == GUARD_MAYBE_USABLE_LATER);
+    tor_assert(data->guard1_state->state ==
+               GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  }
+  update_approx_time(data->start + 33);
+  if (make_circ2_succeed) {
+    r = entry_guard_succeeded(&data->guard2_state);
+    tor_assert(r == GUARD_MAYBE_USABLE_LATER);
+    tor_assert(data->guard2_state->state ==
+               GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD);
+  }
 
-  test_guard->unreachable_since = now - (7*24*60*60 + 1);
-  test_guard->last_attempted = now - (36*60*60 + 1);
+  return data;
+}
+static int
+upgrade_circuits_cleanup(const struct testcase_t *testcase, void *ptr)
+{
+  upgrade_circuits_data_t *data = ptr;
+  // circuit_guard_state_free(data->guard1_state); // held in circ1
+  // circuit_guard_state_free(data->guard2_state); // held in circ2
+  guard_selection_free(data->gs);
+  smartlist_free(data->all_origin_circuits);
+  circuit_free(TO_CIRCUIT(data->circ1));
+  circuit_free(TO_CIRCUIT(data->circ2));
+  tor_free(data);
+  return big_fake_network_cleanup(testcase, NULL);
+}
+
+static void
+test_entry_guard_upgrade_a_circuit(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  /* This is the easy case: we have no COMPLETED circuits, all the
+   * primary guards are down, we have two WAITING circuits: one will
+   * get upgraded to COMPLETED!  (The one that started first.)
+   */
 
-  test_guard->unreachable_since = now - (7*24*60*60 + 1);
-  test_guard->last_attempted = now - (36*60*60 + 1);
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
 
-  retval = entry_is_time_to_retry(test_guard,now);
-  tt_int_op(retval,OP_EQ,1);
+  /* circ1 was started first, so we'll get told to ugrade it... */
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+
+  /* And the guard state should be complete */
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
 
  done:
-  tor_free(test_guard);
+  smartlist_free(result);
 }
 
-/** XXX Do some tests that entry_is_live() */
 static void
-test_entry_is_live(void *arg)
+test_entry_guard_upgrade_blocked_by_live_primary_guards(void *arg)
 {
-  smartlist_t *our_nodelist = NULL;
-  const smartlist_t *all_entry_guards = get_entry_guards();
-  const node_t *test_node = NULL;
-  const entry_guard_t *test_entry = NULL;
-  const char *msg;
-  int which_node;
-
-  (void) arg;
+  upgrade_circuits_data_t *data = arg;
+
+  /* If any primary guards might be up, we can't upgrade any waiting
+   * circuits.
+   */
+  mark_primary_guards_maybe_reachable(data->gs);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("not all primary guards were definitely down.");
 
-  /* The global entry guards smartlist should be empty now. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ, 0);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  /* Walk the nodelist and add all nodes as entry guards. */
-  our_nodelist = nodelist_get_list();
-  tt_int_op(smartlist_len(our_nodelist), OP_EQ, HELPER_NUMBER_OF_DESCRIPTORS);
+static void
+test_entry_guard_upgrade_blocked_by_lack_of_waiting_circuits(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* If no circuits are waiting, we can't upgrade anything.  (The test
+   * setup in this case was told not to make any of the circuits "waiting".)
+   */
+  smartlist_t *result = smartlist_new();
+  int r;
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("Considered upgrading guard-stalled circuits, "
+                            "but didn't find any.");
 
-  SMARTLIST_FOREACH_BEGIN(our_nodelist, const node_t *, node) {
-    const node_t *node_tmp;
-    node_tmp = add_an_entry_guard(node, 0, 1, 0, 0);
-    tt_assert(node_tmp);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-    tt_int_op(node->is_stable, OP_EQ, 0);
-    tt_int_op(node->is_fast, OP_EQ, 0);
-  } SMARTLIST_FOREACH_END(node);
+static void
+test_entry_guard_upgrade_blocked_by_better_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* We'll run through the logic of upgrade_a_circuit below...
+   * and then try again to make sure that circ2 isn't also upgraded.
+   */
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  /* Now, try again. Make sure that circ2 isn't upgraded. */
+  smartlist_clear(result);
+  setup_capture_of_logs(LOG_DEBUG);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("At least one complete circuit had higher "
+                            "priority, so not upgrading.");
 
-  /* Make sure the nodes were added as entry guards. */
-  tt_int_op(smartlist_len(all_entry_guards), OP_EQ,
-            HELPER_NUMBER_OF_DESCRIPTORS);
+ done:
+  teardown_capture_of_logs();
+  smartlist_free(result);
+}
 
-  /* Now get a random test entry that we will use for this unit test. */
-  which_node = 3;  /* (chosen by fair dice roll) */
-  test_entry = smartlist_get(all_entry_guards, which_node);
+static void
+test_entry_guard_upgrade_not_blocked_by_restricted_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* Once more, let circ1 become complete. But this time, we'll claim
+   * that circ2 was restricted to not use the same guard as circ1. */
+  data->guard2_state->restrictions =
+    tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  memcpy(data->guard2_state->restrictions->exclude_id,
+         data->guard1->identity, DIGEST_LEN);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
+  tt_ptr_op(data->guard1_state, OP_NE, NULL);
+  tt_int_op(data->guard1_state->state, OP_EQ, GUARD_CIRC_STATE_COMPLETE);
+
+  /* Now, we try again. Since circ2 has a restriction that circ1 doesn't obey,
+   * circ2 _is_ eligible for upgrade. */
+  smartlist_clear(result);
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc2 = smartlist_get(result, 0);
+  tt_ptr_op(oc2, OP_EQ, data->circ2);
 
-  /* Let's do some entry_is_live() tests! */
+ done:
+  smartlist_free(result);
+}
 
-  /* Require the node to be stable, but it's not. Should fail.
-     Also enable 'assume_reachable' because why not. */
-  test_node = entry_is_live(test_entry,
-                            ENTRY_NEED_UPTIME | ENTRY_ASSUME_REACHABLE,
-                            &msg);
-  tt_assert(!test_node);
+static void
+test_entry_guard_upgrade_not_blocked_by_worse_circ_complete(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+  smartlist_t *result = smartlist_new();
+  /* here we manually make circ2 COMPLETE, and make sure that circ1
+   * gets made complete anyway, since guard1 has higher priority
+   */
+  update_approx_time(data->start + 300);
+  data->guard2_state->state = GUARD_CIRC_STATE_COMPLETE;
+  data->guard2_state->state_set_at = approx_time();
+  update_approx_time(data->start + 301);
+
+  /* Now, try again. Make sure that circ1 is approved. */
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  /* Require the node to be fast, but it's not. Should fail. */
-  test_node = entry_is_live(test_entry,
-                            ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE,
-                            &msg);
-  tt_assert(!test_node);
+ done:
+  smartlist_free(result);
+}
 
-  /* Don't impose any restrictions on the node. Should succeed. */
-  test_node = entry_is_live(test_entry, 0, &msg);
-  tt_assert(test_node);
-  tt_ptr_op(test_node, OP_EQ, node_get_by_id(test_entry->identity));
+static void
+test_entry_guard_upgrade_blocked_by_better_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* circ2 is done, but circ1 is still pending. Since circ1 is better,
+   * we won't upgrade circ2. */
+
+  /* XXXX Prop271 -- this is a kludge.  I'm making sure circ1 _is_ better,
+   * by messing with the guards' confirmed_idx */
+  make_guard_confirmed(data->gs, data->guard1);
+  {
+    int tmp;
+    tmp = data->guard1->confirmed_idx;
+    data->guard1->confirmed_idx = data->guard2->confirmed_idx;
+    data->guard2->confirmed_idx = tmp;
+  }
 
-  /* Require descriptor for this node. It has one so it should succeed. */
-  test_node = entry_is_live(test_entry, ENTRY_NEED_DESCRIPTOR, &msg);
-  tt_assert(test_node);
-  tt_ptr_op(test_node, OP_EQ, node_get_by_id(test_entry->identity));
+  smartlist_t *result = smartlist_new();
+  setup_capture_of_logs(LOG_DEBUG);
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(result), OP_EQ, 0);
+  expect_log_msg_containing("but 1 pending circuit(s) had higher guard "
+                            "priority, so not upgrading.");
 
  done:
-  ; /* XXX */
+  teardown_capture_of_logs();
+  smartlist_free(result);
 }
 
-#define TEST_IPV4_ADDR "123.45.67.89"
-#define TEST_IPV6_ADDR "[1234:5678:90ab:cdef::]"
-
 static void
-test_node_preferred_orport(void *arg)
+test_entry_guard_upgrade_not_blocked_by_restricted_circ_pending(void *arg)
 {
-  (void)arg;
-  tor_addr_t ipv4_addr;
-  const uint16_t ipv4_port = 4444;
-  tor_addr_t ipv6_addr;
-  const uint16_t ipv6_port = 6666;
-  routerinfo_t node_ri;
-  node_t node;
-  tor_addr_port_t ap;
+  upgrade_circuits_data_t *data = arg;
+  /* circ2 is done, but circ1 is still pending. But when there is a
+     restriction on circ2 that circ1 can't satisfy, circ1 can't block
+     circ2. */
+
+  /* XXXX Prop271 -- this is a kludge.  I'm making sure circ1 _is_ better,
+   * by messing with the guards' confirmed_idx */
+  make_guard_confirmed(data->gs, data->guard1);
+  {
+    int tmp;
+    tmp = data->guard1->confirmed_idx;
+    data->guard1->confirmed_idx = data->guard2->confirmed_idx;
+    data->guard2->confirmed_idx = tmp;
+  }
 
-  /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
-  /* We don't test ClientPreferIPv6ORPort here, because it's used in
-   * nodelist_set_consensus to setup node.ipv6_preferred, which we set
-   * directly. */
-  MOCK(get_options, mock_get_options);
+  data->guard2_state->restrictions =
+    tor_malloc_zero(sizeof(entry_guard_restriction_t));
+  memcpy(data->guard2_state->restrictions->exclude_id,
+         data->guard1->identity, DIGEST_LEN);
+
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ2);
 
-  /* Setup IP addresses */
-  tor_addr_parse(&ipv4_addr, TEST_IPV4_ADDR);
-  tor_addr_parse(&ipv6_addr, TEST_IPV6_ADDR);
+ done:
+  smartlist_free(result);
+}
 
-  /* Setup node_ri */
-  memset(&node_ri, 0, sizeof(node_ri));
-  node_ri.addr = tor_addr_to_ipv4h(&ipv4_addr);
-  node_ri.or_port = ipv4_port;
-  tor_addr_copy(&node_ri.ipv6_addr, &ipv6_addr);
-  node_ri.ipv6_orport = ipv6_port;
+static void
+test_entry_guard_upgrade_not_blocked_by_worse_circ_pending(void *arg)
+{
+  upgrade_circuits_data_t *data = arg;
+
+  /* circ1 is done, but circ2 is still pending. Since circ1 is better,
+   * we will upgrade it. */
+  smartlist_t *result = smartlist_new();
+  int r;
+  r = entry_guards_upgrade_waiting_circuits(data->gs,
+                                            data->all_origin_circuits,
+                                            result);
+  tt_int_op(r, OP_EQ, 1);
+  tt_int_op(smartlist_len(result), OP_EQ, 1);
+  origin_circuit_t *oc = smartlist_get(result, 0);
+  tt_ptr_op(oc, OP_EQ, data->circ1);
 
-  /* Setup node */
-  memset(&node, 0, sizeof(node));
-  node.ri = &node_ri;
+ done:
+  smartlist_free(result);
+}
 
-  /* Check the preferred address is IPv4 if we're only using IPv4, regardless
-   * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
-  node.ipv6_preferred = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+static void
+test_enty_guard_should_expire_waiting(void *arg)
+{
+  (void)arg;
+  circuit_guard_state_t *fake_state = tor_malloc_zero(sizeof(*fake_state));
+  /* We'll leave "guard" unset -- it won't matter here. */
 
-  node.ipv6_preferred = 1;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+  /* No state? Can't expire. */
+  tt_assert(! entry_guard_state_should_expire(NULL));
 
-  /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
-   * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
-  tt_assert(ap.port == ipv4_port);
+  /* Let's try one that expires. */
+  fake_state->state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+  fake_state->state_set_at =
+    approx_time() - DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT - 1;
 
-  /* Check the preferred address is IPv6 if we prefer it and
-   * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = 1;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+  tt_assert(entry_guard_state_should_expire(fake_state));
 
-  mocked_options.ClientUseIPv4 = 0;
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+  /* But it wouldn't expire if we changed the state. */
+  fake_state->state = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
+  tt_assert(! entry_guard_state_should_expire(fake_state));
 
-  /* Check the preferred address is IPv6 if we don't prefer it, but
-   * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
-  node_get_pref_orport(&node, &ap);
-  tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
-  tt_assert(ap.port == ipv6_port);
+  /* And it wouldn't have expired a few seconds ago. */
+  fake_state->state = GUARD_CIRC_STATE_WAITING_FOR_BETTER_GUARD;
+  fake_state->state_set_at =
+    approx_time() - DFLT_NONPRIMARY_GUARD_IDLE_TIMEOUT + 5;
+  tt_assert(! entry_guard_state_should_expire(fake_state));
 
  done:
-  UNMOCK(get_options);
+  tor_free(fake_state);
 }
 
-static const struct testcase_setup_t fake_network = {
-  fake_network_setup, fake_network_cleanup
+static const struct testcase_setup_t big_fake_network = {
+  big_fake_network_setup, big_fake_network_cleanup
+};
+
+static const struct testcase_setup_t upgrade_circuits = {
+  upgrade_circuits_setup, upgrade_circuits_cleanup
 };
 
+#define BFN_TEST(name) \
+  { #name, test_entry_guard_ ## name, TT_FORK, &big_fake_network, NULL }
+
+#define UPGRADE_TEST(name, arg)                                         \
+  { #name, test_entry_guard_ ## name, TT_FORK, &upgrade_circuits,       \
+      (void*)(arg) }
+
 struct testcase_t entrynodes_tests[] = {
-  { "entry_is_time_to_retry", test_entry_is_time_to_retry,
-    TT_FORK, NULL, NULL },
-  { "choose_random_entry_no_guards", test_choose_random_entry_no_guards,
-    TT_FORK, &fake_network, NULL },
-  { "choose_random_entry_one_possibleguard",
-    test_choose_random_entry_one_possible_guard,
-    TT_FORK, &fake_network, NULL },
-  { "populate_live_entry_guards_1guard",
-    test_populate_live_entry_guards_1guard,
-    TT_FORK, &fake_network, NULL },
-  { "populate_live_entry_guards_3guards",
-    test_populate_live_entry_guards_3guards,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_parse_state_simple",
-    test_entry_guards_parse_state_simple,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_parse_state_pathbias",
-    test_entry_guards_parse_state_pathbias,
-    TT_FORK, &fake_network, NULL },
-  { "entry_guards_set_from_config",
-    test_entry_guards_set_from_config,
-    TT_FORK, &fake_network, NULL },
-  { "entry_is_live",
-    test_entry_is_live,
-    TT_FORK, &fake_network, NULL },
   { "node_preferred_orport",
     test_node_preferred_orport,
     0, NULL, NULL },
+  { "entry_guard_describe", test_entry_guard_describe, 0, NULL, NULL },
+  { "randomize_time", test_entry_guard_randomize_time, 0, NULL, NULL },
+  { "encode_for_state_minimal",
+    test_entry_guard_encode_for_state_minimal, 0, NULL, NULL },
+  { "encode_for_state_maximal",
+    test_entry_guard_encode_for_state_maximal, 0, NULL, NULL },
+  { "parse_from_state_minimal",
+    test_entry_guard_parse_from_state_minimal, 0, NULL, NULL },
+  { "parse_from_state_maximal",
+    test_entry_guard_parse_from_state_maximal, 0, NULL, NULL },
+  { "parse_from_state_failure",
+    test_entry_guard_parse_from_state_failure, 0, NULL, NULL },
+  { "parse_from_state_partial_failure",
+    test_entry_guard_parse_from_state_partial_failure, 0, NULL, NULL },
+  { "parse_from_state_full",
+    test_entry_guard_parse_from_state_full, TT_FORK, NULL, NULL },
+  { "parse_from_state_broken",
+    test_entry_guard_parse_from_state_broken, TT_FORK, NULL, NULL },
+  { "get_guard_selection_by_name",
+    test_entry_guard_get_guard_selection_by_name, TT_FORK, NULL, NULL },
+  BFN_TEST(choose_selection_initial),
+  BFN_TEST(add_single_guard),
+  BFN_TEST(node_filter),
+  BFN_TEST(expand_sample),
+  BFN_TEST(expand_sample_small_net),
+  BFN_TEST(update_from_consensus_status),
+  BFN_TEST(update_from_consensus_repair),
+  BFN_TEST(update_from_consensus_remove),
+  BFN_TEST(confirming_guards),
+  BFN_TEST(sample_reachable_filtered),
+  BFN_TEST(sample_reachable_filtered_empty),
+  BFN_TEST(retry_unreachable),
+  BFN_TEST(manage_primary),
+  { "guard_preferred", test_entry_guard_guard_preferred, TT_FORK, NULL, NULL },
+  BFN_TEST(select_for_circuit_no_confirmed),
+  BFN_TEST(select_for_circuit_confirmed),
+  BFN_TEST(select_for_circuit_highlevel_primary),
+  BFN_TEST(select_for_circuit_highlevel_confirm_other),
+  BFN_TEST(select_for_circuit_highlevel_primary_retry),
+  BFN_TEST(select_and_cancel),
+  BFN_TEST(drop_guards),
+
+  UPGRADE_TEST(upgrade_a_circuit, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_live_primary_guards, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_lack_of_waiting_circuits, ""),
+  UPGRADE_TEST(upgrade_blocked_by_better_circ_complete, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_complete,
+               "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_complete, "c1-done c2-done"),
+  UPGRADE_TEST(upgrade_blocked_by_better_circ_pending, "c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_restricted_circ_pending,
+               "c2-done"),
+  UPGRADE_TEST(upgrade_not_blocked_by_worse_circ_pending, "c1-done"),
+  { "should_expire_waiting", test_enty_guard_should_expire_waiting, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 1f92780177..fc9f27a5ac 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
@@ -72,9 +72,9 @@ test_ext_or_id_map(void *arg)
  * writes to outbuf. */
 static void
 connection_write_to_buf_impl_replacement(const char *string, size_t len,
-                                         connection_t *conn, int zlib)
+                                         connection_t *conn, int compressed)
 {
-  (void) zlib;
+  (void) compressed;
 
   tor_assert(string);
   tor_assert(conn);
diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c
index 8173e44d47..56006f3cc3 100644
--- a/src/test/test_guardfraction.c
+++ b/src/test/test_guardfraction.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRSERV_PRIVATE
diff --git a/src/test/test_handles.c b/src/test/test_handles.c
index 536a478689..7ddee6e376 100644
--- a/src/test/test_handles.c
+++ b/src/test/test_handles.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index ae9fc7a243..9fada5a675 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,8 +10,10 @@
 #include "orconfig.h"
 #include "or.h"
 
+#include "relay.h"
 #include "routerlist.h"
 #include "nodelist.h"
+#include "buffers.h"
 
 #include "test.h"
 #include "test_helpers.h"
@@ -22,6 +24,8 @@ DISABLE_GCC_WARNING(overlength-strings)
  * at large. */
 #endif
 #include "test_descriptors.inc"
+#include "or.h"
+#include "circuitlist.h"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 ENABLE_GCC_WARNING(overlength-strings)
 #endif
@@ -92,3 +96,50 @@ helper_setup_fake_routerlist(void)
   UNMOCK(router_descriptor_is_older_than);
 }
 
+void
+connection_write_to_buf_mock(const char *string, size_t len,
+                             connection_t *conn, int compressed)
+{
+  (void) compressed;
+
+  tor_assert(string);
+  tor_assert(conn);
+
+  write_to_buf(string, len, conn->outbuf);
+}
+
+/* Set up a fake origin circuit with the specified number of cells,
+ * Return a pointer to the newly-created dummy circuit */
+circuit_t *
+dummy_origin_circuit_new(int n_cells)
+{
+  origin_circuit_t *circ = origin_circuit_new();
+  int i;
+  cell_t cell;
+
+  for (i=0; i < n_cells; ++i) {
+    crypto_rand((void*)&cell, sizeof(cell));
+    cell_queue_append_packed_copy(TO_CIRCUIT(circ),
+                                  &TO_CIRCUIT(circ)->n_chan_cells,
+                                  1, &cell, 1, 0);
+  }
+
+  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  return TO_CIRCUIT(circ);
+}
+
+/** Mock-replacement. As tor_addr_lookup, but always fails on any
+ * address containing a !.  This is necessary for running the unit tests
+ * on networks where DNS hijackers think it's helpful to give answers
+ * for things like 1.2.3.4.5 or "invalidstuff!!"
+ */
+int
+mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
+                                        uint16_t family, tor_addr_t *out)
+{
+  if (name && strchr(name, '!')) {
+    return -1;
+  }
+  return tor_addr_lookup__real(name, family, out);
+}
+
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 684375e1b1..4621631cc1 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_HELPERS_H
@@ -6,11 +6,20 @@
 
 const char *get_yesterday_date_str(void);
 
+circuit_t * dummy_origin_circuit_new(int num_cells);
+
 /* Number of descriptors contained in test_descriptors.txt. */
 #define HELPER_NUMBER_OF_DESCRIPTORS 8
 
 void helper_setup_fake_routerlist(void);
 
+#define GET(path) "GET " path " HTTP/1.0\r\n\r\n"
+void connection_write_to_buf_mock(const char *string, size_t len,
+                                  connection_t *conn, int compressed);
+
+int mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
+                                            uint16_t family, tor_addr_t *out);
+
 extern const char TEST_DESCRIPTORS[];
 
 #endif
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index d572dd8d2e..cd71f025a3 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 #include "test.h"
 #include "control.h"
 #include "config.h"
+#include "hs_common.h"
 #include "rendcommon.h"
 #include "rendservice.h"
 #include "routerset.h"
@@ -136,7 +137,7 @@ test_hs_desc_event(void *arg)
   #define STR_DESC_ID_BASE32 "hba3gmcgpfivzfhx5rtfqkfdhv65yrj3"
 
   int ret;
-  rend_data_t rend_query;
+  rend_data_v2_t rend_query;
   const char *expected_msg;
   char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
 
@@ -148,12 +149,13 @@ test_hs_desc_event(void *arg)
 
   /* setup rend_query struct */
   memset(&rend_query, 0, sizeof(rend_query));
+  rend_query.base_.version = 2;
   strncpy(rend_query.onion_address, STR_HS_ADDR,
           REND_SERVICE_ID_LEN_BASE32+1);
   rend_query.auth_type = REND_NO_AUTH;
-  rend_query.hsdirs_fp = smartlist_new();
-  smartlist_add(rend_query.hsdirs_fp, tor_memdup(HSDIR_EXIST_ID,
-                                                 DIGEST_LEN));
+  rend_query.base_.hsdirs_fp = smartlist_new();
+  smartlist_add(rend_query.base_.hsdirs_fp, tor_memdup(HSDIR_EXIST_ID,
+                                                       DIGEST_LEN));
 
   /* Compute descriptor ID for replica 0, should be STR_DESC_ID_BASE32. */
   ret = rend_compute_v2_desc_id(rend_query.descriptor_id[0],
@@ -167,7 +169,7 @@ test_hs_desc_event(void *arg)
             sizeof(desc_id_base32));
 
   /* test request event */
-  control_event_hs_descriptor_requested(&rend_query, HSDIR_EXIST_ID,
+  control_event_hs_descriptor_requested(&rend_query.base_, HSDIR_EXIST_ID,
                                         STR_DESC_ID_BASE32);
   expected_msg = "650 HS_DESC REQUESTED "STR_HS_ADDR" NO_AUTH "\
                   STR_HSDIR_EXIST_LONGNAME " " STR_DESC_ID_BASE32 "\r\n";
@@ -178,7 +180,7 @@ test_hs_desc_event(void *arg)
   /* test received event */
   rend_query.auth_type = REND_BASIC_AUTH;
   control_event_hs_descriptor_received(rend_query.onion_address,
-                                       &rend_query, HSDIR_EXIST_ID);
+                                       &rend_query.base_, HSDIR_EXIST_ID);
   expected_msg = "650 HS_DESC RECEIVED "STR_HS_ADDR" BASIC_AUTH "\
                   STR_HSDIR_EXIST_LONGNAME " " STR_DESC_ID_BASE32"\r\n";
   tt_assert(received_msg);
@@ -187,7 +189,7 @@ test_hs_desc_event(void *arg)
 
   /* test failed event */
   rend_query.auth_type = REND_STEALTH_AUTH;
-  control_event_hs_descriptor_failed(&rend_query,
+  control_event_hs_descriptor_failed(&rend_query.base_,
                                      HSDIR_NONE_EXIST_ID,
                                      "QUERY_REJECTED");
   expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" STEALTH_AUTH "\
@@ -198,7 +200,7 @@ test_hs_desc_event(void *arg)
 
   /* test invalid auth type */
   rend_query.auth_type = 999;
-  control_event_hs_descriptor_failed(&rend_query,
+  control_event_hs_descriptor_failed(&rend_query.base_,
                                      HSDIR_EXIST_ID,
                                      "QUERY_REJECTED");
   expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" UNKNOWN "\
@@ -208,9 +210,30 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
-  /* test valid content. */
+  /* test no HSDir fingerprint type */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hs_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_NO_HSDIR");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_NO_HSDIR\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
+  /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
+                                      STR_HS_CONTENT_DESC_ID, NULL, NULL);
+  tor_asprintf(&exp_msg, "650+HS_DESC_CONTENT " STR_HS_ADDR " "\
+               STR_HS_CONTENT_DESC_ID " UNKNOWN" \
+               "\r\n\r\n.\r\n650 OK\r\n");
+  tt_assert(received_msg);
+  tt_str_op(received_msg, OP_EQ, exp_msg);
+  tor_free(received_msg);
+  tor_free(exp_msg);
+
+  /* test valid content. */
+  control_event_hs_descriptor_content(rend_query.onion_address,
                                       STR_HS_CONTENT_DESC_ID, HSDIR_EXIST_ID,
                                       hs_desc_content);
   tor_asprintf(&exp_msg, "650+HS_DESC_CONTENT " STR_HS_ADDR " "\
@@ -221,8 +244,8 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg, OP_EQ, exp_msg);
   tor_free(received_msg);
   tor_free(exp_msg);
-  SMARTLIST_FOREACH(rend_query.hsdirs_fp, char *, d, tor_free(d));
-  smartlist_free(rend_query.hsdirs_fp);
+  SMARTLIST_FOREACH(rend_query.base_.hsdirs_fp, char *, d, tor_free(d));
+  smartlist_free(rend_query.base_.hsdirs_fp);
 
  done:
   UNMOCK(queue_control_event_string);
@@ -322,42 +345,46 @@ test_hs_rend_data(void *arg)
   client = rend_data_client_create(STR_HS_ADDR, desc_id, client_cookie,
                                    REND_NO_AUTH);
   tt_assert(client);
-  tt_int_op(client->auth_type, ==, REND_NO_AUTH);
-  tt_str_op(client->onion_address, OP_EQ, STR_HS_ADDR);
-  tt_mem_op(client->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_mem_op(client->descriptor_cookie, OP_EQ, client_cookie,
+  rend_data_v2_t *client_v2 = TO_REND_DATA_V2(client);
+  tt_int_op(client_v2->auth_type, ==, REND_NO_AUTH);
+  tt_str_op(client_v2->onion_address, OP_EQ, STR_HS_ADDR);
+  tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
+  tt_mem_op(client_v2->descriptor_cookie, OP_EQ, client_cookie,
             sizeof(client_cookie));
   tt_assert(client->hsdirs_fp);
   tt_int_op(smartlist_len(client->hsdirs_fp), ==, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    int ret = rend_compute_v2_desc_id(desc_id, client->onion_address,
-                                      client->descriptor_cookie, now, rep);
+    int ret = rend_compute_v2_desc_id(desc_id, client_v2->onion_address,
+                                      client_v2->descriptor_cookie, now, rep);
     /* That shouldn't never fail. */
     tt_int_op(ret, ==, 0);
-    tt_mem_op(client->descriptor_id[rep], OP_EQ, desc_id, sizeof(desc_id));
+    tt_mem_op(client_v2->descriptor_id[rep], OP_EQ, desc_id,
+              sizeof(desc_id));
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client->rend_pk_digest), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_v2->rend_pk_digest), ==, 1);
   tt_int_op(tor_digest_is_zero(client->rend_cookie), ==, 1);
 
   /* Test dup(). */
   client_dup = rend_data_dup(client);
   tt_assert(client_dup);
-  tt_int_op(client_dup->auth_type, ==, client->auth_type);
-  tt_str_op(client_dup->onion_address, OP_EQ, client->onion_address);
-  tt_mem_op(client_dup->desc_id_fetch, OP_EQ, client->desc_id_fetch,
-            sizeof(client_dup->desc_id_fetch));
-  tt_mem_op(client_dup->descriptor_cookie, OP_EQ, client->descriptor_cookie,
-            sizeof(client_dup->descriptor_cookie));
+  rend_data_v2_t *client_dup_v2 = TO_REND_DATA_V2(client_dup);
+  tt_int_op(client_dup_v2->auth_type, ==, client_v2->auth_type);
+  tt_str_op(client_dup_v2->onion_address, OP_EQ, client_v2->onion_address);
+  tt_mem_op(client_dup_v2->desc_id_fetch, OP_EQ, client_v2->desc_id_fetch,
+            sizeof(client_dup_v2->desc_id_fetch));
+  tt_mem_op(client_dup_v2->descriptor_cookie, OP_EQ,
+            client_v2->descriptor_cookie,
+            sizeof(client_dup_v2->descriptor_cookie));
 
   tt_assert(client_dup->hsdirs_fp);
   tt_int_op(smartlist_len(client_dup->hsdirs_fp), ==, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_mem_op(client_dup->descriptor_id[rep], OP_EQ,
-              client->descriptor_id[rep], DIGEST_LEN);
+    tt_mem_op(client_dup_v2->descriptor_id[rep], OP_EQ,
+              client_v2->descriptor_id[rep], DIGEST_LEN);
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client_dup->rend_pk_digest), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_dup_v2->rend_pk_digest), ==, 1);
   tt_int_op(tor_digest_is_zero(client_dup->rend_cookie), ==, 1);
   rend_data_free(client);
   client = NULL;
@@ -373,18 +400,19 @@ test_hs_rend_data(void *arg)
    * zeroed out. */
   client = rend_data_client_create(NULL, desc_id, NULL, REND_BASIC_AUTH);
   tt_assert(client);
-  tt_int_op(client->auth_type, ==, REND_BASIC_AUTH);
-  tt_int_op(strlen(client->onion_address), ==, 0);
-  tt_mem_op(client->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client->descriptor_cookie,
-                            sizeof(client->descriptor_cookie)), ==, 1);
+  client_v2 = TO_REND_DATA_V2(client);
+  tt_int_op(client_v2->auth_type, ==, REND_BASIC_AUTH);
+  tt_int_op(strlen(client_v2->onion_address), ==, 0);
+  tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
+  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+                            sizeof(client_v2->descriptor_cookie)), ==, 1);
   tt_assert(client->hsdirs_fp);
   tt_int_op(smartlist_len(client->hsdirs_fp), ==, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(client->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(client_v2->descriptor_id[rep]), ==, 1);
   }
   /* The rest should be zeroed because this is a client request. */
-  tt_int_op(tor_digest_is_zero(client->rend_pk_digest), ==, 1);
+  tt_int_op(tor_digest_is_zero(client_v2->rend_pk_digest), ==, 1);
   tt_int_op(tor_digest_is_zero(client->rend_cookie), ==, 1);
   rend_data_free(client);
   client = NULL;
@@ -398,37 +426,39 @@ test_hs_rend_data(void *arg)
   service = rend_data_service_create(STR_HS_ADDR, rend_pk_digest,
                                      rend_cookie, REND_NO_AUTH);
   tt_assert(service);
-  tt_int_op(service->auth_type, ==, REND_NO_AUTH);
-  tt_str_op(service->onion_address, OP_EQ, STR_HS_ADDR);
-  tt_mem_op(service->rend_pk_digest, OP_EQ, rend_pk_digest,
+  rend_data_v2_t *service_v2 = TO_REND_DATA_V2(service);
+  tt_int_op(service_v2->auth_type, ==, REND_NO_AUTH);
+  tt_str_op(service_v2->onion_address, OP_EQ, STR_HS_ADDR);
+  tt_mem_op(service_v2->rend_pk_digest, OP_EQ, rend_pk_digest,
             sizeof(rend_pk_digest));
   tt_mem_op(service->rend_cookie, OP_EQ, rend_cookie, sizeof(rend_cookie));
   tt_assert(service->hsdirs_fp);
   tt_int_op(smartlist_len(service->hsdirs_fp), ==, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(service->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(service_v2->descriptor_id[rep]), ==, 1);
   }
   /* The rest should be zeroed because this is a service request. */
-  tt_int_op(tor_digest_is_zero(service->descriptor_cookie), ==, 1);
-  tt_int_op(tor_digest_is_zero(service->desc_id_fetch), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_v2->descriptor_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_v2->desc_id_fetch), ==, 1);
 
   /* Test dup(). */
   service_dup = rend_data_dup(service);
+  rend_data_v2_t *service_dup_v2 = TO_REND_DATA_V2(service_dup);
   tt_assert(service_dup);
-  tt_int_op(service_dup->auth_type, ==, service->auth_type);
-  tt_str_op(service_dup->onion_address, OP_EQ, service->onion_address);
-  tt_mem_op(service_dup->rend_pk_digest, OP_EQ, service->rend_pk_digest,
-            sizeof(service_dup->rend_pk_digest));
+  tt_int_op(service_dup_v2->auth_type, ==, service_v2->auth_type);
+  tt_str_op(service_dup_v2->onion_address, OP_EQ, service_v2->onion_address);
+  tt_mem_op(service_dup_v2->rend_pk_digest, OP_EQ, service_v2->rend_pk_digest,
+            sizeof(service_dup_v2->rend_pk_digest));
   tt_mem_op(service_dup->rend_cookie, OP_EQ, service->rend_cookie,
             sizeof(service_dup->rend_cookie));
   tt_assert(service_dup->hsdirs_fp);
   tt_int_op(smartlist_len(service_dup->hsdirs_fp), ==, 0);
   for (rep = 0; rep < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; rep++) {
-    tt_int_op(tor_digest_is_zero(service_dup->descriptor_id[rep]), ==, 1);
+    tt_int_op(tor_digest_is_zero(service_dup_v2->descriptor_id[rep]), ==, 1);
   }
   /* The rest should be zeroed because this is a service request. */
-  tt_int_op(tor_digest_is_zero(service_dup->descriptor_cookie), ==, 1);
-  tt_int_op(tor_digest_is_zero(service_dup->desc_id_fetch), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_dup_v2->descriptor_cookie), ==, 1);
+  tt_int_op(tor_digest_is_zero(service_dup_v2->desc_id_fetch), ==, 1);
 
  done:
   rend_data_free(service);
@@ -545,6 +575,7 @@ test_single_onion_poisoning(void *arg)
   char *dir2 = tor_strdup(get_fname_rnd("test_hs_dir2"));
   smartlist_t *services = smartlist_new();
   char *poison_path = NULL;
+  char *err_msg = NULL;
 
   /* No services, no service to verify, no problem! */
   mock_options->HiddenServiceSingleHopMode = 0;
@@ -580,7 +611,6 @@ test_single_onion_poisoning(void *arg)
   /* Add port to service 1 */
   service_1->ports = smartlist_new();
   service_2->ports = smartlist_new();
-  char *err_msg = NULL;
   rend_service_port_config_t *port1 = rend_service_parse_port_config("80", " ",
                                                                      &err_msg);
   tt_assert(port1);
@@ -779,6 +809,137 @@ test_single_onion_poisoning(void *arg)
   rend_service_free(service_2);
   UNMOCK(get_options);
   tor_free(mock_options->DataDirectory);
+  tor_free(err_msg);
+}
+
+static rend_service_t *
+helper_create_rend_service(const char *path)
+{
+  rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t));
+  s->ports = smartlist_new();
+  s->intro_nodes = smartlist_new();
+  s->expiring_nodes = smartlist_new();
+  if (path) {
+    s->directory = tor_strdup(path);
+  }
+  return s;
+}
+
+static void
+test_prune_services_on_reload(void *arg)
+{
+  smartlist_t *new = smartlist_new(), *old = smartlist_new();
+  /* Non ephemeral service. */
+  rend_service_t *s1 = helper_create_rend_service("SomePath");
+  /* Create a non ephemeral service with the _same_ path as so we can test the
+   * transfer of introduction point between the same services on reload. */
+  rend_service_t *s2 = helper_create_rend_service(s1->directory);
+  /* Ephemeral service (directory is NULL). */
+  rend_service_t *e1 = helper_create_rend_service(NULL);
+  rend_service_t *e2 = helper_create_rend_service(NULL);
+
+  (void) arg;
+
+  {
+    /* Add both services to the old list. */
+    smartlist_add(old, s1);
+    smartlist_add(old, e1);
+    /* Only put the non ephemeral in the new list. */
+    smartlist_add(new, s1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    /* We expect that the ephemeral one is in the new list but removed from
+     * the old one. */
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    tt_assert(smartlist_get(old, 0) == s1);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+    tt_assert(smartlist_get(new, 0) == s1);
+    tt_assert(smartlist_get(new, 1) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* This test will make sure that only the ephemeral service is kept if the
+     * new list is empty. The old list should contain only the non ephemeral
+     * one. */
+    smartlist_add(old, s1);
+    smartlist_add(old, e1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    tt_assert(smartlist_get(old, 0) == s1);
+    tt_int_op(smartlist_len(new), OP_EQ, 1);
+    tt_assert(smartlist_get(new, 0) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* This test makes sure that the new list stays the same even from the old
+     * list being completely different. */
+    smartlist_add(new, s1);
+    smartlist_add(new, e1);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+    tt_assert(smartlist_get(new, 0) == s1);
+    tt_assert(smartlist_get(new, 1) == e1);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+  }
+
+  {
+    rend_intro_point_t ip1;
+    /* This IP should be found in the s2 service after pruning. */
+    smartlist_add(s1->intro_nodes, &ip1);
+    /* Setup our list. */
+    smartlist_add(old, s1);
+    smartlist_add(new, s2);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    tt_int_op(smartlist_len(old), OP_EQ, 1);
+    /* Intro nodes have been moved to the s2 in theory so it must be empty. */
+    tt_int_op(smartlist_len(s1->intro_nodes), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 1);
+    rend_service_t *elem = smartlist_get(new, 0);
+    tt_assert(elem);
+    tt_assert(elem == s2);
+    tt_int_op(smartlist_len(elem->intro_nodes), OP_EQ, 1);
+    tt_assert(smartlist_get(elem->intro_nodes, 0) == &ip1);
+    smartlist_clear(s1->intro_nodes);
+    smartlist_clear(s2->intro_nodes);
+    /* Cleanup for next test. */
+    smartlist_clear(new);
+    smartlist_clear(old);
+  }
+
+  {
+    /* Test two ephemeral services. */
+    smartlist_add(old, e1);
+    smartlist_add(old, e2);
+    set_rend_service_list(old);
+    set_rend_rend_service_staging_list(new);
+    rend_service_prune_list_impl_();
+    /* Check if they've all been transfered. */
+    tt_int_op(smartlist_len(old), OP_EQ, 0);
+    tt_int_op(smartlist_len(new), OP_EQ, 2);
+  }
+
+ done:
+  rend_service_free(s1);
+  rend_service_free(s2);
+  rend_service_free(e1);
+  rend_service_free(e2);
+  smartlist_free(new);
+  smartlist_free(old);
 }
 
 struct testcase_t hs_tests[] = {
@@ -801,6 +962,9 @@ struct testcase_t hs_tests[] = {
     TT_FORK, &passthrough_setup, (void*)(CREATE_HS_DIR2) },
   { "single_onion_poisoning_create_dir_both", test_single_onion_poisoning,
     TT_FORK, &passthrough_setup, (void*)(CREATE_HS_DIR1 | CREATE_HS_DIR2) },
+  { "prune_services_on_reload", test_prune_services_on_reload, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
new file mode 100644
index 0000000000..40f50b322a
--- /dev/null
+++ b/src/test/test_hs_cache.c
@@ -0,0 +1,443 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cache.c
+ * \brief Test hidden service caches.
+ */
+
+#define CONNECTION_PRIVATE
+#define HS_CACHE_PRIVATE
+
+#include "ed25519_cert.h"
+#include "hs_cache.h"
+#include "rendcache.h"
+#include "directory.h"
+#include "connection.h"
+
+#include "hs_test_helpers.h"
+#include "test_helpers.h"
+#include "test.h"
+
+/* Static variable used to encoded the HSDir query. */
+static char query_b64[256];
+
+/* Build an HSDir query using a ed25519 public key. */
+static const char *
+helper_get_hsdir_query(const hs_descriptor_t *desc)
+{
+  ed25519_public_to_base64(query_b64, &desc->plaintext_data.blinded_pubkey);
+  return query_b64;
+}
+
+static void
+init_test(void)
+{
+  /* Always needed. Initialize the subsystem. */
+  hs_cache_init();
+  /* We need the v2 cache since our OOM and cache cleanup does poke at it. */
+  rend_cache_init();
+}
+
+static void
+test_directory(void *arg)
+{
+  int ret;
+  size_t oom_size;
+  char *desc1_str = NULL;
+  const char *desc_out;
+  ed25519_keypair_t signing_kp1;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  init_test();
+  /* Generate a valid descriptor with normal values. */
+  ret = ed25519_keypair_generate(&signing_kp1, 0);
+  tt_int_op(ret, ==, 0);
+  desc1 = hs_helper_build_hs_desc_with_ip(&signing_kp1);
+  tt_assert(desc1);
+  ret = hs_desc_encode_descriptor(desc1, &signing_kp1, &desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Very first basic test, should be able to be stored, survive a
+   * clean, found with a lookup and then cleaned by our OOM. */
+  {
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Re-add, it should fail since we already have it. */
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, -1);
+    /* Try to clean now which should be fine, there is at worst few seconds
+     * between the store and this call. */
+    hs_cache_clean_as_dir(time(NULL));
+    /* We should find it in our cache. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, desc1_str);
+    /* Tell our OOM to run and to at least remove a byte which will result in
+     * removing the descriptor from our cache. */
+    oom_size = hs_cache_handle_oom(time(NULL), 1);
+    tt_int_op(oom_size, >=, 1);
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Store two descriptors and remove the expiring one only. */
+  {
+    ed25519_keypair_t signing_kp_zero;
+    ret = ed25519_keypair_generate(&signing_kp_zero, 0);
+    tt_int_op(ret, ==, 0);
+    hs_descriptor_t *desc_zero_lifetime;
+    desc_zero_lifetime = hs_helper_build_hs_desc_with_ip(&signing_kp_zero);
+    tt_assert(desc_zero_lifetime);
+    desc_zero_lifetime->plaintext_data.revision_counter = 1;
+    desc_zero_lifetime->plaintext_data.lifetime_sec = 0;
+    char *desc_zero_lifetime_str;
+    ret = hs_desc_encode_descriptor(desc_zero_lifetime, &signing_kp_zero,
+                                    &desc_zero_lifetime_str);
+    tt_int_op(ret, OP_EQ, 0);
+
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_store_as_dir(desc_zero_lifetime_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* This one should clear out our zero lifetime desc. */
+    hs_cache_clean_as_dir(time(NULL));
+    /* We should find desc1 in our cache. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, desc1_str);
+    /* We should NOT find our zero lifetime desc in our cache. */
+    ret = hs_cache_lookup_as_dir(3,
+                                 helper_get_hsdir_query(desc_zero_lifetime),
+                                 NULL);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Cleanup our entire cache. */
+    oom_size = hs_cache_handle_oom(time(NULL), 1);
+    tt_int_op(oom_size, >=, 1);
+    hs_descriptor_free(desc_zero_lifetime);
+    tor_free(desc_zero_lifetime_str);
+  }
+
+  /* Throw junk at it. */
+  {
+    ret = hs_cache_store_as_dir("blah");
+    tt_int_op(ret, OP_EQ, -1);
+    /* Poor attempt at tricking the decoding. */
+    ret = hs_cache_store_as_dir("hs-descriptor 3\nJUNK");
+    tt_int_op(ret, OP_EQ, -1);
+    /* Undecodable base64 query. */
+    ret = hs_cache_lookup_as_dir(3, "blah", NULL);
+    tt_int_op(ret, OP_EQ, -1);
+    /* Decodable base64 query but wrong ed25519 size. */
+    ret = hs_cache_lookup_as_dir(3, "dW5pY29ybg==", NULL);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Test descriptor replacement with revision counter. */
+  {
+    char *new_desc_str;
+
+    /* Add a descriptor. */
+    ret = hs_cache_store_as_dir(desc1_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    /* Bump revision counter. */
+    desc1->plaintext_data.revision_counter++;
+    ret = hs_desc_encode_descriptor(desc1, &signing_kp1, &new_desc_str);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_cache_store_as_dir(new_desc_str);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Look it up, it should have been replaced. */
+    ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), &desc_out);
+    tt_int_op(ret, OP_EQ, 1);
+    tt_str_op(desc_out, OP_EQ, new_desc_str);
+    tor_free(new_desc_str);
+  }
+
+ done:
+  hs_descriptor_free(desc1);
+  tor_free(desc1_str);
+}
+
+static void
+test_clean_as_dir(void *arg)
+{
+  size_t ret;
+  char *desc1_str = NULL;
+  time_t now = time(NULL);
+  hs_descriptor_t *desc1 = NULL;
+  ed25519_keypair_t signing_kp1;
+
+  (void) arg;
+
+  init_test();
+
+  /* Generate a valid descriptor with values. */
+  ret = ed25519_keypair_generate(&signing_kp1, 0);
+  tt_int_op(ret, ==, 0);
+  desc1 = hs_helper_build_hs_desc_with_ip(&signing_kp1);
+  tt_assert(desc1);
+  ret = hs_desc_encode_descriptor(desc1, &signing_kp1, &desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = hs_cache_store_as_dir(desc1_str);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* With the lifetime being 3 hours, a cleanup shouldn't remove it. */
+  ret = cache_clean_v3_as_dir(now, 0);
+  tt_int_op(ret, ==, 0);
+  /* Should be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 1);
+  /* Set a cutoff 100 seconds in the past. It should not remove the entry
+   * since the entry is still recent enough. */
+  ret = cache_clean_v3_as_dir(now, now - 100);
+  tt_int_op(ret, ==, 0);
+  /* Should be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 1);
+  /* Set a cutoff of 100 seconds in the future. It should remove the entry
+   * that we've just added since it's not too old for the cutoff. */
+  ret = cache_clean_v3_as_dir(now, now + 100);
+  tt_int_op(ret, >, 0);
+  /* Shouldn't be present after clean up. */
+  ret = hs_cache_lookup_as_dir(3, helper_get_hsdir_query(desc1), NULL);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  hs_descriptor_free(desc1);
+  tor_free(desc1_str);
+}
+
+/* Test helper: Fetch an HS descriptor from an HSDir (for the hidden service
+   with <b>blinded_key</b>. Return the received descriptor string. */
+static char *
+helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
+{
+  int retval;
+
+  char *received_desc = NULL;
+  char *hsdir_query_str = NULL;
+
+  /* The dir conn we are going to simulate */
+  dir_connection_t *conn = NULL;
+
+  /* First extract the blinded public key that we are going to use in our
+     query, and then build the actual query string. */
+  {
+    char hsdir_cache_key[ED25519_BASE64_LEN+1];
+
+    retval = ed25519_public_to_base64(hsdir_cache_key,
+                                      blinded_key);
+    tt_int_op(retval, ==, 0);
+    tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
+  }
+
+  /* Simulate an HTTP GET request to the HSDir */
+  conn = dir_connection_new(AF_INET);
+  tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
+  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+  retval = directory_handle_command_get(conn, hsdir_query_str,
+                                        NULL, 0);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Read the descriptor that the HSDir just served us */
+  {
+    char *headers = NULL;
+    size_t body_used = 0;
+
+    fetch_from_buf_http(TO_CONN(conn)->outbuf, &headers, MAX_HEADERS_SIZE,
+                        &received_desc, &body_used, HS_DESC_MAX_LEN, 0);
+    tor_free(headers);
+  }
+
+ done:
+  tor_free(hsdir_query_str);
+  if (conn)
+    connection_free_(TO_CONN(conn));
+
+  return received_desc;
+}
+
+/* Publish a descriptor to the HSDir, then fetch it. Check that the received
+   descriptor matches the published one. */
+static void
+test_upload_and_download_hs_desc(void *arg)
+{
+  int retval;
+  hs_descriptor_t *published_desc = NULL;
+
+  char *published_desc_str = NULL;
+  char *received_desc_str = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  /* Test a descriptor not found in the directory cache. */
+  {
+    ed25519_public_key_t blinded_key;
+    memset(&blinded_key.pubkey, 'A', sizeof(blinded_key.pubkey));
+    received_desc_str = helper_fetch_desc_from_hsdir(&blinded_key);
+    tt_int_op(strlen(received_desc_str), OP_EQ, 0);
+    tor_free(received_desc_str);
+  }
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    ed25519_keypair_t signing_kp;
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, ==, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* Publish descriptor to the HSDir */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, ==, 200);
+  }
+
+  /* Simulate a fetch of the previously published descriptor */
+  {
+    const ed25519_public_key_t *blinded_key;
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+  }
+
+  /* Verify we received the exact same descriptor we published earlier */
+  tt_str_op(received_desc_str, OP_EQ, published_desc_str);
+  tor_free(received_desc_str);
+
+  /* With a valid descriptor in the directory cache, try again an invalid. */
+  {
+    ed25519_public_key_t blinded_key;
+    memset(&blinded_key.pubkey, 'A', sizeof(blinded_key.pubkey));
+    received_desc_str = helper_fetch_desc_from_hsdir(&blinded_key);
+    tt_int_op(strlen(received_desc_str), OP_EQ, 0);
+  }
+
+ done:
+  tor_free(received_desc_str);
+  tor_free(published_desc_str);
+  hs_descriptor_free(published_desc);
+}
+
+/* Test that HSDirs reject outdated descriptors based on their revision
+ * counter. Also test that HSDirs correctly replace old descriptors with newer
+ * descriptors. */
+static void
+test_hsdir_revision_counter_check(void *arg)
+{
+  int retval;
+
+  ed25519_keypair_t signing_kp;
+
+  hs_descriptor_t *published_desc = NULL;
+  char *published_desc_str = NULL;
+
+  char *received_desc_str = NULL;
+  hs_descriptor_t *received_desc = NULL;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  init_test();
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    retval = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(retval, ==, 0);
+    published_desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    tt_assert(published_desc);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  /* Publish descriptor to the HSDir */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, ==, 200);
+  }
+
+  /* Try publishing again with the same revision counter: Should fail. */
+  {
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, ==, 400);
+  }
+
+  /* Fetch the published descriptor and validate the revision counter. */
+  {
+    const ed25519_public_key_t *blinded_key;
+
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+
+    retval = hs_desc_decode_descriptor(received_desc_str,NULL, &received_desc);
+    tt_int_op(retval, ==, 0);
+    tt_assert(received_desc);
+
+    /* Check that the revision counter is correct */
+    tt_u64_op(received_desc->plaintext_data.revision_counter, ==, 42);
+
+    hs_descriptor_free(received_desc);
+    received_desc = NULL;
+    tor_free(received_desc_str);
+  }
+
+  /* Increment the revision counter and try again. Should work. */
+  {
+    published_desc->plaintext_data.revision_counter = 1313;
+    tor_free(published_desc_str);
+    retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
+                                       &published_desc_str);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = handle_post_hs_descriptor("/tor/hs/3/publish",published_desc_str);
+    tt_int_op(retval, ==, 200);
+  }
+
+  /* Again, fetch the published descriptor and perform the revision counter
+     validation. The revision counter must have changed. */
+  {
+    const ed25519_public_key_t *blinded_key;
+
+    blinded_key = &published_desc->plaintext_data.blinded_pubkey;
+    received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
+
+    retval = hs_desc_decode_descriptor(received_desc_str,NULL, &received_desc);
+    tt_int_op(retval, ==, 0);
+    tt_assert(received_desc);
+
+    /* Check that the revision counter is the latest */
+    tt_u64_op(received_desc->plaintext_data.revision_counter, ==, 1313);
+  }
+
+ done:
+  hs_descriptor_free(published_desc);
+  hs_descriptor_free(received_desc);
+  tor_free(received_desc_str);
+  tor_free(published_desc_str);
+}
+
+struct testcase_t hs_cache[] = {
+  /* Encoding tests. */
+  { "directory", test_directory, TT_FORK,
+    NULL, NULL },
+  { "clean_as_dir", test_clean_as_dir, TT_FORK,
+    NULL, NULL },
+  { "hsdir_revision_counter_check", test_hsdir_revision_counter_check, TT_FORK,
+    NULL, NULL },
+  { "upload_and_download_hs_desc", test_upload_and_download_hs_desc, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
new file mode 100644
index 0000000000..a40616c61e
--- /dev/null
+++ b/src/test/test_hs_descriptor.c
@@ -0,0 +1,889 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_descriptor.c
+ * \brief Test hidden service descriptor encoding and decoding.
+ */
+
+#define HS_DESCRIPTOR_PRIVATE
+
+#include "crypto_ed25519.h"
+#include "ed25519_cert.h"
+#include "or.h"
+#include "hs_descriptor.h"
+#include "test.h"
+#include "torcert.h"
+
+#include "hs_test_helpers.h"
+#include "test_helpers.h"
+#include "log_test_helpers.h"
+
+#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
+DISABLE_GCC_WARNING(overlength-strings)
+/* We allow huge string constants in the unit tests, but not in the code
+ * at large. */
+#endif
+#include "test_hs_descriptor.inc"
+ENABLE_GCC_WARNING(overlength-strings)
+
+/* Test certificate encoding put in a descriptor. */
+static void
+test_cert_encoding(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  time_t now = time(NULL);
+  ed25519_keypair_t kp;
+  ed25519_public_key_t signed_key;
+  ed25519_secret_key_t secret_key;
+  tor_cert_t *cert = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&kp, 0);
+  tt_int_op(ret, == , 0);
+  ret = ed25519_secret_key_generate(&secret_key, 0);
+  tt_int_op(ret, == , 0);
+  ret = ed25519_public_key_generate(&signed_key, &secret_key);
+  tt_int_op(ret, == , 0);
+
+  cert = tor_cert_create(&kp, CERT_TYPE_SIGNING_AUTH, &signed_key,
+                         now, 3600 * 2, CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(cert);
+
+  /* Test the certificate encoding function. */
+  ret = tor_cert_encode_ed22519(cert, &encoded);
+  tt_int_op(ret, ==, 0);
+
+  /* Validated the certificate string. */
+  {
+    char *end, *pos = encoded;
+    char *b64_cert, buf[256];
+    size_t b64_cert_len;
+    tor_cert_t *parsed_cert;
+
+    tt_int_op(strcmpstart(pos, "-----BEGIN ED25519 CERT-----\n"), ==, 0);
+    pos += strlen("-----BEGIN ED25519 CERT-----\n");
+
+    /* Isolate the base64 encoded certificate and try to decode it. */
+    end = strstr(pos, "-----END ED25519 CERT-----");
+    tt_assert(end);
+    b64_cert = pos;
+    b64_cert_len = end - pos;
+    ret = base64_decode(buf, sizeof(buf), b64_cert, b64_cert_len);
+    tt_int_op(ret, >, 0);
+    /* Parseable? */
+    parsed_cert = tor_cert_parse((uint8_t *) buf, ret);
+    tt_assert(parsed_cert);
+    /* Signature is valid? */
+    ret = tor_cert_checksig(parsed_cert, &kp.pubkey, now + 10);
+    tt_int_op(ret, ==, 0);
+    ret = tor_cert_eq(cert, parsed_cert);
+    tt_int_op(ret, ==, 1);
+    /* The cert did have the signing key? */
+    ret= ed25519_pubkey_eq(&parsed_cert->signing_key, &kp.pubkey);
+    tt_int_op(ret, ==, 1);
+    tor_cert_free(parsed_cert);
+
+    /* Get to the end part of the certificate. */
+    pos += b64_cert_len;
+    tt_int_op(strcmpstart(pos, "-----END ED25519 CERT-----"), ==, 0);
+    pos += strlen("-----END ED25519 CERT-----");
+  }
+
+ done:
+  tor_cert_free(cert);
+  tor_free(encoded);
+}
+
+/* Test the descriptor padding. */
+static void
+test_descriptor_padding(void *arg)
+{
+  char *plaintext;
+  size_t plaintext_len, padded_len;
+  uint8_t *padded_plaintext = NULL;
+
+/* Example: if l = 129, the ceiled division gives 2 and then multiplied by 128
+ * to give 256. With l = 127, ceiled division gives 1 then times 128. */
+#define PADDING_EXPECTED_LEN(l) \
+  CEIL_DIV(l, HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE) * \
+  HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE
+
+  (void) arg;
+
+  { /* test #1: no padding */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE;
+    plaintext = tor_malloc(plaintext_len);
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+  { /* test #2: one byte padding? */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE - 1;
+    plaintext = tor_malloc(plaintext_len);
+    padded_plaintext = NULL;
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+  { /* test #3: Lots more bytes of padding? */
+    plaintext_len = HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE + 1;
+    plaintext = tor_malloc(plaintext_len);
+    padded_plaintext = NULL;
+    padded_len = build_plaintext_padding(plaintext, plaintext_len,
+                                         &padded_plaintext);
+    tt_assert(padded_plaintext);
+    tor_free(plaintext);
+    /* Make sure our padding has been zeroed. */
+    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+                              padded_len - plaintext_len), OP_EQ, 1);
+    tor_free(padded_plaintext);
+    /* Never never have a padded length smaller than the plaintext. */
+    tt_int_op(padded_len, OP_GE, plaintext_len);
+    tt_int_op(padded_len, OP_EQ, PADDING_EXPECTED_LEN(plaintext_len));
+  }
+
+ done:
+  return;
+}
+
+static void
+test_link_specifier(void *arg)
+{
+  ssize_t ret;
+  hs_desc_link_specifier_t spec;
+  smartlist_t *link_specifiers = smartlist_new();
+  char buf[256];
+  char *b64 = NULL;
+  link_specifier_t *ls = NULL;
+
+  (void) arg;
+
+  /* Always this port. */
+  spec.u.ap.port = 42;
+  smartlist_add(link_specifiers, &spec);
+
+  /* Test IPv4 for starter. */
+  {
+    uint32_t ipv4;
+
+    spec.type = LS_IPV4;
+    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
+    tt_int_op(ret, ==, AF_INET);
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, >, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), ==, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    tt_int_op(ret, ==, 8);
+    /* Should be 2 bytes for port and 4 bytes for IPv4. */
+    tt_int_op(link_specifier_get_ls_len(ls), ==, 6);
+    ipv4 = link_specifier_get_un_ipv4_addr(ls);
+    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), ==, ipv4);
+    tt_int_op(link_specifier_get_un_ipv4_port(ls), ==, spec.u.ap.port);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+  /* Test IPv6. */
+  {
+    uint8_t ipv6[16];
+
+    spec.type = LS_IPV6;
+    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
+    tt_int_op(ret, ==, AF_INET6);
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, >, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), ==, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    tt_int_op(ret, ==, 20);
+    /* Should be 2 bytes for port and 16 bytes for IPv6. */
+    tt_int_op(link_specifier_get_ls_len(ls), ==, 18);
+    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
+      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
+    }
+    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), ==, ipv6, sizeof(ipv6));
+    tt_int_op(link_specifier_get_un_ipv6_port(ls), ==, spec.u.ap.port);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+  /* Test legacy. */
+  {
+    uint8_t *id;
+
+    spec.type = LS_LEGACY_ID;
+    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
+    b64 = encode_link_specifiers(link_specifiers);
+    tt_assert(b64);
+
+    /* Decode it and validate the format. */
+    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
+    tt_int_op(ret, >, 0);
+    /* First byte is the number of link specifier. */
+    tt_int_op(get_uint8(buf), ==, 1);
+    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
+    /* 20 bytes digest + 1 byte type + 1 byte len. */
+    tt_int_op(ret, ==, 22);
+    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
+    /* Digest length is 20 bytes. */
+    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
+    id = link_specifier_getarray_un_legacy_id(ls);
+    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
+
+    link_specifier_free(ls);
+    ls = NULL;
+    tor_free(b64);
+  }
+
+ done:
+  link_specifier_free(ls);
+  tor_free(b64);
+  smartlist_free(link_specifiers);
+}
+
+static void
+test_encode_descriptor(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, ==, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+  ret = hs_desc_encode_descriptor(desc, &signing_kp, &encoded);
+  tt_int_op(ret, ==, 0);
+  tt_assert(encoded);
+
+ done:
+  hs_descriptor_free(desc);
+  tor_free(encoded);
+}
+
+static void
+test_decode_descriptor(void *arg)
+{
+  int ret;
+  char *encoded = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+  hs_descriptor_t *decoded = NULL;
+  hs_descriptor_t *desc_no_ip = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, ==, 0);
+  desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+
+  /* Give some bad stuff to the decoding function. */
+  ret = hs_desc_decode_descriptor("hladfjlkjadf", NULL, &decoded);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = hs_desc_encode_descriptor(desc, &signing_kp, &encoded);
+  tt_int_op(ret, ==, 0);
+  tt_assert(encoded);
+
+  ret = hs_desc_decode_descriptor(encoded, NULL, &decoded);
+  tt_int_op(ret, ==, 0);
+  tt_assert(decoded);
+
+  hs_helper_desc_equal(desc, decoded);
+
+  /* Decode a descriptor with _no_ introduction points. */
+  {
+    ed25519_keypair_t signing_kp_no_ip;
+    ret = ed25519_keypair_generate(&signing_kp_no_ip, 0);
+    tt_int_op(ret, ==, 0);
+    desc_no_ip = hs_helper_build_hs_desc_no_ip(&signing_kp_no_ip);
+    tt_assert(desc_no_ip);
+    tor_free(encoded);
+    ret = hs_desc_encode_descriptor(desc_no_ip, &signing_kp_no_ip, &encoded);
+    tt_int_op(ret, ==, 0);
+    tt_assert(encoded);
+    hs_descriptor_free(decoded);
+    ret = hs_desc_decode_descriptor(encoded, NULL, &decoded);
+    tt_int_op(ret, ==, 0);
+    tt_assert(decoded);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+  hs_descriptor_free(desc_no_ip);
+  hs_descriptor_free(decoded);
+  tor_free(encoded);
+}
+
+static void
+test_supported_version(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  /* Unsupported. */
+  ret = hs_desc_is_supported_version(42);
+  tt_int_op(ret, OP_EQ, 0);
+  /* To early. */
+  ret = hs_desc_is_supported_version(HS_DESC_SUPPORTED_FORMAT_VERSION_MIN - 1);
+  tt_int_op(ret, OP_EQ, 0);
+  /* One too new. */
+  ret = hs_desc_is_supported_version(HS_DESC_SUPPORTED_FORMAT_VERSION_MAX + 1);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Valid version. */
+  ret = hs_desc_is_supported_version(3);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_encrypted_data_len(void *arg)
+{
+  int ret;
+  size_t value;
+
+  (void) arg;
+
+  /* No length, error. */
+  ret = encrypted_data_length_is_valid(0);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Valid value. */
+  value = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN + 1;
+  ret = encrypted_data_length_is_valid(value);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+static void
+test_decode_invalid_intro_point(void *arg)
+{
+  int ret;
+  char *encoded_ip = NULL;
+  size_t len_out;
+  hs_desc_intro_point_t *ip = NULL;
+  ed25519_keypair_t signing_kp;
+  hs_descriptor_t *desc = NULL;
+
+  (void) arg;
+
+  /* Seperate pieces of a valid encoded introduction point. */
+  const char *intro_point =
+    "introduction-point AQIUMDI5OUYyNjhGQ0E5RDU1Q0QxNTc=";
+  const char *auth_key =
+    "auth-key\n"
+    "-----BEGIN ED25519 CERT-----\n"
+    "AQkACOhAAQW8ltYZMIWpyrfyE/b4Iyi8CNybCwYs6ADk7XfBaxsFAQAgBAD3/BE4\n"
+    "XojGE/N2bW/wgnS9r2qlrkydGyuCKIGayYx3haZ39LD4ZTmSMRxwmplMAqzG/XNP\n"
+    "0Kkpg4p2/VnLFJRdU1SMFo1lgQ4P0bqw7Tgx200fulZ4KUM5z5V7m+a/mgY=\n"
+    "-----END ED25519 CERT-----";
+  const char *enc_key =
+    "enc-key ntor bpZKLsuhxP6woDQ3yVyjm5gUKSk7RjfAijT2qrzbQk0=";
+  const char *enc_key_cert =
+    "enc-key-cert\n"
+    "-----BEGIN ED25519 CERT-----\n"
+    "AQsACOhZAUpNvCZ1aJaaR49lS6MCdsVkhVGVrRqoj0Y2T4SzroAtAQAgBABFOcGg\n"
+    "lbTt1DF5nKTE/gU3Fr8ZtlCIOhu1A+F5LM7fqCUupfesg0KTHwyIZOYQbJuM5/he\n"
+    "/jDNyLy9woPJdjkxywaY2RPUxGjLYtMQV0E8PUxWyICV+7y52fTCYaKpYQw=\n"
+    "-----END ED25519 CERT-----";
+
+  /* Try to decode a junk string. */
+  {
+    hs_descriptor_free(desc);
+    desc = NULL;
+    ret = ed25519_keypair_generate(&signing_kp, 0);
+    tt_int_op(ret, ==, 0);
+    desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
+    const char *junk = "this is not a descriptor";
+    ip = decode_introduction_point(desc, junk);
+    tt_assert(!ip);
+    desc_intro_point_free(ip);
+    ip = NULL;
+  }
+
+  /* Invalid link specifiers. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "introduction-point blah";
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) enc_key);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_assert(!ip);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+    desc_intro_point_free(ip);
+    ip = NULL;
+  }
+
+  /* Invalid auth key type. */
+  {
+    smartlist_t *lines = smartlist_new();
+    /* Try to put a valid object that our tokenize function will be able to
+     * parse but that has nothing to do with the auth_key. */
+    const char *bad_line =
+      "auth-key\n"
+      "-----BEGIN UNICORN CERT-----\n"
+      "MIGJAoGBAO4bATcW8kW4h6RQQAKEgg+aXCpF4JwbcO6vGZtzXTDB+HdPVQzwqkbh\n"
+      "XzFM6VGArhYw4m31wcP1Z7IwULir7UMnAFd7Zi62aYfU6l+Y1yAoZ1wzu1XBaAMK\n"
+      "ejpwQinW9nzJn7c2f69fVke3pkhxpNdUZ+vplSA/l9iY+y+v+415AgMBAAE=\n"
+      "-----END UNICORN CERT-----";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_assert(!ip);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line =
+      "enc-key unicorn bpZKLsuhxP6woDQ3yVyjm5gUKSk7RjfAijT2qrzbQk0=";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_assert(!ip);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key object. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "enc-key ntor";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_assert(!ip);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key base64 curv25519 key. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "enc-key ntor blah===";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_assert(!ip);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+  /* Invalid enc-key invalid legacy. */
+  {
+    smartlist_t *lines = smartlist_new();
+    const char *bad_line = "legacy-key blah===";
+    /* Build intro point text. */
+    smartlist_add(lines, (char *) intro_point);
+    smartlist_add(lines, (char *) auth_key);
+    smartlist_add(lines, (char *) bad_line);
+    smartlist_add(lines, (char *) enc_key_cert);
+    encoded_ip = smartlist_join_strings(lines, "\n", 0, &len_out);
+    tt_assert(encoded_ip);
+    ip = decode_introduction_point(desc, encoded_ip);
+    tt_assert(!ip);
+    tor_free(encoded_ip);
+    smartlist_free(lines);
+  }
+
+ done:
+  hs_descriptor_free(desc);
+  desc_intro_point_free(ip);
+}
+
+/** Make sure we fail gracefully when decoding the bad desc from #23233. */
+static void
+test_decode_bad_signature(void *arg)
+{
+  hs_desc_plaintext_data_t desc_plaintext;
+  int ret;
+
+  (void) arg;
+
+  /* Update approx time to dodge cert expiration */
+  update_approx_time(1502661599);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  ret = hs_desc_decode_plaintext(HS_DESC_BAD_SIG, &desc_plaintext);
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("Malformed signature line. Rejecting.");
+  teardown_capture_of_logs();
+
+ done:
+  desc_plaintext_data_free_contents(&desc_plaintext);
+}
+
+static void
+test_decode_plaintext(void *arg)
+{
+  int ret;
+  hs_desc_plaintext_data_t desc_plaintext;
+  const char *bad_value = "unicorn";
+
+  (void) arg;
+
+#define template \
+    "hs-descriptor %s\n" \
+    "descriptor-lifetime %s\n" \
+    "descriptor-signing-key-cert\n" \
+    "-----BEGIN ED25519 CERT-----\n" \
+    "AQgABjvPAQaG3g+dc6oV/oJV4ODAtkvx56uBnPtBT9mYVuHVOhn7AQAgBABUg3mQ\n" \
+    "myBr4bu5LCr53wUEbW2EXui01CbUgU7pfo9LvJG3AcXRojj6HlfsUs9BkzYzYdjF\n" \
+    "A69Apikgu0ewHYkFFASt7Il+gB3w6J8YstQJZT7dtbtl+doM7ug8B68Qdg8=\n" \
+    "-----END ED25519 CERT-----\n" \
+    "revision-counter %s\n" \
+    "encrypted\n" \
+    "-----BEGIN %s-----\n" \
+    "UNICORN\n" \
+    "-----END MESSAGE-----\n" \
+    "signature m20WJH5agqvwhq7QeuEZ1mYyPWQDO+eJOZUjLhAiKu8DbL17DsDfJE6kXbWy" \
+    "HimbNj2we0enV3cCOOAsmPOaAw\n"
+
+  /* Invalid version. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, bad_value, "180", "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Missing fields. */
+  {
+    const char *plaintext = "hs-descriptor 3\n";
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Max length. */
+  {
+    size_t big = 64000;
+    /* Must always be bigger than HS_DESC_MAX_LEN. */
+    tt_int_op(HS_DESC_MAX_LEN, <, big);
+    char *plaintext = tor_malloc_zero(big);
+    memset(plaintext, 'a', big);
+    plaintext[big - 1] = '\0';
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Bad lifetime value. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", bad_value, "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Huge lifetime value. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "7181615", "42", "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Invalid encrypted section. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "180", "42", bad_value);
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Invalid revision counter. */
+  {
+    char *plaintext;
+    tor_asprintf(&plaintext, template, "3", "180", bad_value, "MESSAGE");
+    ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
+    tor_free(plaintext);
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_validate_cert(void *arg)
+{
+  int ret;
+  time_t now = time(NULL);
+  ed25519_keypair_t kp;
+  tor_cert_t *cert = NULL;
+
+  (void) arg;
+
+  ret = ed25519_keypair_generate(&kp, 0);
+  tt_int_op(ret, ==, 0);
+
+  /* Cert of type CERT_TYPE_AUTH_HS_IP_KEY. */
+  cert = tor_cert_create(&kp, CERT_TYPE_AUTH_HS_IP_KEY,
+                                     &kp.pubkey, now, 3600,
+                                     CERT_FLAG_INCLUDE_SIGNING_KEY);
+  tt_assert(cert);
+  /* Test with empty certificate. */
+  ret = cert_is_valid(NULL, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  /* Test with a bad type. */
+  ret = cert_is_valid(cert, CERT_TYPE_SIGNING_HS_DESC, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  /* Normal validation. */
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 1);
+  /* Break signing key so signature verification will fails. */
+  memset(&cert->signing_key, 0, sizeof(cert->signing_key));
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+  tor_cert_free(cert);
+
+  /* Try a cert without including the signing key. */
+  cert = tor_cert_create(&kp, CERT_TYPE_AUTH_HS_IP_KEY, &kp.pubkey, now,
+                         3600, 0);
+  tt_assert(cert);
+  /* Test with a bad type. */
+  ret = cert_is_valid(cert, CERT_TYPE_AUTH_HS_IP_KEY, "unicorn");
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  tor_cert_free(cert);
+}
+
+static void
+test_desc_signature(void *arg)
+{
+  int ret;
+  char *data = NULL, *desc = NULL;
+  char sig_b64[ED25519_SIG_BASE64_LEN + 1];
+  ed25519_keypair_t kp;
+  ed25519_signature_t sig;
+
+  (void) arg;
+
+  ed25519_keypair_generate(&kp, 0);
+  /* Setup a phoony descriptor but with a valid signature token that is the
+   * signature is verifiable. */
+  tor_asprintf(&data, "This is a signed descriptor\n");
+  ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
+                              "Tor onion service descriptor sig v3", &kp);
+  tt_int_op(ret, ==, 0);
+  ret = ed25519_signature_to_base64(sig_b64, &sig);
+  tt_int_op(ret, ==, 0);
+  /* Build the descriptor that should be valid. */
+  tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
+  ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
+  tt_int_op(ret, ==, 1);
+  /* Junk signature. */
+  ret = desc_sig_is_valid("JUNK", &kp.pubkey, desc, strlen(desc));
+  tt_int_op(ret, ==, 0);
+
+ done:
+  tor_free(desc);
+  tor_free(data);
+}
+
+/* bad desc auth type */
+static const char bad_superencrypted_text1[] = "desc-auth-type scoobysnack\n"
+  "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC"
+  "BiYWQgYXQgYWxs\n"
+  "-----END MESSAGE-----\n";
+
+/* bad ephemeral key */
+static const char bad_superencrypted_text2[] = "desc-auth-type x25519\n"
+  "desc-auth-ephemeral-key differentalphabet\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC"
+  "BiYWQgYXQgYWxs\n"
+  "-----END MESSAGE-----\n";
+
+/* bad encrypted msg */
+static const char bad_superencrypted_text3[] = "desc-auth-type x25519\n"
+  "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "SO SMALL NOT GOOD\n"
+  "-----END MESSAGE-----\n";
+
+static const char correct_superencrypted_text[] = "desc-auth-type x25519\n"
+  "desc-auth-ephemeral-key A/O8DVtnUheb3r1JqoB8uJB7wxXL1XJX3eny4yB+eFA=\n"
+  "auth-client oiNrQB8WwKo S5D02W7vKgiWIMygrBl8RQ FB//SfOBmLEx1kViEWWL1g\n"
+  "auth-client Od09Qu636Qo /PKLzqewAdS/+0+vZC+MvQ dpw4NFo13zDnuPz45rxrOg\n"
+  "auth-client JRr840iGYN0 8s8cxYqF7Lx23+NducC4Qg zAafl4wPLURkuEjJreZq1g\n"
+  "encrypted\n"
+  "-----BEGIN MESSAGE-----\n"
+  "YmVpbmcgb24gbW91bnRhaW5zLCB0aGlua2luZyBhYm91dCBjb21wdXRlcnMsIGlzIG5vdC"
+  "BiYWQgYXQgYWxs\n"
+  "-----END MESSAGE-----\n";
+
+static const char correct_encrypted_plaintext[] = "being on mountains, "
+  "thinking about computers, is not bad at all";
+
+static void
+test_parse_hs_desc_superencrypted(void *arg)
+{
+  (void) arg;
+  size_t retval;
+  uint8_t *encrypted_out = NULL;
+
+  {
+    setup_full_capture_of_logs(LOG_WARN);
+    retval = decode_superencrypted(bad_superencrypted_text1,
+                                   strlen(bad_superencrypted_text1),
+                                   &encrypted_out);
+    tt_u64_op(retval, ==, 0);
+    tt_assert(!encrypted_out);
+    expect_log_msg_containing("Unrecognized desc auth type");
+    teardown_capture_of_logs();
+  }
+
+  {
+    setup_full_capture_of_logs(LOG_WARN);
+    retval = decode_superencrypted(bad_superencrypted_text2,
+                                   strlen(bad_superencrypted_text2),
+                                   &encrypted_out);
+    tt_u64_op(retval, ==, 0);
+    tt_assert(!encrypted_out);
+    expect_log_msg_containing("Bogus desc auth key in HS desc");
+    teardown_capture_of_logs();
+  }
+
+  {
+    setup_full_capture_of_logs(LOG_WARN);
+    retval = decode_superencrypted(bad_superencrypted_text3,
+                                   strlen(bad_superencrypted_text3),
+                                   &encrypted_out);
+    tt_u64_op(retval, ==, 0);
+    tt_assert(!encrypted_out);
+    expect_log_msg_containing("Length of descriptor\'s encrypted data "
+                              "is too small.");
+    teardown_capture_of_logs();
+  }
+
+  /* Now finally the good one */
+  retval = decode_superencrypted(correct_superencrypted_text,
+                                 strlen(correct_superencrypted_text),
+                                 &encrypted_out);
+
+  tt_u64_op(retval, ==, strlen(correct_encrypted_plaintext));
+  tt_mem_op(encrypted_out, OP_EQ, correct_encrypted_plaintext,
+            strlen(correct_encrypted_plaintext));
+
+ done:
+  tor_free(encrypted_out);
+}
+
+struct testcase_t hs_descriptor[] = {
+  /* Encoding tests. */
+  { "cert_encoding", test_cert_encoding, TT_FORK,
+    NULL, NULL },
+  { "link_specifier", test_link_specifier, TT_FORK,
+    NULL, NULL },
+  { "encode_descriptor", test_encode_descriptor, TT_FORK,
+    NULL, NULL },
+  { "descriptor_padding", test_descriptor_padding, TT_FORK,
+    NULL, NULL },
+
+  /* Decoding tests. */
+  { "decode_descriptor", test_decode_descriptor, TT_FORK,
+    NULL, NULL },
+  { "encrypted_data_len", test_encrypted_data_len, TT_FORK,
+    NULL, NULL },
+  { "decode_invalid_intro_point", test_decode_invalid_intro_point, TT_FORK,
+    NULL, NULL },
+  { "decode_plaintext", test_decode_plaintext, TT_FORK,
+    NULL, NULL },
+  { "decode_bad_signature", test_decode_bad_signature, TT_FORK,
+    NULL, NULL },
+
+  /* Misc. */
+  { "version", test_supported_version, TT_FORK,
+    NULL, NULL },
+  { "validate_cert", test_validate_cert, TT_FORK,
+    NULL, NULL },
+  { "desc_signature", test_desc_signature, TT_FORK,
+    NULL, NULL },
+
+  { "parse_hs_desc_superencrypted", test_parse_hs_desc_superencrypted,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_descriptor.inc b/src/test/test_hs_descriptor.inc
new file mode 100644
index 0000000000..70a2c7c2f7
--- /dev/null
+++ b/src/test/test_hs_descriptor.inc
@@ -0,0 +1,224 @@
+static const char* HS_DESC_BAD_SIG =
+"hs-descriptor 3\n"
+"descriptor-lifetime 180\n"
+"descriptor-signing-key-cert\n"
+"-----BEGIN ED25519 CERT-----\n"
+"AQgABl5+AQoPXRnCGEOxIup3AcjQXb8npNiUFm2Qv7A6JKk/K+EuAQAgBAD18iUD\n"
+"nbkUblnUvTHzipq4bcr6aPyFVB42Ptobg4xr8s3VjHiJtjs9MDEdr6nXS7UlyhEl\n"
+"78vsuFEvLp7cvAgGxYY1xGXdn5RdHMCdi8W9yZLKMQX9OuJckmp1C6q+cA4=\n"
+"-----END ED25519 CERT-----\n"
+"revision-counter 42\n"
+"superencrypted\n"
+"-----BEGIN MESSAGE-----\n"
+"BxzghAOjM4De6Z6eGTvBrTP2SJDdQOYV/u9qtvlFsa2FRQWk20Adv3zJ/AI10CQO\n"
+"mUP4DNXM8FWQYGTvmD7wGz2/cXGjKwBXg1qO7zF5eP/D/My1sXsIfCcb41mkheNt\n"
+"xn1I5eKXcnghtd4lw7OkPVjSb/Z+VARUMmf+0qSNgmHLgEVnAoGJsn8W8B4qtIay\n"
+"4h4PuV0jPPlqJx6jMFOOEW72uqnfmqeNvClENXXW60xhnaxsf0up62fuW8ktu6Wf\n"
+"lnX/lvTstBFZZQ8/XI1+G+BPf8TZf7mxu0WYVg1s/KWYasYMSw46as59nkqdq2Ii\n"
+"qJnqHX/R20mWBhgpLse6wO0aNpky/rozEnikaPqyO1DShf6a6jXY8ADBg7spnK2/\n"
+"h7sf1+F1xfi2dy2WGxc1EUMP1kTVUmbft7kOo2nA7+3YZwQuSJHaN/66HrzU2x5z\n"
+"ayRUJ8+qDtfpEf17xthc/Uh253blFK96IoJJiqBfI6xt3IqOdHJq0OOC9zBbF6Rj\n"
+"vKMsaxmc/nc6uOB2WePYSgkZ0qs/dRKBJs6+Ahn1KdGkadyd8mDKL86Oe8lncHdB\n"
+"m/6sQjhKqFgngkCDOIlEJyWizqfN84AGqD5Zyxq0rbsN+9KLsHFfEbCRjgqjO5nS\n"
+"FYSFtuKgCZl2gaYEslL1pIEYE6BD2Whjn/HWTRyWiULJr6SuavgcbxeNEQDuVCC+\n"
+"fm0X7Z+qERaMAMR0vTMJK/NzT4GifrDpgmgbxc+34CtNBF5TriM8aXTNZZlsW00k\n"
+"d0XRxFbbbtiT5VOaEHbny7R3MdTVutEc9E/BhLBvjSSrGX7vrryh6Oj++nthIIzm\n"
+"F4M7I11S0TyA+UE06qF1C8rKmhcqU9MWy1SiccJ9KOWhJ5xwlsXBIID7wVygUhVl\n"
+"ovzfKkDDPfRoBch6NdVkxNJx3gb63CUmC2TzfwOMh973nntMVzqqw9A7jYkro9ln\n"
+"217kHUwMk3e83UgFL4nn7NCf3Kj0zhJ4jSfAsuQpV6e7dhzrlNya0lqrUsY2zFXP\n"
+"xv8wUtg6Vo1KewgVQas4oElkgFjDN8RJ7uBAwfuE/b9NnYJoQd76G8DHei/1PHbu\n"
+"tbtwN9I5RHaTvEOfetsJFnIAkCG6O4CQpzwHu1DdvEP4s6/el10b/4awBJ4VwOVZ\n"
+"YHSe4X0DStTV4Cu6aLh5OvrOmGbieRj6HdGQ6syYCaEBTuxbBUUpjIAfVlReAIph\n"
+"6aOrY6HNcCmeVmL5qm4dKr2XXOREsnUFuMqmfQuQd9pN3zlmS+RqCgSJuFrguFpd\n"
+"mjo6UxZvbjE7yJjtCih38HRe5BaigP5RDRkXmiXjqJ4koLJpyjQh19k3BYGcdxUC\n"
+"RCcYXydbGF7qHlnoaX9HnX7y6ZRsyKQpt91PMTGOUsB4fS8NhsqPpl2gdp4poLNs\n"
+"+hqjWZJ3uuLotXBcgM39Dtq9tqqu9vM12T80UAfWnVEHrBphmukh49EhEr2sx/la\n"
+"kAzRoTbLyTdlGVei8hI7/RtZIaIcOkzlhcFI5zmBlydyrv6/79vzt6WI/w9GVGpM\n"
+"OuSM0NS2CDJ7Iw412nz3CV1pEXB551ZBmbme6NHUe4EtEsDbgkP1Z201H4j51yVz\n"
+"wNoIksE5Bh5XRKuu4We5f9KZb+AEG9kxKJ5DbJk2YGJEQFTyfv0H68pl9urstPXD\n"
+"aMQF806COe2uhGm5gV/skvPVTeEvStE3K8DxZgcWNcTMVk8ZjrUHNfguVVToP8hT\n"
+"Fl4Iqo3r+JZEAGXnAbTpxUVC2Xxspf3jsT5xhUfB/NOexZxrXWnQZ+pscsbow0ba\n"
+"GATtakD3TF2WBqq5WscmOex+lrJcBCWVIzVWdwi5ngAtm1S7efkJlFUvmi4OuYnN\n"
+"RyZfxVIpoer8f2/xPXvxkOWFminDy5sFEvlh2/pnymfKOUV+CKih9ZApt+izlRJn\n"
+"+sMIOW6Jhf/WYyjeN6KQpwi6CDpclQJXA1SVoOVVL5A3lotLjs0x7ThIcBoxCZBq\n"
+"rFBhBu1gJgJ8guMySAHssIvhHHwXJsYEwzWCVAg/zIUXy4PLwIkgHApl+vGcldGv\n"
+"Br5HNCuqQ2pD9z2RvzNneB/LrYB214i+BP2piO5HbmeJBhby93blGXVfQewQT6aF\n"
+"dBlK8/jQM0rvb+LkmvQm2ypOttRpX2kyQXooJHYTTusaUr4jVmgngCvGtgqAQVqD\n"
+"HULXfHWvugZbAh6dXF7gKnnsyDOWwAgy4OJRi8i0jCaZ8aWSFRUjeGKT26dg/ayB\n"
+"U4QfMb8vL8tMdXVBfQLGcBgvrzQYrY69//pV6bX3SbLfUfWXV9eqUVWVPqVyPEwa\n"
+"Tz/aGVnGv/dY8h2cVnrgSXJGlOO+mCwSl+k9nk7VcEaKYuNlaOP3ZlKJvVj1LefM\n"
+"FODh4qTDBo5NkyfKu5fcZcOqDMBeGWXZzltE7CmvY7fOpDNMsuAoXYWI7q9gK82F\n"
+"w+nS0tVFCIWYa9DgGMv9GKTOk4Ia9elkbWypdRE/4oz4QxmHsArEsK4gDI+wmcp7\n"
+"/NsAZeuy96r2YDIUam4uASKOiAqrEfCv6B6cYctdYwZbAEXdo4fkGrCIjNRZmZGv\n"
+"kcZzHzIymnAmKRTkPt/LQ7Rx27Qd/Vt++B3zt2ORFuopqowOP0ocGZtkm0daK3Fc\n"
+"YDXMwIpf6Z8PwvvsG1bQHcSR+cUZi7vK7+hj/LGhMPafHM7HmFUbAxpJYr5CvR6y\n"
+"V1pZQYltT8xWayCeMHlLAAg10RgDkqCnY4dHnrY4GdwI2O7Wpxomni7qVHMjn+cN\n"
+"UTrd7EeVw+dxAIYosuqG7ua7ee3VGoOs+XMLrscAqHahfGbyYC+j+6Tow4qwWBdU\n"
+"/W3NJXnRWaHTXFHllpClnxggPRQx4yPtgTOmBBVl/O0T6i4Bv0ygsJeZAqC3VmAJ\n"
+"QodQTzGf2jwqsZf4uHKQa0EKGQvTGjFVgAFNpHmAuzyqh0b1pq5JeXiFERGsKC3j\n"
+"xcJilq1XeIx4SL38YNuCxi4pnyJyLnGGHpNjdjeFO5lvgCaKPegsPo4hpNpTvBJ1\n"
+"D7+o3E5CqxzjRt9kQmtwBbuH/SQX2T0x8aQ6vhwjj8ftDfw+FbjpMR9zfU0Lf8V7\n"
+"UjVGIl2yiVBGScBZu1nSD83PxjFy3XdFtBYoU5OrlXwBEYQs91jwK7UCiGtjI2Ao\n"
+"ZGkJaBd4AqP6voyJiGnC3LWFcmeMyzfExgiclQwfhFqqf762TX5JwG6xGqtdcNKS\n"
+"k54LlcI/RfvJw3ncSs9YsodZr6Jz5irpRTHX5WwCrX9mLukP96SXo29bIXEZAqEr\n"
+"ZxEcF0zlYE+km5bRfRCRcVVrScugCshSNLOdQp6fOAtHCl7rdQ/8Rz7oHuqieLVi\n"
+"UldRsAmpk9fIfRLphXj4j24jRP0VtL/LoJwakWTa0xO8K7eBAMVITI+HgFfN4wSO\n"
+"Yh1B+bGD5WKxFsWSgBMmW+YLF5ZtxVmmbg7wK2dIpJs4pjg2YO/MTO7SifJ9kjcb\n"
+"bCc74Tjs5mLLGGjGCIoXfda6WXbt2it40XhFk2zUAcPPsgjbctftkaWph7JSZpmZ\n"
+"fVcPqKdhmA1U0LA2XEOMTxGyCAeseH6pJXZm9LdBozc1CwyWP8XEDHHJf35vfPKY\n"
+"JDe2PanFepIOHaoRTgE7ZkGWKzOIKlS0Ucr1ezVfcxiFgQUNM+MYXXbUz51BVVq1\n"
+"Dulg4VvX104nt/ULijcfa/TsE+uklEnkyk1mhavH337NQg38XF4cAngNlUF4nSW/\n"
+"j0jizbAtaSx1f7q6xqPm3zPRlHrGQizHXLyl+SLzDUVPOXbPwcoeev97YeeyB6h5\n"
+"NBbIK9hmekNDmYIwI0bmlrg6IXhC5pyvRe8sQlV+9wBY2liF0M1mq5onW3a55afp\n"
+"+ynxXfQucb1HxZLXvRIGMBgWSQ7HfIPASqSE90Vu6qQCfkOW5PDqONr4BM65V4+g\n"
+"AYsVEgaosgHw9CF7yKgkvmZpToOtGpCHVcdUeeY2/rrQnAQeSy19gj/baJ+OKl6Q\n"
+"i1EGU8Yqo2r0d4XDFp/eKgC4sv57qp1PwkYQ/HKqoelJ09IAZL2sQWc05BGwt1A0\n"
+"11qDIEdkZBjzK3qUnY3QlOuoZtALZrnPg56SlF1RGDOPqbcF+3opqsvzBoiikh4V\n"
+"WV5OUYjRDMUDLQqf/OkuktdYf5N3RcbYP0XsAvY0ZWG3Gp068b3p8peCpkDzrF9p\n"
+"bQ2ZvS304tN7+p0hif3+JyZy5/sxl17RxTeg5I3mo2+J0ptQDYwF/WadONO8r7uU\n"
+"YlRltFtQfyMzyVzHON4NHGjZh7dDGtWp0MGeHRBHQsC8bEChhvWme19VXhgZoWpl\n"
+"dUIZkSuvRwiURXjhKbZrEdJbVmr9FX6zoyOahv3VnmcEARoR+umxzvo3hGQPbHyH\n"
+"jTsQtSBjs75/9fCxcYmBWkh3JHVDVsCbV+z+5KZpk3m50J4Y1hC8hvepC9CaBqOM\n"
+"DjfyXh58x1yKiueEbcjSWsRuF7CjcrYnFUBHOs9U1j9WytCI3fhOWPMgR4UZpGuU\n"
+"WlcR1BXg1wYxX273xOS/jYn9MLAVlbRpPTUMIH9VRP+sc8+XaxKpJSCl4C+vcwNY\n"
+"1YdKD2QiuoBJ3fXGtqMVRtn9eZvatSJuY9CnRKRbf0hWmFD4D5RkiwE0WkdtwoHR\n"
+"uEXJ47RlF0/JDU1fY1mXBkq3usvB4Absy78qL06vh45xkk9bHbdf+7Ao1RQKmqiB\n"
+"NL5XnjBu+YX535WG7t7Su3mTCJXYHvn72ATxry8yhSLgWqt81STkRwc14HmrOGG8\n"
+"Gw7bz7y5vikj/rnPyr7ry+QRgNNDDayAqenAu2vEAzWir0RQC/iZ9rc/r7YQWGgL\n"
+"Xrd4TQ6rTZePARhwB3VomnLDDvLvi2oq/jPzLKSYM2a7qj/vBSbJ/NnNaDW5Ccew\n"
+"RjMI1lIHeedqYTVAW/CKoSEPcFSAzi/Ija0gcWLgX5xsFDGIYBepAX0KS9426kMu\n"
+"0r/V66zmPMusMilqRTx7KW+jZMVxXVc2zClcdmohMmtjsbqLkczprfSbdGswMv9Q\n"
+"I1ktHJHIRD0vPeZXnvKZsRKZw3sKb0ltZi33ZxCJFQPeGGtM5aAFthj6awcXy6Tt\n"
+"DPUQdCU/vh1zmGRAX17/Xb0irfvN+GhQLEl42pzhigJXc/rCG3a4Na8wT+xAIZVf\n"
+"WUI7hMslx5wA+iB4lrAjCq0YIrjINI/lHYpotXUZGmz5wz0jOciTmXMSx9du4cpk\n"
+"fIQJfR+fr5tG3fjHMgSP+p+RewHkd/7RUAmHC2k3cuk5pCJvUVJrhUIqsi1fa0LG\n"
+"GA0UU6Nr9tpYdNr1WkbKQjxTg0D//AXe61jmUS5XUU4AQf6zQVfN0TMtmuYeacbK\n"
+"4r6Z1CSIRbsgcnL1BN8GSd4KddkCqSk941aJUCoX+77ou4t0btVSB9FnLKipigtE\n"
+"E/Rpmv+81lA4fLiIag62/pcJ3uppsZ9aaHdR10SMmuCjAVLYHqhJfrHHn32dyqLK\n"
+"UI8kEZJ6GQzHLUXcGbbdnk1Qm6JwO8TeF/oQvh9y9py+oAyFy0qzP2UeUMUI2yRQ\n"
+"mlWSy+wX1DbVDQ3UHwJjWp65CgyYXuW8eCB0AbyF0kF4KGf7/7Ae7tEGbmYSm5MA\n"
+"71z+Azxtv5gRyRb787V2dyo0wcmbRlL7iUBVXNM/czQo31tAZIwLc+lKNp0SPH6g\n"
+"gJ2yX/GeDSFNAeEVUZ/f4KZIa7QQsnGWrUr+agSnQFkySmIjWYjwC/abJwah0v0d\n"
+"ulwr3tECaaXtoWVdYXa3utEclBz9umBwMJ9MQCm4Kx7dTYUWFT3bMM/ESTkGPcfm\n"
+"m+C4FsqFBs80WY0ududu50vTDSdJt1RqZ7Sg6DNH6acBvWyXOpT5mPJKUjnSFwyG\n"
+"oVLgv0aDDx7lLZdCkhyz/Ff5LNmBgQsjGllPszJ2gTZxZ5LD68S4kUirQG/qtzlS\n"
+"PGfDOC79SMZGgsoAnr4wV3RUTxsTVFlxVHsBMB+EXOFHAr3wHTVxUGBbGzxBlQ9w\n"
+"I/jlu8LIIexXAU75HS5KCGGfg0Z7BLqEzqpMKqcBQC7BD7GnCXrDSQ2DCXnl7bLN\n"
+"lIrQ/z2Y8AgSdED46R40MqyyN6CPPNiOCjONHZ30fLEXuEgCp4R/+x0WWsWpjGk2\n"
+"Ydkc03cx/X6moUYxB5HTqTodBmAQuWMX0rxFDrnR0SWghWjdWth9gjd+dvZ82tt1\n"
+"UMUywDPhcYchtUi2lnqnYJm5p00GN9Mk14MC5ZC5qP57IJVqxu0ktOMpks+CLPnz\n"
+"qp9OBpI4sIzd0y0aUJC2Gd+E9aAhlREIiicyBDmxLdk1i37QeeCralI3eubLNmE+\n"
+"CjDjD8t8FUGPpKglSD3lfLTqbp2TUvyWfvJC6ulFPNsAbeLHTnPnpyPQmWxhMNGt\n"
+"h67B9tbYww2TvNwqIgmB4+YIR4/pSs15TpAqvuUvjpmRwGklqgiSmrQrlIxCxux/\n"
+"mfsaL3KE97wm8BsaMpMkjUL7ByTIFhFZ/gHPTxaFpbqTZ4G+lABLgp3bIsB9Dl/P\n"
+"ovoqX+qL2Mq9T0GrVJGfRBuA5hISw63hx5zdsj2Cj3A3khHPqR+GRN/rVYUuOpLm\n"
+"z3v5pU/74vZRmNMAIhyhmweSEPNtyVkgSdgbFErqvhxN0om2Cd/7cWh2g5BXHyUL\n"
+"PBr7ZkgfsE9TnuDH7Z0JoBqXJki+MO6nqz73oH2Mm86yxcXp6O/ieKTollrUJ3yQ\n"
+"P6hLcEbYPzUV99del7Va5Wi0nn0wbRXCGVQdwY+iWc7pT+VVlncyg0TvLXi0OtOt\n"
+"O8xbT2DAzVXxMwOsKV9ZgS/0dtwzwICpnTzBI/47V8GYhHbOUNTBPZ52GaXMeWlX\n"
+"cuRGb0+7OkKWuriyOQ5z5xaASCVfqgnOwSZYiAk0gcDoK+JHdr64/sMoJhH87R4i\n"
+"2TO90whkScgiGR7A06Ba42bT1nJtI6pxvzdB2b4BDAs2Lr2OdcB3BY1dtzKjFkw/\n"
+"qfIw3F55UQwcs84ZEFQDAB/tmfNHajblDFpXR4N5QvU/PdWVWJUub7oNyhIX6ruu\n"
+"ln4H7lpTUHJZ7jkr1qpnvkztZtHGlpJ0QdUHgyMYER1xU58Hg77yzIW3EdAa2PyK\n"
+"1t4udKbQKChShlShIMzwzj57ss/69QobrpYAHYi6IRMaMUGBfipGBACK3yeXsXz0\n"
+"c3Q2J5vI6QbxNsiJ5t7Ry1IqotbJcU7HND/yVUAUbEg5CpEDOSeSOW/ulyLuFxEV\n"
+"lRTwIO/68BoIoR7umlP23/1N5OYzaBHhH2nThILBovHeJRXnGXSgeFfwSj7LIYEV\n"
+"c1MdDSg/HzoADPXyEPLzqFzHRHeNiqEolmOPnFh0hRzbMZ0W5TQPDGWJdF21g816\n"
+"vA0WW4UQjLM+vnX9kKKLA1ut+9JWk1dGKsmWtdWUDfJjUP/L6dS4OYEl6O6+SjM9\n"
+"GcyGvHTiC5OpJllYpvELP/NjtTf9or8Bmruuga/axeOuS5ocYLK/sGRlmO6Z96da\n"
+"QSlyGWEQAnM2D1cDmdd4CetPslOVIcQ41+coWCi2xg3UjO/bFK1CA4R1rb4ekXfs\n"
+"s5U2XChyHhUPgl57y1r0ILXRXWJTJ0/F9hhu4aYQVFeIV/IuzJbmTKKkAcCOH6ys\n"
+"qnu2BXz8Pm2tU10JFfRcuZ8rHuUyUErA40ESsLijON98GMwL4Rat9ZSCNS5hlK7y\n"
+"yRJdr0ITp8oTbduAoulgWOvtcw1L87QBVojWz3cbhXra+WITirYuGNbzfmZn1WQM\n"
+"kukEZUEHSypGOrHr1XiuY4Rw/DBaJSLyZ+VybEOfXqXkDBh5s1ayypBvzrzFZCIn\n"
+"PJxIVsvrkhrpEbTJ9d7zLWjhOa9ZWw8lAubllbGm+7qCfdHmGsfBtvJdzx6zhB1Y\n"
+"otL/PCis2XVTBEDJeB8pGqKFOZjNz8PC5qP+ymtAfy2ktl/u4HsFlxV7CsEKGYPm\n"
+"p3LqnhPUy5M5gin4E4uPPyzzD2kcM3way49FKWUKlblQU0SyWtHRmMB3vcVmyT85\n"
+"BRULXF7jgog7XR/EMltwQyJI6GcUCrnWZu+G0BEwXG+CsgCzE7assDavc1NSGLZM\n"
+"rmzXiFFyfk7CE6lW2Lm+oWaFwKdvpmNZJFGGX8ZHRE9ZvkFMnfw9MYf2W7xa0jf7\n"
+"k3c6X5wMuk9mznVtq5itNFVXh1mT1ujeWOiiqyH5UhQQjj6O+ZXt4gqt/jT6dd1i\n"
+"jRuhhxaUGOlhpVBW/ySXhZ+HgOy9aCJ/bgjRGaqGixogk4f4rcgigHruwTpOQuDn\n"
+"xDZ3Xns70S40WtHSYN+Gbl9nIh4yl78aNnA4FVtTAuLlVKEKlMJi9OBFuP5TEczG\n"
+"+0HTwL/VPSCI+8FUZBhlz3YwecYq6dY5mS46+luPW+5Wl+5jtzb8V9oxVnRx2hQq\n"
+"B5HJsM5FOOhHDHMXoCsevj7N/ufK7cU7Wbr0DkgYRwvb0ZJB5WYgcaQ0W7aduhGb\n"
+"MQsandhP8Ajb2cmLobi3mHHPbcEkvjT8JP9Sim5xtfF+oCMMB5ByA5bI2aIFybZm\n"
+"jX9e/V8wNgtpDKDVKPjB3+9dj5gU1N5JsrjQwQDB0kVRMWdpJCtD4hZ2+T/QE3SI\n"
+"f8Rdk8pj8qBzRPbnhW6qsoWZdjMRC8qixZqHw4jol09UF7Ab9hjEF5ZDTfNGXwy8\n"
+"/hz8su+mr8hhrlCrOF2vBYUayAA96zhbDWfg3Pdxo9bTn3/DmyAngL4J5Gu679xK\n"
+"rWN4j7uQG4bzTa8WJb09/lW49UzWvmrz0c6/yexk3T//xDD067FafdnP5pYs4Cvp\n"
+"rCoHpXbKjxx99DJmb5iXW0JRLSpFSCbf1HPHbmzST3minSXap5FCWDJcSgExKIJp\n"
+"DXZ9rk0LMnQA74MWC5gjjM+5t0AHKuNRhJbQSwYWTKqeApXho53T/COlfDlSs2tb\n"
+"Vz1Ia5z7IOfu1QheE93huNAHT3Ob+mSmUq782SqFPr6uwud/l5uP3HpcuwugdlFm\n"
+"Jw8uBBOQ53W4lLbYfQYTVgieClVhmYMu7Ye0xYZ5B2jf714sjZRMa0LCbsyj58xH\n"
+"uzs8ddNN1fLMzb0JRBE8JWj5PbxhA/sTwMkD7SnEMBUTtP0obmuQ982aTfyvQCH/\n"
+"ve8OUPtYf5XWNv18mpR+h+riMt1Y8Eb6BJzTMFNWagMJAe3JV6A6upHroNFo2FxY\n"
+"1XPRM1Rt0zKo7GD+oXnixfpl1aG8yqZhYo1ZC9buaHwH6zvM+xoiGD0iujeDtpVy\n"
+"Vp6cAqqaGmrNwcPVBLc7hNKrJnbFKyhjL5/xp9j6jQov1aWQ8HsaNvh0p2ljmlwb\n"
+"daTYZcwLgSgPna7HhiqnOSAmXZ7St/qe/b9TqBtIVzwzmtevgMyG98QV0syFP5X6\n"
+"2Jc1g9733sTZp7njq4Cu07JhpICpinhLWR3nkODJbjk/mpLcQZgtV6W749AUo8oT\n"
+"jRVEJ8MpCo1h0bVDxsRnA3DrMneD88L8/b10aHs+bPm1HKbCmT+kJAFaUQNa8JvJ\n"
+"pReN37qTWvZCte7vaPAIP5cboATMu/J4t3izpm+YJoJlWcIegGx3kQ+17P4MbgDl\n"
+"S93U4sOLvTk9+MoyPo9yGWU/zHgzcQ6wCFdzWMDRswuh+/4TJ2+yg6maq3iBtj39\n"
+"gNLMR+sRgGGvYisqE9bfvNQy5IWrABBKcSBTXeTM1DmW6jv3TI8DoCzCbpjqcIwT\n"
+"u2J+7k8wJEHPcAwnBjlyWphVvwNwM0cXqOnlJZ/4z7OGgjiNEem7TMuvxk+YkiXK\n"
+"OzftdTjeIpzBwsGRP8/teMBpjS95M7GloKtxO+muBVxXbmsq8GBRC9vtNJ2Ma/xP\n"
+"bXvd+7caytD3ob6ZfOzCpi4ZS8uByEfIMxlgZ5Sn3jhgEkcIU+YW9b3teMZOuWdA\n"
+"QpDCoMpXaHVyRqwVV59JjmftiBnNBEo1/QzRj2UxRi7fHMfmNxL5LRM4CHSLUSCq\n"
+"Y3A3pkxvBHUzemhynSFvtCPa8GHiUpe9so0V/2hlgaENAVELPjMlWytaYufRllgy\n"
+"tUnCd32C5PrrmYzMKnxKRPXLcxLgziruJGSks9vIspoPk0pWgkZm+M9fRpJKlWHF\n"
+"yT9OOGBW2yynw/yvXssxJmdUDxVcWL4uS2bZc4s0Zc6RSL9uQPjZVX0JLj+cXfx4\n"
+"93Gn5bDhMgm+CGM6j3RiAAD7tT5V0sytNFjXd1A4U1u8yj3wzhKqOtZpDmuGUlMn\n"
+"EODu7I5KtWxOTPThy7TecI5r+F/6KL+2MOtRhj2PmlT/Xed6PaAmDkQeiXGps08x\n"
+"u0JIpuB61axvT4PAsKZNUd4ExbzNxRDAARUMgY8krpmyKZyHVFIQ19uHM2lGl9/i\n"
+"h3PKlLHYI8RsHutHElzq+F5tWd5AA99LVRZX4axAVIQNiqRg8IMSoCwUaCCbjUMz\n"
+"sJCo2t36GYk5S2BRnfrCqYoZRHw+ENYN0tDEMhXq1OqjvNHW3TzL3DsUhM6EZU5n\n"
+"cRR4ynUvPqqWFphLefRW10vCtaW9roJQZyFYf9kd8xgW/BhcDNbTTaQ1U6xCHgX+\n"
+"78DKee/NvY1WIEBR8X0iVk5XlSJb14eRtxNawXFyebVdmC/DiMNgnTBncMbePnZi\n"
+"KCl1r5xqo7tSIoJ6Z0l6qINd89T9fcg9mujTVwsfQ+5/kdEy0Iw7CQcTOGvMaoPX\n"
+"IAJlWSVeZ8eu8kmsD1Z8ewoPufMKiY4cPRAK5bCDgsrK6bAExOlCwPnNNM8Ym1Hz\n"
+"aYFeGs5sW468Qww+Nbl5xcNFKtwUKZ6EebRHjwttiyTgCdAhv9wL1u2WFydWWgkG\n"
+"rwUbNpSLKls+pijCeJAscvxzbZz96iOaYrY8IyzGBFwfgFAESfnzBc8SQjZzMzoO\n"
+"vmYIRon2m/5w5AZA2IjQ4VxXJDK6XExD/ZLsxNXzMnROD++hE+s8DvPlRPmN4egF\n"
+"gAzJs/9t7IyE/dDf7gSSBqzEBbwduD8ozzYHwELUc4ERdRzjEdBM0azT61g7Yilr\n"
+"iT5Hy+2iw/pNwiqVOYiAbj2lwcoMlFZmdxviD4IMXdsNVWsCAVJL0PqIh1UDDb3z\n"
+"Urv3idBJeSBuuFr6AFS6kAgvrwV/pEGoBoHuyii/rZxVugGKeuMynKEvSHuFNuQU\n"
+"qIHcNgqQR34v2Ut5pQ1R8s7K3Rae/AhE5GncJa6FJmB9TF8MYMu9PlSZV/eGv8UL\n"
+"IDWQ7sY3NdhZini//xtwPqIw29yOeZ0X6Aqsek9tfh21UwKSpHb7T+PwXYmoB+23\n"
+"p3FXkP/rv4AGRq1xJqFYzKJvwsXqTFuNFWP74yhTg6rC90w2p5TeH1rJMAnv4u0L\n"
+"hGtG/NL+D1Tzdf00TYAjno5Ia5dQJDd/eO+Ygqnhl6hAqGtS6r9JhIEXw1nQD7SC\n"
+"lj96ZuKdUWO8rpIiAtvHAsn++xvMVPm/S1SwA8oE049iVwS8/eNNiMKoSlTlYc7o\n"
+"pusBZQrVF4We4HHYFjysBbcXlvoXDd8LkZ8Nh63VQPnoIGNKH2U6aXCnQcJ8dZqO\n"
+"DNxL4uyM4A578FUUR6vxqt2asnLHQ0Z7pPE4uqtz/WgbiHI/i2oHS8oe1clsifCw\n"
+"3ZY33kflqLftkTNka1oiftDb0OqFLjkS7/AUorqHazw53gM3gqJY5EXA3Px9+nhu\n"
+"NzxSK/t41JoCfgQJHMkIWb3yUcO4OFZeGCeAxIJY95hv/brt6/WNielXjNaohYvc\n"
+"lsSUHEJRHwVxQmWK0LS+g13HAgOI7cNt3MA8sSkzTneHGFgEvmrSyb0wCEmushC9\n"
+"mjQThvaxfQk9douA/cR2bHr7axXqv9vjztmxUr0a30a7lvLMBQbJmFtJJylW+tJe\n"
+"v/vKNOB+9mK793cttr2JFnMhwUKFKWiFDQJtxw/eLQWY4BJ19Rs2x4BJgmV+u1jB\n"
+"zR8uvxuArG/cqVEJsoC6uuSzhAWSwdvumijO6yuyWF6nHY6aAcy8dyFQlDFHAd+/\n"
+"J05Lrbzj4N9lcI7hPalh0uMdERGvtUdT8QRm5ebP1zogYEkZk/1GOU29dMawkAt/\n"
+"SWhp2yWdjLt8f5HQKu72vUF/yyTfzfdqQqJwfthP7+vp+sHDO85AMF45uU9g3pxW\n"
+"IbXSbZ4fFGC1/41db/2GOHFgaheMXj0SIWHqQE1jtihr3BBBO4b3Ccz5QCnrn48J\n"
+"8L+QRdh4a/cAx4ty/oHEiXwpSBBSFRl5+y2NijC8GITA5dRjCRWP+Y0zuTrJ7j1a\n"
+"h+3kGs1kxqskhaEuhXnXyknGLjXrU+ewRGhHzP23o5betVhX+c1XjVqmJNZ5OPn/\n"
+"wrqx/XwoIl/3F5lMmGDG9mPtyg0E227nKl9Sy0Vbwx2tu1unjOlzSCa7lpoD4TIX\n"
+"PBJ5+Zb0CE6HEt3V0ec1m4uUe/xObAnzyr4UbzdqLaMy8vTcF/qsncXyPBjwqdjR\n"
+"ReDAtt99bAPY4roPKGt8dgKUPE0t/XoY+SlmUp75TkZDXrOIJXpEW0GpLPf53T+W\n"
+"Ex3KtfLAnZzrw8+dIageY7IgoQ85h3sYE7uEI8QlcO/o4udqUzTp4Sn4sWvdTLrx\n"
+"W7ImvK2rsU5ubVdsEaFKM7+7nxGn2JyMpIWFz0SbP34CkXHhrXxyRD+GhMIDHFxV\n"
+"uBnZnjJsw+ooIm1rL4I7/VMWEwmVegreT6w9Gsmb5igw+zu9v2YBgTOhysA9XZd4\n"
+"7O3VjqKkhTXcBqdpRWuz8gPQ+4rfwij28Gg2alG04Eh3G3868NOCFJhhaHVmwYR2\n"
+"ygRm6N9eDW1bHhYSN75HSEb6aIebk+1AT4S1QtJaPSH0EduIXO++JYAs+jIFKy2c\n"
+"jCVFlO/LbXl7iCdXurJHpSbMNmZFNUri6zEolENODLwke836jBOKiVrWzLnEMxHI\n"
+"WDDTpLTYhR3C7sEprpEQm9SX2Eik3WxVb4ZTb7SZFU1y1d4tWnjGu3U1D+vO9wVq\n"
+"Sss9lDipbkhQ9k4j1/Pqozaxvi8lYLbh3WEjK3Iwpr66Bk6Ai2oRg4b+7vzV4o+6\n"
+"L47JPJhajdHac0CIlmupyA4eejECS6OpoLDf5Wr/616k3dxM//3kAWGUnXVw9GSo\n"
+"UF5W8AaKlaGZ6EZk09NyGSFRjEs18z+g5ckviGF0EhZI7ZPWQQmlqWUsL9O0S4GO\n"
+"ZZ9f0UhNmHEspcugbs7e1yfjwGVyxIBkrmxpkmfHE4Gb47UGlJevg2OvZOPT3wMH\n"
+"vOds2BtqdT3tuss9k+7hsISGse7isEOb7TN5MHb6yyzqnCUZhp5m3Iag7TUkiyfU\n"
+"jKH5R13tHqKUoJ2rofWoLO2H5xSfp/lqF9sLd4rJ+Pbjhiuvfwz5copYsuTNL4kB\n"
+"SPUikHlTxSOgTBYNV77qxpsqOI3+iziCrSqHsxNdlaA1T3fiq6SeZBNdD822AYm9\n"
+"L5hbcgpDPEEwT/n5kWNbRNueerJkJwboaOnT1ZX1601Pwj5QDi+YM1NYy5PsdWxb\n"
+"bPGpQyZ+uf917q9gV7Ykr5cic10YD11khAghr0n6fYfb8Ijc22uP6m47KItDqQc1\n"
+"eFym149F56B0yg5FR85Arg==\n"
+"-----END MESSAGE-----\n"
+" signature Of+jvQKzH9ot2NV5twlDO2CFbzLSB4absWTwG58TCHb+TWgQi3z6SZIoTnGGY/uicJgEkCN++bZZR49GiyHyCQ\n";
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
new file mode 100644
index 0000000000..c6197875b5
--- /dev/null
+++ b/src/test/test_hs_intropoint.c
@@ -0,0 +1,888 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_service.c
+ * \brief Test hidden service functionality.
+ */
+
+#define HS_SERVICE_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+#define RENDSERVICE_PRIVATE
+#define CIRCUITLIST_PRIVATE
+
+#include "test.h"
+#include "log_test_helpers.h"
+#include "crypto.h"
+#include "log_test_helpers.h"
+
+#include "or.h"
+#include "ht.h"
+
+/* Trunnel. */
+#include "hs/cell_establish_intro.h"
+#include "hs/cell_introduce1.h"
+#include "hs/cell_common.h"
+#include "hs_service.h"
+#include "hs_common.h"
+#include "hs_circuitmap.h"
+#include "hs_intropoint.h"
+
+#include "circuitlist.h"
+#include "circuituse.h"
+#include "rendservice.h"
+#include "relay.h"
+
+/* Mock function to avoid networking in unittests */
+static int
+mock_send_intro_established_cell(or_circuit_t *circ)
+{
+  (void) circ;
+  return 0;
+}
+
+static int
+mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
+                                  uint8_t relay_command, const char *payload,
+                                  size_t payload_len,
+                                  crypt_path_t *cpath_layer,
+                                  const char *filename, int lineno)
+{
+  (void) stream_id;
+  (void) circ;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  (void) cpath_layer;
+  (void) filename;
+  (void) lineno;
+  return 0;
+}
+
+static or_circuit_t *
+helper_create_intro_circuit(void)
+{
+  or_circuit_t *circ = or_circuit_new(0, NULL);
+  tt_assert(circ);
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+ done:
+  return circ;
+}
+
+static trn_cell_introduce1_t *
+helper_create_introduce1_cell(void)
+{
+  trn_cell_introduce1_t *cell = NULL;
+  ed25519_keypair_t auth_key_kp;
+
+  /* Generate the auth_key of the cell. */
+  if (ed25519_keypair_generate(&auth_key_kp, 0) < 0) {
+    goto err;
+  }
+
+  cell = trn_cell_introduce1_new();
+  tt_assert(cell);
+
+  /* Set the auth key. */
+  {
+    size_t auth_key_len = sizeof(auth_key_kp.pubkey);
+    trn_cell_introduce1_set_auth_key_type(cell,
+                                         HS_INTRO_AUTH_KEY_TYPE_ED25519);
+    trn_cell_introduce1_set_auth_key_len(cell, auth_key_len);
+    trn_cell_introduce1_setlen_auth_key(cell, auth_key_len);
+    uint8_t *auth_key_ptr = trn_cell_introduce1_getarray_auth_key(cell);
+    memcpy(auth_key_ptr, auth_key_kp.pubkey.pubkey, auth_key_len);
+  }
+
+  /* Set the cell extentions to none. */
+  {
+    trn_cell_extension_t *ext = trn_cell_extension_new();
+    trn_cell_extension_set_num(ext, 0);
+    trn_cell_introduce1_set_extensions(cell, ext);
+  }
+
+  /* Set the encrypted section to some data. */
+  {
+    size_t enc_len = 128;
+    trn_cell_introduce1_setlen_encrypted(cell, enc_len);
+    uint8_t *enc_ptr = trn_cell_introduce1_getarray_encrypted(cell);
+    memset(enc_ptr, 'a', enc_len);
+  }
+
+  return cell;
+ err:
+ done:
+  trn_cell_introduce1_free(cell);
+  return NULL;
+}
+
+/* Try sending an ESTABLISH_INTRO cell on a circuit that is already an intro
+ * point. Should fail. */
+static void
+test_establish_intro_wrong_purpose(void *arg)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  memcpy(intro_circ->rend_circ_nonce, circuit_key_material, DIGEST_LEN);
+
+  /* Set a bad circuit purpose!! :) */
+  circuit_change_purpose(TO_CIRCUIT(intro_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                           sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                         establish_intro_cell);
+  tt_int_op(cell_len, >, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Rejecting ESTABLISH_INTRO on non-OR circuit.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  trn_cell_establish_intro_free(establish_intro_cell);
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Prepare a circuit for accepting an ESTABLISH_INTRO cell */
+static void
+helper_prepare_circ_for_intro(or_circuit_t *circ,
+                              uint8_t *circuit_key_material)
+{
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  memcpy(circ->rend_circ_nonce, circuit_key_material, DIGEST_LEN);
+}
+
+/* Send an empty ESTABLISH_INTRO cell. Should fail. */
+static void
+test_establish_intro_wrong_keytype(void *arg)
+{
+  int retval;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, (uint8_t*)"", 0);
+  expect_log_msg_containing("Empty ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Send an ESTABLISH_INTRO cell with an unknown auth key type. Should fail. */
+static void
+test_establish_intro_wrong_keytype2(void *arg)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                           sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                         establish_intro_cell);
+  tt_int_op(cell_len, >, 0);
+
+  /* Mutate the auth key type! :) */
+  cell_body[0] = 42;
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Unrecognized AUTH_KEY_TYPE 42.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  trn_cell_establish_intro_free(establish_intro_cell);
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong MAC. Should fail. */
+static void
+test_establish_intro_wrong_mac(void *arg)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                                sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  /* Mangle one byte of the MAC. */
+  uint8_t *handshake_ptr =
+    trn_cell_establish_intro_getarray_handshake_mac(establish_intro_cell);
+  handshake_ptr[TRUNNEL_SHA3_256_LEN - 1]++;
+  /* We need to resign the payload with that change. */
+  {
+    ed25519_signature_t sig;
+    ed25519_keypair_t key_struct;
+    /* New keypair for the signature since we don't have access to the private
+     * key material generated earlier when creating the cell. */
+    retval = ed25519_keypair_generate(&key_struct, 0);
+    tt_int_op(retval, OP_EQ, 0);
+    uint8_t *auth_key_ptr =
+      trn_cell_establish_intro_getarray_auth_key(establish_intro_cell);
+    memcpy(auth_key_ptr, key_struct.pubkey.pubkey, ED25519_PUBKEY_LEN);
+    /* Encode payload so we can sign it. */
+    cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                           establish_intro_cell);
+    tt_int_op(cell_len, >, 0);
+
+    retval = ed25519_sign_prefixed(&sig, cell_body,
+                                   cell_len -
+                                   (ED25519_SIG_LEN +
+                                    sizeof(establish_intro_cell->sig_len)),
+                                   ESTABLISH_INTRO_SIG_PREFIX, &key_struct);
+    tt_int_op(retval, OP_EQ, 0);
+    /* And write the signature to the cell */
+    uint8_t *sig_ptr =
+      trn_cell_establish_intro_getarray_sig(establish_intro_cell);
+    memcpy(sig_ptr, sig.sig, establish_intro_cell->sig_len);
+    /* Re-encode with the new signature. */
+    cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                           establish_intro_cell);
+  }
+
+  /* Receive the cell. Should fail because our MAC is wrong. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO handshake_auth not as expected");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  trn_cell_establish_intro_free(establish_intro_cell);
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong auth key length. Should
+ * fail. */
+static void
+test_establish_intro_wrong_auth_key_len(void *arg)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  size_t bad_auth_key_len = ED25519_PUBKEY_LEN - 1;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                               sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  /* Mangle the auth key length. */
+  trn_cell_establish_intro_set_auth_key_len(establish_intro_cell,
+                                           bad_auth_key_len);
+  trn_cell_establish_intro_setlen_auth_key(establish_intro_cell,
+                                          bad_auth_key_len);
+  cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                         establish_intro_cell);
+  tt_int_op(cell_len, >, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO auth key length is invalid");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  trn_cell_establish_intro_free(establish_intro_cell);
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but with a wrong sig length. Should
+ * fail. */
+static void
+test_establish_intro_wrong_sig_len(void *arg)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  size_t bad_sig_len = ED25519_SIG_LEN - 1;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                               sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  /* Mangle the signature length. */
+  trn_cell_establish_intro_set_sig_len(establish_intro_cell, bad_sig_len);
+  trn_cell_establish_intro_setlen_sig(establish_intro_cell, bad_sig_len);
+  cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                         establish_intro_cell);
+  tt_int_op(cell_len, >, 0);
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("ESTABLISH_INTRO sig len is invalid");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  trn_cell_establish_intro_free(establish_intro_cell);
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Send a legit ESTABLISH_INTRO cell but slightly change the signature. Should
+ * fail. */
+static void
+test_establish_intro_wrong_sig(void *arg)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  (void)arg;
+
+  /* Get the auth key of the intro point */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                           sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                         establish_intro_cell);
+  tt_int_op(cell_len, >, 0);
+
+  /* Mutate the last byte (signature)! :) */
+  cell_body[cell_len-1]++;
+
+  /* Receive the cell. Should fail. */
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  expect_log_msg_containing("Failed to verify ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_int_op(retval, ==, -1);
+
+ done:
+  trn_cell_establish_intro_free(establish_intro_cell);
+  circuit_free(TO_CIRCUIT(intro_circ));
+}
+
+/* Helper function: Send a well-formed v3 ESTABLISH_INTRO cell to
+ * <b>intro_circ</b>. Return the cell. */
+static trn_cell_establish_intro_t *
+helper_establish_intro_v3(or_circuit_t *intro_circ)
+{
+  int retval;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  tt_assert(intro_circ);
+
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  establish_intro_cell = generate_establish_intro_cell(circuit_key_material,
+                                           sizeof(circuit_key_material));
+  tt_assert(establish_intro_cell);
+  cell_len = get_establish_intro_payload(cell_body, sizeof(cell_body),
+                                         establish_intro_cell);
+  tt_int_op(cell_len, >, 0);
+
+  /* Receive the cell */
+  retval = hs_intro_received_establish_intro(intro_circ, cell_body, cell_len);
+  tt_int_op(retval, ==, 0);
+
+ done:
+  return establish_intro_cell;
+}
+
+/* Helper function: Send a well-formed v2 ESTABLISH_INTRO cell to
+ * <b>intro_circ</b>. Return the public key advertised in the cell. */
+static crypto_pk_t *
+helper_establish_intro_v2(or_circuit_t *intro_circ)
+{
+  crypto_pk_t *key1 = NULL;
+  int retval;
+  uint8_t cell_body[RELAY_PAYLOAD_SIZE];
+  ssize_t cell_len = 0;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  tt_assert(intro_circ);
+
+  /* Prepare the circuit for the incoming ESTABLISH_INTRO */
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+  helper_prepare_circ_for_intro(intro_circ, circuit_key_material);
+
+  /* Send legacy establish_intro */
+  key1 = pk_generate(0);
+
+  /* Use old circuit_key_material why not */
+  cell_len = encode_establish_intro_cell_legacy((char*)cell_body,
+                                                sizeof(cell_body),
+                                                key1,
+                                                (char *) circuit_key_material);
+  tt_int_op(cell_len, >, 0);
+
+  /* Receive legacy establish_intro */
+  retval = hs_intro_received_establish_intro(intro_circ,
+                                       cell_body, cell_len);
+  tt_int_op(retval, ==, 0);
+
+ done:
+  return key1;
+}
+
+/* Helper function: test circuitmap free_all function outside of
+ * test_intro_point_registration to prevent Coverity from seeing a
+ * double free if the assertion hypothetically fails.
+ */
+static void
+test_circuitmap_free_all(void)
+{
+  hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+  the_hs_circuitmap = get_hs_circuitmap();
+  tt_assert(the_hs_circuitmap);
+  hs_circuitmap_free_all();
+  the_hs_circuitmap = get_hs_circuitmap();
+  tt_assert(!the_hs_circuitmap);
+ done:
+  ;
+}
+
+/** Successfuly register a v2 intro point and a v3 intro point. Ensure that HS
+ *  circuitmap is maintained properly. */
+static void
+test_intro_point_registration(void *arg)
+{
+  int retval;
+  hs_circuitmap_ht *the_hs_circuitmap = NULL;
+
+  or_circuit_t *intro_circ = NULL;
+  trn_cell_establish_intro_t *establish_intro_cell = NULL;
+  ed25519_public_key_t auth_key;
+
+  crypto_pk_t *legacy_auth_key = NULL;
+  or_circuit_t *legacy_intro_circ = NULL;
+
+  or_circuit_t *returned_intro_circ = NULL;
+
+  (void) arg;
+
+  MOCK(hs_intro_send_intro_established_cell, mock_send_intro_established_cell);
+
+  hs_circuitmap_init();
+
+  /* Check that the circuitmap is currently empty */
+  {
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(0, ==, HT_SIZE(the_hs_circuitmap));
+    /* Do a circuitmap query in any case */
+    returned_intro_circ =hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    tt_ptr_op(returned_intro_circ, ==, NULL);
+  }
+
+  /* Create a v3 intro point */
+  {
+    intro_circ = or_circuit_new(0, NULL);
+    tt_assert(intro_circ);
+    establish_intro_cell = helper_establish_intro_v3(intro_circ);
+
+    /* Check that the intro point was registered on the HS circuitmap */
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(1, ==, HT_SIZE(the_hs_circuitmap));
+    get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
+                           establish_intro_cell);
+    returned_intro_circ =
+      hs_circuitmap_get_intro_circ_v3_relay_side(&auth_key);
+    tt_ptr_op(intro_circ, ==, returned_intro_circ);
+  }
+
+  /* Create a v2 intro point */
+  {
+    char key_digest[DIGEST_LEN];
+
+    legacy_intro_circ = or_circuit_new(1, NULL);
+    tt_assert(legacy_intro_circ);
+    legacy_auth_key = helper_establish_intro_v2(legacy_intro_circ);
+    tt_assert(legacy_auth_key);
+
+    /* Check that the circuitmap now has two elements */
+    the_hs_circuitmap = get_hs_circuitmap();
+    tt_assert(the_hs_circuitmap);
+    tt_int_op(2, ==, HT_SIZE(the_hs_circuitmap));
+
+    /* Check that the new element is our legacy intro circuit. */
+    retval = crypto_pk_get_digest(legacy_auth_key, key_digest);
+    tt_int_op(retval, ==, 0);
+    returned_intro_circ =
+      hs_circuitmap_get_intro_circ_v2_relay_side((uint8_t*)key_digest);
+    tt_ptr_op(legacy_intro_circ, ==, returned_intro_circ);
+  }
+
+  /* XXX Continue test and try to register a second v3 intro point with the
+   * same auth key. Make sure that old intro circuit gets closed. */
+
+ done:
+  crypto_pk_free(legacy_auth_key);
+  circuit_free(TO_CIRCUIT(intro_circ));
+  circuit_free(TO_CIRCUIT(legacy_intro_circ));
+  trn_cell_establish_intro_free(establish_intro_cell);
+  test_circuitmap_free_all();
+
+  UNMOCK(hs_intro_send_intro_established_cell);
+}
+
+static void
+test_introduce1_suitable_circuit(void *arg)
+{
+  int ret;
+  or_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Valid suitable circuit. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 1);
+  }
+
+  /* Test if the circuit purpose safeguard works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Test the non-edge circuit safeguard works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    /* Bogus pointer, the check is against NULL on n_chan. */
+    circ->base_.n_chan = (channel_t *) circ;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Mangle the circuit a bit more so see if our only one INTRODUCE1 cell
+   * limit works correctly. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+    circ->already_received_introduce1 = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    circuit_free(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  ;
+}
+
+static void
+test_introduce1_is_legacy(void *arg)
+{
+  int ret;
+  uint8_t request[256];
+
+  (void) arg;
+
+  /* For a cell to be considered legacy, according to the specification, the
+   * first 20 bytes MUST BE non-zero else it's a v3 cell. */
+  memset(request, 'a', DIGEST_LEN);
+  memset(request + DIGEST_LEN, 0, sizeof(request) - DIGEST_LEN);
+  ret = introduce1_cell_is_legacy(request);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* This is a NON legacy cell. */
+  memset(request, 0, DIGEST_LEN);
+  memset(request + DIGEST_LEN, 'a', sizeof(request) - DIGEST_LEN);
+  ret = introduce1_cell_is_legacy(request);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  ;
+}
+
+static void
+test_introduce1_validation(void *arg)
+{
+  int ret;
+  trn_cell_introduce1_t *cell = NULL;
+
+  (void) arg;
+
+  /* Create our decoy cell that we'll modify as we go to test the validation
+   * function of that parsed cell. */
+  cell = helper_create_introduce1_cell();
+
+  /* It should NOT be a legacy cell which will trigger a BUG(). */
+  memset(cell->legacy_key_id, 'a', sizeof(cell->legacy_key_id));
+  tor_capture_bugs_(1);
+  ret = validate_introduce1_parsed_cell(cell);
+  tor_end_capture_bugs_();
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset legacy ID and make sure it's correct. */
+  memset(cell->legacy_key_id, 0, sizeof(cell->legacy_key_id));
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Non existing auth key type. */
+  cell->auth_key_type = 42;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset is to correct value and make sure it's correct. */
+  cell->auth_key_type = HS_INTRO_AUTH_KEY_TYPE_ED25519;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Really bad key length. */
+  cell->auth_key_len = 0;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  cell->auth_key_len = UINT16_MAX;
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Correct size, let's try that. */
+  cell->auth_key_len = sizeof(ed25519_public_key_t);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Set an invalid size of the auth key buffer. */
+  trn_cell_introduce1_setlen_auth_key(cell, 3);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset auth key buffer and make sure it works. */
+  trn_cell_introduce1_setlen_auth_key(cell, sizeof(ed25519_public_key_t));
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Empty encrypted section. */
+  trn_cell_introduce1_setlen_encrypted(cell, 0);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, -1);
+  /* Reset it to some non zero bytes and validate. */
+  trn_cell_introduce1_setlen_encrypted(cell, 1);
+  ret = validate_introduce1_parsed_cell(cell);
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  trn_cell_introduce1_free(cell);
+}
+
+static void
+test_received_introduce1_handling(void *arg)
+{
+  int ret;
+  uint8_t *request = NULL, buf[128];
+  trn_cell_introduce1_t *cell = NULL;
+  or_circuit_t *circ = NULL;
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Too small request length. An INTRODUCE1 expect at the very least a
+   * DIGEST_LEN size. */
+  {
+    circ = helper_create_intro_circuit();
+    ret = hs_intro_received_introduce1(circ, buf, DIGEST_LEN - 1);
+    tt_int_op(ret, OP_EQ, -1);
+    circuit_free(TO_CIRCUIT(circ));
+  }
+
+  /* We have a unit test only for the suitability of a circuit to receive an
+   * INTRODUCE1 cell so from now on we'll only test the handling of a cell. */
+
+  /* Bad request. */
+  {
+    circ = helper_create_intro_circuit();
+    uint8_t test[2]; /* Too small request. */
+    ret = handle_introduce1(circ, test, sizeof(test));
+    tor_free(circ->p_chan);
+    circuit_free(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, -1);
+  }
+
+  /* Valid case. */
+  {
+    cell = helper_create_introduce1_cell();
+    ssize_t request_len = trn_cell_introduce1_encoded_len(cell);
+    tt_int_op((int)request_len, OP_GT, 0);
+    request = tor_malloc_zero(request_len);
+    ssize_t encoded_len =
+      trn_cell_introduce1_encode(request, request_len, cell);
+    tt_int_op((int)encoded_len, OP_GT, 0);
+
+    circ = helper_create_intro_circuit();
+    or_circuit_t *service_circ = helper_create_intro_circuit();
+    circuit_change_purpose(TO_CIRCUIT(service_circ),
+                           CIRCUIT_PURPOSE_INTRO_POINT);
+    /* Register the circuit in the map for the auth key of the cell. */
+    ed25519_public_key_t auth_key;
+    const uint8_t *cell_auth_key =
+      trn_cell_introduce1_getconstarray_auth_key(cell);
+    memcpy(auth_key.pubkey, cell_auth_key, ED25519_PUBKEY_LEN);
+    hs_circuitmap_register_intro_circ_v3_relay_side(service_circ, &auth_key);
+    ret = hs_intro_received_introduce1(circ, request, request_len);
+    circuit_free(TO_CIRCUIT(circ));
+    circuit_free(TO_CIRCUIT(service_circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Valid legacy cell. */
+  {
+    tor_free(request);
+    trn_cell_introduce1_free(cell);
+    cell = helper_create_introduce1_cell();
+    uint8_t *legacy_key_id = trn_cell_introduce1_getarray_legacy_key_id(cell);
+    memset(legacy_key_id, 'a', DIGEST_LEN);
+    /* Add an arbitrary amount of data for the payload of a v2 cell. */
+    size_t request_len = trn_cell_introduce1_encoded_len(cell) + 256;
+    tt_size_op(request_len, OP_GT, 0);
+    request = tor_malloc_zero(request_len + 256);
+    ssize_t encoded_len =
+      trn_cell_introduce1_encode(request, request_len, cell);
+    tt_int_op((int)encoded_len, OP_GT, 0);
+
+    circ = helper_create_intro_circuit();
+    or_circuit_t *service_circ = helper_create_intro_circuit();
+    circuit_change_purpose(TO_CIRCUIT(service_circ),
+                           CIRCUIT_PURPOSE_INTRO_POINT);
+    /* Register the circuit in the map for the auth key of the cell. */
+    uint8_t token[REND_TOKEN_LEN];
+    memcpy(token, legacy_key_id, sizeof(token));
+    hs_circuitmap_register_intro_circ_v2_relay_side(service_circ, token);
+    ret = hs_intro_received_introduce1(circ, request, request_len);
+    circuit_free(TO_CIRCUIT(circ));
+    circuit_free(TO_CIRCUIT(service_circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+ done:
+  trn_cell_introduce1_free(cell);
+  tor_free(request);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+struct testcase_t hs_intropoint_tests[] = {
+  { "intro_point_registration",
+    test_intro_point_registration, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_keytype",
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_keytype2",
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_purpose",
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_sig",
+    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_sig_len",
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_auth_key_len",
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+
+  { "receive_establish_intro_wrong_mac",
+    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+
+  { "introduce1_suitable_circuit",
+    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+
+  { "introduce1_is_legacy",
+    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+
+  { "introduce1_validation",
+    test_introduce1_validation, TT_FORK, NULL, NULL },
+
+  { "received_introduce1_handling",
+    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_hs_ntor.sh b/src/test/test_hs_ntor.sh
new file mode 100755
index 0000000000..8a0003d44a
--- /dev/null
+++ b/src/test/test_hs_ntor.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+# Validate Tor's ntor implementation.
+
+exitcode=0
+
+# Run the python integration test sand return the exitcode of the python
+# script.  The python script might ask the testsuite to skip it if not all
+# python dependencies are covered.
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/hs_ntor_ref.py" || exitcode=$?
+
+exit ${exitcode}
diff --git a/src/test/test_hs_ntor_cl.c b/src/test/test_hs_ntor_cl.c
new file mode 100644
index 0000000000..ed1eda58ea
--- /dev/null
+++ b/src/test/test_hs_ntor_cl.c
@@ -0,0 +1,255 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/** This is a wrapper over the little-t-tor HS ntor functions. The wrapper is
+ *  used by src/test/hs_ntor_ref.py to conduct the HS ntor integration
+ *  tests.
+ *
+ *  The logic of this wrapper is basically copied from src/test/test_ntor_cl.c
+ */
+
+#include "orconfig.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ONION_NTOR_PRIVATE
+#include "or.h"
+#include "util.h"
+#include "compat.h"
+#include "crypto.h"
+#include "crypto_curve25519.h"
+#include "hs_ntor.h"
+#include "onion_ntor.h"
+
+#define N_ARGS(n) STMT_BEGIN {                                  \
+    if (argc < (n)) {                                           \
+      fprintf(stderr, "%s needs %d arguments.\n",argv[1],n);    \
+      return 1;                                                 \
+    }                                                           \
+  } STMT_END
+#define BASE16(idx, var, n) STMT_BEGIN {                                \
+    const char *s = argv[(idx)];                                        \
+    if (base16_decode((char*)var, n, s, strlen(s)) < (int)n ) {              \
+      fprintf(stderr, "couldn't decode argument %d (%s)\n",idx,s);      \
+      return 1;                                                         \
+    }                                                                   \
+  } STMT_END
+#define INT(idx, var) STMT_BEGIN {                                      \
+    var = atoi(argv[(idx)]);                                            \
+    if (var <= 0) {                                                     \
+      fprintf(stderr, "bad integer argument %d (%s)\n",idx,argv[(idx)]); \
+    }                                                                   \
+  } STMT_END
+
+/** The first part of the HS ntor protocol. The client-side computes all
+    necessary key material and sends the appropriate message to the service. */
+static int
+client1(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_public_key_t intro_enc_pubkey;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
+
+  char buf[256];
+
+  N_ARGS(6);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(4, client_ephemeral_enc_keypair.seckey.secret_key,
+         CURVE25519_SECKEY_LEN);
+  BASE16(5, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
+                                 &client_ephemeral_enc_keypair.seckey);
+
+  retval = hs_ntor_client_get_introduce1_keys(&intro_auth_pubkey,
+                                              &intro_enc_pubkey,
+                                              &client_ephemeral_enc_keypair,
+                                              subcredential,
+                                              &hs_ntor_intro_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send ENC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.enc_key,
+                sizeof(hs_ntor_intro_cell_keys.enc_key));
+  printf("%s\n", buf);
+  /* Send MAC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.mac_key,
+                sizeof(hs_ntor_intro_cell_keys.mac_key));
+  printf("%s\n", buf);
+
+ done:
+  return retval;
+}
+
+/** The second part of the HS ntor protocol. The service-side computes all
+    necessary key material and sends the appropriate message to the client */
+static int
+server1(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_keypair_t intro_enc_keypair;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_public_key_t client_ephemeral_enc_pubkey;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
+  hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+  curve25519_keypair_t service_ephemeral_rend_keypair;
+
+  char buf[256];
+
+  N_ARGS(6);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, intro_enc_keypair.seckey.secret_key, CURVE25519_SECKEY_LEN);
+  BASE16(4, client_ephemeral_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(5, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&intro_enc_keypair.pubkey,
+                                 &intro_enc_keypair.seckey);
+  curve25519_keypair_generate(&service_ephemeral_rend_keypair, 0);
+
+  /* Get INTRODUCE1 keys */
+  retval = hs_ntor_service_get_introduce1_keys(&intro_auth_pubkey,
+                                               &intro_enc_keypair,
+                                               &client_ephemeral_enc_pubkey,
+                                               subcredential,
+                                               &hs_ntor_intro_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Get RENDEZVOUS1 keys */
+  retval = hs_ntor_service_get_rendezvous1_keys(&intro_auth_pubkey,
+                                               &intro_enc_keypair,
+                                               &service_ephemeral_rend_keypair,
+                                               &client_ephemeral_enc_pubkey,
+                                               &hs_ntor_rend_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send ENC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.enc_key,
+                sizeof(hs_ntor_intro_cell_keys.enc_key));
+  printf("%s\n", buf);
+  /* Send MAC_KEY */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_intro_cell_keys.mac_key,
+                sizeof(hs_ntor_intro_cell_keys.mac_key));
+  printf("%s\n", buf);
+  /* Send AUTH_MAC */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+                sizeof(hs_ntor_rend_cell_keys.rend_cell_auth_mac));
+  printf("%s\n", buf);
+  /* Send NTOR_KEY_SEED */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.ntor_key_seed,
+                sizeof(hs_ntor_rend_cell_keys.ntor_key_seed));
+  printf("%s\n", buf);
+  /* Send service ephemeral pubkey (Y) */
+  base16_encode(buf, sizeof(buf),
+                (const char*)service_ephemeral_rend_keypair.pubkey.public_key,
+                sizeof(service_ephemeral_rend_keypair.pubkey.public_key));
+  printf("%s\n", buf);
+
+ done:
+  return retval;
+}
+
+/** The final step of the ntor protocol, the client computes and returns the
+ *  rendezvous key material. */
+static int
+client2(int argc, char **argv)
+{
+  int retval;
+
+  /* Inputs */
+  curve25519_public_key_t intro_enc_pubkey;
+  ed25519_public_key_t intro_auth_pubkey;
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+  curve25519_public_key_t service_ephemeral_rend_pubkey;
+  uint8_t subcredential[DIGEST256_LEN];
+
+  /* Output */
+  hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
+
+  char buf[256];
+
+  N_ARGS(7);
+  BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
+  BASE16(3, client_ephemeral_enc_keypair.seckey.secret_key,
+         CURVE25519_SECKEY_LEN);
+  BASE16(4, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(5, service_ephemeral_rend_pubkey.public_key, CURVE25519_PUBKEY_LEN);
+  BASE16(6, subcredential, DIGEST256_LEN);
+
+  /* Generate keypair */
+  curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
+                                 &client_ephemeral_enc_keypair.seckey);
+
+  /* Get RENDEZVOUS1 keys */
+  retval = hs_ntor_client_get_rendezvous1_keys(&intro_auth_pubkey,
+                                               &client_ephemeral_enc_keypair,
+                                               &intro_enc_pubkey,
+                                               &service_ephemeral_rend_pubkey,
+                                               &hs_ntor_rend_cell_keys);
+  if (retval < 0) {
+    goto done;
+  }
+
+  /* Send AUTH_MAC */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+                sizeof(hs_ntor_rend_cell_keys.rend_cell_auth_mac));
+  printf("%s\n", buf);
+  /* Send NTOR_KEY_SEED */
+  base16_encode(buf, sizeof(buf),
+                (const char*)hs_ntor_rend_cell_keys.ntor_key_seed,
+                sizeof(hs_ntor_rend_cell_keys.ntor_key_seed));
+  printf("%s\n", buf);
+
+ done:
+  return 1;
+}
+
+/** Perform a different part of the protocol depdning on the argv used. */
+int
+main(int argc, char **argv)
+{
+  if (argc < 2) {
+    fprintf(stderr, "I need arguments. Read source for more info.\n");
+    return 1;
+  }
+
+  curve25519_init();
+  if (!strcmp(argv[1], "client1")) {
+    return client1(argc, argv);
+  } else if (!strcmp(argv[1], "server1")) {
+    return server1(argc, argv);
+  } else if (!strcmp(argv[1], "client2")) {
+    return client2(argc, argv);
+  } else {
+    fprintf(stderr, "What's a %s?\n", argv[1]);
+    return 1;
+  }
+}
+
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
new file mode 100644
index 0000000000..fcfb3b992d
--- /dev/null
+++ b/src/test/test_hs_service.c
@@ -0,0 +1,250 @@
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_service.c
+ * \brief Test hidden service functionality.
+ */
+
+#define HS_COMMON_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+
+#include "test.h"
+#include "log_test_helpers.h"
+#include "crypto.h"
+
+#include "hs/cell_establish_intro.h"
+#include "hs_common.h"
+#include "hs_service.h"
+#include "hs_intropoint.h"
+
+#include "hs_ntor.h"
+
+/** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
+ *  parse it from the receiver side. */
+static void
+test_gen_establish_intro_cell(void *arg)
+{
+  (void) arg;
+  ssize_t retval;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+  uint8_t buf[RELAY_PAYLOAD_SIZE];
+  trn_cell_establish_intro_t *cell_out = NULL;
+  trn_cell_establish_intro_t *cell_in = NULL;
+
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+
+  /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
+     attempt to parse it. */
+  {
+    cell_out = generate_establish_intro_cell(circuit_key_material,
+                                             sizeof(circuit_key_material));
+    tt_assert(cell_out);
+
+    retval = get_establish_intro_payload(buf, sizeof(buf), cell_out);
+    tt_int_op(retval, >=, 0);
+  }
+
+  /* Parse it as the receiver */
+  {
+    ssize_t parse_result = trn_cell_establish_intro_parse(&cell_in,
+                                                         buf, sizeof(buf));
+    tt_int_op(parse_result, >=, 0);
+
+    retval = verify_establish_intro_cell(cell_in,
+                                         circuit_key_material,
+                                         sizeof(circuit_key_material));
+    tt_int_op(retval, >=, 0);
+  }
+
+ done:
+  trn_cell_establish_intro_free(cell_out);
+  trn_cell_establish_intro_free(cell_in);
+}
+
+/* Mocked ed25519_sign_prefixed() function that always fails :) */
+static int
+mock_ed25519_sign_prefixed(ed25519_signature_t *signature_out,
+                           const uint8_t *msg, size_t msg_len,
+                           const char *prefix_str,
+                           const ed25519_keypair_t *keypair) {
+  (void) signature_out;
+  (void) msg;
+  (void) msg_len;
+  (void) prefix_str;
+  (void) keypair;
+  return -1;
+}
+
+/** We simulate a failure to create an ESTABLISH_INTRO cell */
+static void
+test_gen_establish_intro_cell_bad(void *arg)
+{
+  (void) arg;
+  trn_cell_establish_intro_t *cell = NULL;
+  uint8_t circuit_key_material[DIGEST_LEN] = {0};
+
+  MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
+
+  crypto_rand((char *) circuit_key_material, sizeof(circuit_key_material));
+
+  setup_full_capture_of_logs(LOG_WARN);
+  /* Easiest way to make that function fail is to mock the
+     ed25519_sign_prefixed() function and make it fail. */
+  cell = generate_establish_intro_cell(circuit_key_material,
+                                       sizeof(circuit_key_material));
+  expect_log_msg_containing("Unable to gen signature for "
+                            "ESTABLISH_INTRO cell.");
+  teardown_capture_of_logs();
+  tt_assert(!cell);
+
+ done:
+  trn_cell_establish_intro_free(cell);
+  UNMOCK(ed25519_sign_prefixed);
+}
+
+/** Test the HS ntor handshake. Simulate the sending of an encrypted INTRODUCE1
+ *  cell, and verify the proper derivation of decryption keys on the other end.
+ *  Then simulate the sending of an authenticated RENDEZVOUS1 cell and verify
+ *  the proper verification on the other end. */
+static void
+test_hs_ntor(void *arg)
+{
+  int retval;
+
+  uint8_t subcredential[DIGEST256_LEN];
+
+  ed25519_keypair_t service_intro_auth_keypair;
+  curve25519_keypair_t service_intro_enc_keypair;
+  curve25519_keypair_t service_ephemeral_rend_keypair;
+
+  curve25519_keypair_t client_ephemeral_enc_keypair;
+
+  hs_ntor_intro_cell_keys_t client_hs_ntor_intro_cell_keys;
+  hs_ntor_intro_cell_keys_t service_hs_ntor_intro_cell_keys;
+
+  hs_ntor_rend_cell_keys_t service_hs_ntor_rend_cell_keys;
+  hs_ntor_rend_cell_keys_t client_hs_ntor_rend_cell_keys;
+
+  (void) arg;
+
+  /* Generate fake data for this unittest */
+  {
+    /* Generate fake subcredential */
+    memset(subcredential, 'Z', DIGEST256_LEN);
+
+    /* service */
+    curve25519_keypair_generate(&service_intro_enc_keypair, 0);
+    ed25519_keypair_generate(&service_intro_auth_keypair, 0);
+    curve25519_keypair_generate(&service_ephemeral_rend_keypair, 0);
+    /* client */
+    curve25519_keypair_generate(&client_ephemeral_enc_keypair, 0);
+  }
+
+  /* Client: Simulate the sending of an encrypted INTRODUCE1 cell */
+  retval =
+    hs_ntor_client_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
+                                       &service_intro_enc_keypair.pubkey,
+                                       &client_ephemeral_enc_keypair,
+                                       subcredential,
+                                       &client_hs_ntor_intro_cell_keys);
+  tt_int_op(retval, ==, 0);
+
+  /* Service: Simulate the decryption of the received INTRODUCE1 */
+  retval =
+    hs_ntor_service_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
+                                        &service_intro_enc_keypair,
+                                        &client_ephemeral_enc_keypair.pubkey,
+                                        subcredential,
+                                        &service_hs_ntor_intro_cell_keys);
+  tt_int_op(retval, ==, 0);
+
+  /* Test that the INTRODUCE1 encryption/mac keys match! */
+  tt_mem_op(client_hs_ntor_intro_cell_keys.enc_key, OP_EQ,
+            service_hs_ntor_intro_cell_keys.enc_key,
+            CIPHER256_KEY_LEN);
+  tt_mem_op(client_hs_ntor_intro_cell_keys.mac_key, OP_EQ,
+            service_hs_ntor_intro_cell_keys.mac_key,
+            DIGEST256_LEN);
+
+  /* Service: Simulate creation of RENDEZVOUS1 key material. */
+  retval =
+    hs_ntor_service_get_rendezvous1_keys(&service_intro_auth_keypair.pubkey,
+                                         &service_intro_enc_keypair,
+                                         &service_ephemeral_rend_keypair,
+                                         &client_ephemeral_enc_keypair.pubkey,
+                                         &service_hs_ntor_rend_cell_keys);
+  tt_int_op(retval, ==, 0);
+
+  /* Client: Simulate the verification of a received RENDEZVOUS1 cell */
+  retval =
+    hs_ntor_client_get_rendezvous1_keys(&service_intro_auth_keypair.pubkey,
+                                        &client_ephemeral_enc_keypair,
+                                        &service_intro_enc_keypair.pubkey,
+                                        &service_ephemeral_rend_keypair.pubkey,
+                                        &client_hs_ntor_rend_cell_keys);
+  tt_int_op(retval, ==, 0);
+
+  /* Test that the RENDEZVOUS1 key material match! */
+  tt_mem_op(client_hs_ntor_rend_cell_keys.rend_cell_auth_mac, OP_EQ,
+            service_hs_ntor_rend_cell_keys.rend_cell_auth_mac,
+            DIGEST256_LEN);
+  tt_mem_op(client_hs_ntor_rend_cell_keys.ntor_key_seed, OP_EQ,
+            service_hs_ntor_rend_cell_keys.ntor_key_seed,
+            DIGEST256_LEN);
+
+ done:
+  ;
+}
+
+/** Test that our HS time period calculation functions work properly */
+static void
+test_time_period(void *arg)
+{
+  (void) arg;
+  uint64_t tn;
+  int retval;
+  time_t fake_time;
+
+  /* Let's do the example in prop224 section [TIME-PERIODS] */
+  retval = parse_rfc1123_time("Wed, 13 Apr 2016 11:00:00 UTC",
+                              &fake_time);
+  tt_int_op(retval, ==, 0);
+
+  /* Check that the time period number is right */
+  tn = get_time_period_num(fake_time);
+  tt_u64_op(tn, ==, 16903);
+
+  /* Increase current time to 11:59:59 UTC and check that the time period
+     number is still the same */
+  fake_time += 3599;
+  tn = get_time_period_num(fake_time);
+  tt_u64_op(tn, ==, 16903);
+
+  /* Now take time to 12:00:00 UTC and check that the time period rotated */
+  fake_time += 1;
+  tn = get_time_period_num(fake_time);
+  tt_u64_op(tn, ==, 16904);
+
+  /* Now also check our hs_get_next_time_period_num() function */
+  tn = hs_get_next_time_period_num(fake_time);
+  tt_u64_op(tn, ==, 16905);
+
+ done:
+  ;
+}
+
+struct testcase_t hs_service_tests[] = {
+  { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
+    NULL, NULL },
+  { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
+    NULL, NULL },
+  { "hs_ntor", test_hs_ntor, TT_FORK,
+    NULL, NULL },
+  { "time_period", test_time_period, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 810b03c93d..cfb8d83b1d 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c
index 95657349c6..d2ec8e9ca7 100644
--- a/src/test/test_keypin.c
+++ b/src/test/test_keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index ddf66f4d34..c5508b0f04 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,13 +6,20 @@
 #define CHANNELTLS_PRIVATE
 #define CONNECTION_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
+#define TORTLS_PRIVATE
+
+#include "compat.h"
+
 #include "or.h"
 #include "config.h"
 #include "connection.h"
 #include "connection_or.h"
 #include "channeltls.h"
 #include "link_handshake.h"
+#include "router.h"
+#include "routerkeys.h"
 #include "scheduler.h"
+#include "torcert.h"
 
 #include "test.h"
 #include "log_test_helpers.h"
@@ -37,6 +44,16 @@ mock_tls_cert_matches_key(const tor_tls_t *tls, const tor_x509_cert_t *cert)
   (void) cert; // XXXX look at this.
   return 1;
 }
+static tor_tls_t *mock_peer_cert_expect_tortls = NULL;
+static tor_x509_cert_t *mock_peer_cert = NULL;
+static tor_x509_cert_t *
+mock_get_peer_cert(tor_tls_t *tls)
+{
+  if (mock_peer_cert_expect_tortls &&
+      mock_peer_cert_expect_tortls != tls)
+    return NULL;
+  return tor_x509_cert_dup(mock_peer_cert);
+}
 
 static int mock_send_netinfo_called = 0;
 static int
@@ -57,14 +74,29 @@ mock_close_for_err(or_connection_t *orconn, int flush)
 }
 
 static int mock_send_authenticate_called = 0;
+static int mock_send_authenticate_called_with_type = 0;
 static int
 mock_send_authenticate(or_connection_t *conn, int type)
 {
   (void) conn;
-  (void) type;
+  mock_send_authenticate_called_with_type = type;
   ++mock_send_authenticate_called;// XXX check_this
   return 0;
 }
+static int
+mock_export_key_material(tor_tls_t *tls, uint8_t *secrets_out,
+                         const uint8_t *context,
+                         size_t context_len,
+                         const char *label)
+{
+  (void) tls;
+  (void)secrets_out;
+  (void)context;
+  (void)context_len;
+  (void)label;
+  memcpy(secrets_out, "int getRandomNumber(){return 4;}", 32);
+  return 0;
+}
 
 static tor_x509_cert_t *mock_own_cert = NULL;
 static tor_x509_cert_t *
@@ -78,20 +110,23 @@ mock_get_own_cert(tor_tls_t *tls)
 static void
 test_link_handshake_certs_ok(void *arg)
 {
-  (void) arg;
-
   or_connection_t *c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   or_connection_t *c2 = or_connection_new(CONN_TYPE_OR, AF_INET);
   var_cell_t *cell1 = NULL, *cell2 = NULL;
   certs_cell_t *cc1 = NULL, *cc2 = NULL;
   channel_tls_t *chan1 = NULL, *chan2 = NULL;
   crypto_pk_t *key1 = NULL, *key2 = NULL;
+  const int with_ed = !strcmp((const char *)arg, "Ed25519");
+
+  tor_addr_from_ipv4h(&c1->base_.addr, 0x7f000001);
+  tor_addr_from_ipv4h(&c2->base_.addr, 0x7f000001);
 
   scheduler_init();
 
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
+  MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);
   MOCK(tor_tls_get_own_cert, mock_get_own_cert);
 
   key1 = pk_generate(2);
@@ -103,6 +138,15 @@ test_link_handshake_certs_ok(void *arg)
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
                                  key1, key2, 86400), ==, 0);
 
+  if (with_ed) {
+    /* If we're making a CERTS cell for an ed handshake, let's make sure we
+     * have some Ed25519 certificates and keys. */
+    init_mock_ed_keys(key2);
+  } else {
+    certs_cell_ed25519_disabled_for_testing = 1;
+  }
+
+  /* c1 has started_here == 1 */
   {
     const tor_x509_cert_t *link_cert = NULL;
     tt_assert(!tor_tls_get_my_certs(1, &link_cert, NULL));
@@ -113,6 +157,7 @@ test_link_handshake_certs_ok(void *arg)
   c1->link_proto = 3;
   tt_int_op(connection_init_or_handshake_state(c1, 1), ==, 0);
 
+  /* c2 has started_here == 0 */
   c2->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   c2->link_proto = 3;
   tt_int_op(connection_init_or_handshake_state(c2, 0), ==, 0);
@@ -136,8 +181,13 @@ test_link_handshake_certs_ok(void *arg)
   tt_int_op(cell2->payload_len, ==,
             certs_cell_parse(&cc2, cell2->payload, cell2->payload_len));
 
-  tt_int_op(2, ==, cc1->n_certs);
-  tt_int_op(2, ==, cc2->n_certs);
+  if (with_ed) {
+    tt_int_op(5, ==, cc1->n_certs);
+    tt_int_op(5, ==, cc2->n_certs);
+  } else {
+    tt_int_op(2, ==, cc1->n_certs);
+    tt_int_op(2, ==, cc2->n_certs);
+  }
 
   tt_int_op(certs_cell_get_certs(cc1, 0)->cert_type, ==,
             CERTTYPE_RSA1024_ID_AUTH);
@@ -149,6 +199,22 @@ test_link_handshake_certs_ok(void *arg)
   tt_int_op(certs_cell_get_certs(cc2, 1)->cert_type, ==,
             CERTTYPE_RSA1024_ID_ID);
 
+  if (with_ed) {
+    tt_int_op(certs_cell_get_certs(cc1, 2)->cert_type, ==,
+              CERTTYPE_ED_ID_SIGN);
+    tt_int_op(certs_cell_get_certs(cc1, 3)->cert_type, ==,
+              CERTTYPE_ED_SIGN_AUTH);
+    tt_int_op(certs_cell_get_certs(cc1, 4)->cert_type, ==,
+              CERTTYPE_RSA1024_ID_EDID);
+
+    tt_int_op(certs_cell_get_certs(cc2, 2)->cert_type, ==,
+              CERTTYPE_ED_ID_SIGN);
+    tt_int_op(certs_cell_get_certs(cc2, 3)->cert_type, ==,
+              CERTTYPE_ED_SIGN_LINK);
+    tt_int_op(certs_cell_get_certs(cc2, 4)->cert_type, ==,
+              CERTTYPE_RSA1024_ID_EDID);
+  }
+
   chan1 = tor_malloc_zero(sizeof(*chan1));
   channel_tls_common_init(chan1);
   c1->chan = chan1;
@@ -159,13 +225,39 @@ test_link_handshake_certs_ok(void *arg)
   c1->base_.conn_array_index = -1;
   crypto_pk_get_digest(key2, c1->identity_digest);
 
+  if (with_ed) {
+    const tor_x509_cert_t *linkc, *idc;
+    tor_tls_get_my_certs(1, &linkc, &idc);
+    mock_peer_cert_expect_tortls = c1->tls; /* We should see this tls... */
+    mock_peer_cert = tor_x509_cert_dup(linkc); /* and when we do, the peer's
+                                                *  cert is this... */
+  }
   channel_tls_process_certs_cell(cell2, chan1);
+  mock_peer_cert_expect_tortls = NULL;
+  tor_x509_cert_free(mock_peer_cert);
+  mock_peer_cert = NULL;
+
+  tor_assert(c1->handshake_state->authenticated);
 
   tt_assert(c1->handshake_state->received_certs_cell);
-  tt_assert(c1->handshake_state->auth_cert == NULL);
-  tt_assert(c1->handshake_state->id_cert);
+  tt_assert(c1->handshake_state->certs->auth_cert == NULL);
+  tt_assert(c1->handshake_state->certs->ed_sign_auth == NULL);
+  tt_assert(c1->handshake_state->certs->id_cert);
+  if (with_ed) {
+    tt_assert(c1->handshake_state->certs->ed_sign_link);
+    tt_assert(c1->handshake_state->certs->ed_rsa_crosscert);
+    tt_assert(c1->handshake_state->certs->ed_id_sign);
+    tt_assert(c1->handshake_state->authenticated_rsa);
+    tt_assert(c1->handshake_state->authenticated_ed25519);
+  } else {
+    tt_assert(c1->handshake_state->certs->ed_sign_link == NULL);
+    tt_assert(c1->handshake_state->certs->ed_rsa_crosscert == NULL);
+    tt_assert(c1->handshake_state->certs->ed_id_sign == NULL);
+    tt_assert(c1->handshake_state->authenticated_rsa);
+    tt_assert(! c1->handshake_state->authenticated_ed25519);
+  }
   tt_assert(! tor_mem_is_zero(
-                  (char*)c1->handshake_state->authenticated_peer_id, 20));
+                (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
   channel_tls_common_init(chan2);
@@ -180,18 +272,34 @@ test_link_handshake_certs_ok(void *arg)
   channel_tls_process_certs_cell(cell1, chan2);
 
   tt_assert(c2->handshake_state->received_certs_cell);
-  tt_assert(c2->handshake_state->auth_cert);
-  tt_assert(c2->handshake_state->id_cert);
+  if (with_ed) {
+    tt_assert(c2->handshake_state->certs->ed_sign_auth);
+    tt_assert(c2->handshake_state->certs->ed_rsa_crosscert);
+    tt_assert(c2->handshake_state->certs->ed_id_sign);
+  } else {
+    tt_assert(c2->handshake_state->certs->auth_cert);
+    tt_assert(c2->handshake_state->certs->ed_sign_auth == NULL);
+    tt_assert(c2->handshake_state->certs->ed_rsa_crosscert == NULL);
+    tt_assert(c2->handshake_state->certs->ed_id_sign == NULL);
+  }
+  tt_assert(c2->handshake_state->certs->id_cert);
   tt_assert(tor_mem_is_zero(
-                (char*)c2->handshake_state->authenticated_peer_id, 20));
+              (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
+  /* no authentication has happened yet, since we haen't gotten an AUTH cell.
+   */
+  tt_assert(! c2->handshake_state->authenticated);
+  tt_assert(! c2->handshake_state->authenticated_rsa);
+  tt_assert(! c2->handshake_state->authenticated_ed25519);
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
   UNMOCK(connection_or_write_var_cell_to_buf);
   UNMOCK(connection_or_send_netinfo);
+  UNMOCK(tor_tls_get_peer_cert);
   UNMOCK(tor_tls_get_own_cert);
   tor_x509_cert_free(mock_own_cert);
-  mock_own_cert = NULL;
+  tor_x509_cert_free(mock_peer_cert);
+  mock_own_cert = mock_peer_cert = NULL;
   memset(c1->identity_digest, 0, sizeof(c1->identity_digest));
   memset(c2->identity_digest, 0, sizeof(c2->identity_digest));
   connection_free_(TO_CONN(c1));
@@ -211,6 +319,8 @@ test_link_handshake_certs_ok(void *arg)
 }
 
 typedef struct certs_data_s {
+  int is_ed;
+  int is_link_cert;
   or_connection_t *c;
   channel_tls_t *chan;
   certs_cell_t *ccell;
@@ -226,11 +336,13 @@ recv_certs_cleanup(const struct testcase_t *test, void *obj)
   UNMOCK(tor_tls_cert_matches_key);
   UNMOCK(connection_or_send_netinfo);
   UNMOCK(connection_or_close_for_error);
+  UNMOCK(tor_tls_get_peer_cert);
+  UNMOCK(tor_tls_get_own_cert);
 
   if (d) {
     tor_free(d->cell);
     certs_cell_free(d->ccell);
-    connection_or_remove_from_identity_map(d->c);
+    connection_or_clear_identity(d->c);
     connection_free_(TO_CONN(d->c));
     circuitmux_free(d->chan->base_.cmux);
     tor_free(d->chan);
@@ -238,6 +350,7 @@ recv_certs_cleanup(const struct testcase_t *test, void *obj)
     crypto_pk_free(d->key2);
     tor_free(d);
   }
+  routerkeys_free_all();
   return 1;
 }
 
@@ -249,11 +362,18 @@ recv_certs_setup(const struct testcase_t *test)
   certs_cell_cert_t *ccc1 = NULL;
   certs_cell_cert_t *ccc2 = NULL;
   ssize_t n;
+  int is_ed = d->is_ed = !strcmpstart(test->setup_data, "Ed25519");
+  int is_rsa = !strcmpstart(test->setup_data, "RSA");
+  int is_link = d->is_link_cert = !strcmpend(test->setup_data, "-Link");
+  int is_auth = !strcmpend(test->setup_data, "-Auth");
+  tor_assert(is_ed != is_rsa);
+  tor_assert(is_link != is_auth);
 
   d->c = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->chan = tor_malloc_zero(sizeof(*d->chan));
   d->c->chan = d->chan;
   d->c->base_.address = tor_strdup("HaveAnAddress");
+  tor_addr_from_ipv4h(&d->c->base_.addr, 0x801f0127);
   d->c->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->chan->conn = d->c;
   tt_int_op(connection_init_or_handshake_state(d->c, 1), ==, 0);
@@ -264,19 +384,25 @@ recv_certs_setup(const struct testcase_t *test)
 
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
                                  d->key1, d->key2, 86400), ==, 0);
+  if (is_ed) {
+    init_mock_ed_keys(d->key2);
+  } else {
+    routerkeys_free_all();
+  }
+
   d->ccell = certs_cell_new();
   ccc1 = certs_cell_cert_new();
   certs_cell_add_certs(d->ccell, ccc1);
   ccc2 = certs_cell_cert_new();
   certs_cell_add_certs(d->ccell, ccc2);
   d->ccell->n_certs = 2;
-  ccc1->cert_type = 1;
+  ccc1->cert_type = is_link ? 1 : 3;
   ccc2->cert_type = 2;
 
   const tor_x509_cert_t *a,*b;
   const uint8_t *enca, *encb;
   size_t lena, lenb;
-  tor_tls_get_my_certs(1, &a, &b);
+  tor_tls_get_my_certs(is_link ? 1 : 0, &a, &b);
   tor_x509_cert_get_der(a, &enca, &lena);
   tor_x509_cert_get_der(b, &encb, &lenb);
   certs_cell_cert_setlen_body(ccc1, lena);
@@ -287,6 +413,41 @@ recv_certs_setup(const struct testcase_t *test)
   memcpy(certs_cell_cert_getarray_body(ccc1), enca, lena);
   memcpy(certs_cell_cert_getarray_body(ccc2), encb, lenb);
 
+  if (is_ed) {
+    certs_cell_cert_t *ccc3 = NULL; /* Id->Sign */
+    certs_cell_cert_t *ccc4 = NULL; /* Sign->Link or Sign->Auth. */
+    certs_cell_cert_t *ccc5 = NULL; /* RSAId->Ed Id. */
+    const tor_cert_t *id_sign = get_master_signing_key_cert();
+    const tor_cert_t *secondary =
+      is_link ? get_current_link_cert_cert() : get_current_auth_key_cert();
+    const uint8_t *cc = NULL;
+    size_t cc_sz;
+    get_master_rsa_crosscert(&cc, &cc_sz);
+
+    ccc3 = certs_cell_cert_new();
+    ccc4 = certs_cell_cert_new();
+    ccc5 = certs_cell_cert_new();
+    certs_cell_add_certs(d->ccell, ccc3);
+    certs_cell_add_certs(d->ccell, ccc4);
+    certs_cell_add_certs(d->ccell, ccc5);
+    ccc3->cert_len = id_sign->encoded_len;
+    ccc4->cert_len = secondary->encoded_len;
+    ccc5->cert_len = cc_sz;
+    certs_cell_cert_setlen_body(ccc3, ccc3->cert_len);
+    certs_cell_cert_setlen_body(ccc4, ccc4->cert_len);
+    certs_cell_cert_setlen_body(ccc5, ccc5->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc3), id_sign->encoded,
+           ccc3->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc4), secondary->encoded,
+           ccc4->cert_len);
+    memcpy(certs_cell_cert_getarray_body(ccc5), cc, ccc5->cert_len);
+    ccc3->cert_type = 4;
+    ccc4->cert_type = is_link ? 5 : 6;
+    ccc5->cert_type = 7;
+
+    d->ccell->n_certs = 5;
+  }
+
   d->cell = var_cell_new(4096);
   d->cell->command = CELL_CERTS;
 
@@ -297,6 +458,12 @@ recv_certs_setup(const struct testcase_t *test)
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
   MOCK(connection_or_close_for_error, mock_close_for_err);
+  MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);
+
+  if (is_link) {
+    /* Say that this is the peer's certificate */
+    mock_peer_cert = tor_x509_cert_dup(a);
+  }
 
   tt_int_op(0, ==, d->c->handshake_state->received_certs_cell);
   tt_int_op(0, ==, mock_send_authenticate_called);
@@ -320,9 +487,24 @@ test_link_handshake_recv_certs_ok(void *arg)
   channel_tls_process_certs_cell(d->cell, d->chan);
   tt_int_op(0, ==, mock_close_called);
   tt_int_op(d->c->handshake_state->authenticated, ==, 1);
+  tt_int_op(d->c->handshake_state->authenticated_rsa, ==, 1);
   tt_int_op(d->c->handshake_state->received_certs_cell, ==, 1);
-  tt_assert(d->c->handshake_state->id_cert != NULL);
-  tt_assert(d->c->handshake_state->auth_cert == NULL);
+  tt_assert(d->c->handshake_state->certs->id_cert != NULL);
+  tt_assert(d->c->handshake_state->certs->auth_cert == NULL);
+
+  if (d->is_ed) {
+    tt_assert(d->c->handshake_state->certs->ed_id_sign != NULL);
+    tt_assert(d->c->handshake_state->certs->ed_sign_link != NULL);
+    tt_assert(d->c->handshake_state->certs->ed_sign_auth == NULL);
+    tt_assert(d->c->handshake_state->certs->ed_rsa_crosscert != NULL);
+    tt_int_op(d->c->handshake_state->authenticated_ed25519, ==, 1);
+  } else {
+    tt_assert(d->c->handshake_state->certs->ed_id_sign == NULL);
+    tt_assert(d->c->handshake_state->certs->ed_sign_link == NULL);
+    tt_assert(d->c->handshake_state->certs->ed_sign_auth == NULL);
+    tt_assert(d->c->handshake_state->certs->ed_rsa_crosscert == NULL);
+    tt_int_op(d->c->handshake_state->authenticated_ed25519, ==, 0);
+  }
 
  done:
   ;
@@ -333,17 +515,20 @@ test_link_handshake_recv_certs_ok_server(void *arg)
 {
   certs_data_t *d = arg;
   d->c->handshake_state->started_here = 0;
-  certs_cell_get_certs(d->ccell, 0)->cert_type = 3;
-  certs_cell_get_certs(d->ccell, 1)->cert_type = 2;
-  ssize_t n = certs_cell_encode(d->cell->payload, 2048, d->ccell);
-  tt_int_op(n, >, 0);
-  d->cell->payload_len = n;
+  d->c->handshake_state->certs->started_here = 0;
   channel_tls_process_certs_cell(d->cell, d->chan);
   tt_int_op(0, ==, mock_close_called);
   tt_int_op(d->c->handshake_state->authenticated, ==, 0);
   tt_int_op(d->c->handshake_state->received_certs_cell, ==, 1);
-  tt_assert(d->c->handshake_state->id_cert != NULL);
-  tt_assert(d->c->handshake_state->auth_cert != NULL);
+  tt_assert(d->c->handshake_state->certs->id_cert != NULL);
+  tt_assert(d->c->handshake_state->certs->link_cert == NULL);
+  if (d->is_ed) {
+    tt_assert(d->c->handshake_state->certs->ed_sign_auth != NULL);
+    tt_assert(d->c->handshake_state->certs->auth_cert == NULL);
+  } else {
+    tt_assert(d->c->handshake_state->certs->ed_sign_auth == NULL);
+    tt_assert(d->c->handshake_state->certs->auth_cert != NULL);
+  }
 
  done:
   ;
@@ -361,6 +546,8 @@ test_link_handshake_recv_certs_ok_server(void *arg)
     tt_int_op(1, ==, mock_close_called);                                \
     tt_int_op(0, ==, mock_send_authenticate_called);                    \
     tt_int_op(0, ==, mock_send_netinfo_called);                         \
+    tt_int_op(0, ==, d->c->handshake_state->authenticated_rsa);         \
+    tt_int_op(0, ==, d->c->handshake_state->authenticated_ed25519);     \
     if (require_failure_message) {                                      \
       expect_log_msg_containing(require_failure_message);               \
     }                                                                   \
@@ -401,12 +588,41 @@ CERTS_FAIL(truncated_3,
              d->cell->payload_len = 7;
              memcpy(d->cell->payload, "\x01\x01\x00\x05""abc", 7);
            })
+CERTS_FAIL(truncated_4, /* ed25519 */
+           {
+             require_failure_message = "It couldn't be parsed";
+             d->cell->payload_len -= 10;
+           })
+CERTS_FAIL(truncated_5, /* ed25519 */
+           {
+             require_failure_message = "It couldn't be parsed";
+             d->cell->payload_len -= 100;
+           })
+
 #define REENCODE() do {                                                 \
+    const char *msg = certs_cell_check(d->ccell);                       \
+    if (msg) puts(msg);                                                 \
     ssize_t n = certs_cell_encode(d->cell->payload, 4096, d->ccell);    \
     tt_int_op(n, >, 0);                                                 \
     d->cell->payload_len = n;                                           \
   } while (0)
 
+CERTS_FAIL(truncated_6, /* ed25519 */
+   {
+     /* truncate the link certificate */
+     require_failure_message = "undecodable Ed certificate";
+     certs_cell_cert_setlen_body(certs_cell_get_certs(d->ccell, 3), 7);
+     certs_cell_get_certs(d->ccell, 3)->cert_len = 7;
+     REENCODE();
+   })
+CERTS_FAIL(truncated_7, /* ed25519 */
+   {
+     /* truncate the crosscert */
+     require_failure_message = "Unparseable or overlong crosscert";
+     certs_cell_cert_setlen_body(certs_cell_get_certs(d->ccell, 4), 7);
+     certs_cell_get_certs(d->ccell, 4)->cert_len = 7;
+     REENCODE();
+   })
 CERTS_FAIL(not_x509,
   {
     require_failure_message = "Received undecodable certificate";
@@ -435,6 +651,201 @@ CERTS_FAIL(both_auth,
     certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
     REENCODE();
   })
+CERTS_FAIL(duplicate_id, /* ed25519 */
+  {
+    require_failure_message = "Duplicate Ed25519 certificate";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 4;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 4;
+    REENCODE();
+  })
+CERTS_FAIL(duplicate_link, /* ed25519 */
+  {
+    require_failure_message = "Duplicate Ed25519 certificate";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 5;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 5;
+    REENCODE();
+  })
+CERTS_FAIL(duplicate_crosscert, /* ed25519 */
+  {
+    require_failure_message = "Duplicate RSA->Ed25519 crosscert";
+    certs_cell_get_certs(d->ccell, 2)->cert_type = 7;
+    certs_cell_get_certs(d->ccell, 3)->cert_type = 7;
+    REENCODE();
+  })
+static void
+test_link_handshake_recv_certs_missing_id(void *arg) /* ed25519 */
+{
+  certs_data_t *d = arg;
+  tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+  certs_cell_set_certs(d->ccell, 2, certs_cell_get_certs(d->ccell, 4));
+  certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+  certs_cell_setlen_certs(d->ccell, 4);
+  d->ccell->n_certs = 4;
+  REENCODE();
+
+  /* This handshake succeeds, but since we have no ID cert, we will
+   * just do the RSA handshake. */
+  channel_tls_process_certs_cell(d->cell, d->chan);
+  tt_int_op(0, ==, mock_close_called);
+  tt_int_op(0, ==, d->c->handshake_state->authenticated_ed25519);
+  tt_int_op(1, ==, d->c->handshake_state->authenticated_rsa);
+ done:
+  ;
+}
+CERTS_FAIL(missing_signing_key, /* ed25519 */
+  {
+    require_failure_message = "No Ed25519 signing key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 2);
+    tt_int_op(cert->cert_type, ==, CERTTYPE_ED_ID_SIGN);
+    /* replace this with a valid master->signing cert, but with no
+     * signing key. */
+    const ed25519_keypair_t *mk = get_master_identity_keypair();
+    const ed25519_keypair_t *sk = get_master_signing_keypair();
+    tor_cert_t *bad_cert = tor_cert_create(mk, CERT_TYPE_ID_SIGNING,
+                                           &sk->pubkey, time(NULL), 86400,
+                                           0 /* don't include signer */);
+    certs_cell_cert_setlen_body(cert, bad_cert->encoded_len);
+    memcpy(certs_cell_cert_getarray_body(cert),
+           bad_cert->encoded, bad_cert->encoded_len);
+    cert->cert_len = bad_cert->encoded_len;
+    tor_cert_free(bad_cert);
+    REENCODE();
+  })
+CERTS_FAIL(missing_link, /* ed25519 */
+  {
+    require_failure_message = "No Ed25519 link key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 3, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_auth, /* ed25519 */
+  {
+    d->c->handshake_state->started_here = 0;
+    d->c->handshake_state->certs->started_here = 0;
+    require_failure_message = "No Ed25519 link authentication key";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 3, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_crosscert, /* ed25519 */
+  {
+    require_failure_message = "Missing RSA->Ed25519 crosscert";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(missing_rsa_id, /* ed25519 */
+  {
+    require_failure_message = "Missing legacy RSA ID cert";
+    tt_int_op(certs_cell_getlen_certs(d->ccell), OP_EQ, 5);
+    certs_cell_set_certs(d->ccell, 1, certs_cell_get_certs(d->ccell, 4));
+    certs_cell_set0_certs(d->ccell, 4, NULL); /* prevent free */
+    certs_cell_setlen_certs(d->ccell, 4);
+    d->ccell->n_certs = 4;
+    REENCODE();
+  })
+CERTS_FAIL(link_mismatch, /* ed25519 */
+  {
+    require_failure_message = "Link certificate does not match "
+      "TLS certificate";
+    const tor_x509_cert_t *idc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    tor_x509_cert_free(mock_peer_cert);
+    /* Pretend that the peer cert was something else. */
+    mock_peer_cert = tor_x509_cert_dup(idc);
+    /* No reencode needed. */
+  })
+CERTS_FAIL(bad_ed_sig, /* ed25519 */
+  {
+    require_failure_message = "At least one Ed25519 certificate was "
+      "badly signed";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 3);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(bad_crosscert, /*ed25519*/
+  {
+    require_failure_message = "Invalid RSA->Ed25519 crosscert";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 4);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(bad_rsa_id_cert, /*ed25519*/
+  {
+    require_failure_message = "legacy RSA ID certificate was not valid";
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 1);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    ssize_t body_len = certs_cell_cert_getlen_body(cert);
+    /* Frob a byte in the signature */
+    body[body_len - 13] ^= 7;
+    REENCODE();
+  })
+CERTS_FAIL(expired_rsa_id, /* both */
+  {
+    require_failure_message = "Certificate already expired";
+    /* we're going to replace the identity cert with an expired one. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 1);
+    const tor_x509_cert_t *idc;
+    tor_tls_get_my_certs(1, NULL, &idc);
+    tor_x509_cert_t *newc;
+    time_t new_end = time(NULL) - 86400 * 10;
+    newc = tor_x509_cert_replace_expiration(idc, new_end, d->key2);
+    certs_cell_cert_setlen_body(cert, newc->encoded_len);
+    memcpy(certs_cell_cert_getarray_body(cert),
+           newc->encoded, newc->encoded_len);
+    REENCODE();
+    tor_x509_cert_free(newc);
+  })
+CERTS_FAIL(expired_ed_id, /* ed25519 */
+  {
+    /* we're going to replace the Ed Id->sign cert with an expired one. */
+    require_failure_message = "At least one certificate expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 2);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [2..5].  It is in HOURS since the
+     * epoch. */
+    set_uint32(body+2, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+CERTS_FAIL(expired_ed_link, /* ed25519 */
+  {
+    /* we're going to replace the Ed Sign->link cert with an expired one. */
+    require_failure_message = "At least one certificate expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 3);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [2..5].  It is in HOURS since the
+     * epoch. */
+    set_uint32(body+2, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+CERTS_FAIL(expired_crosscert, /* ed25519 */
+  {
+    /* we're going to replace the Ed Sign->link cert with an expired one. */
+    require_failure_message = "Crosscert is expired";
+    /* We don't need to re-sign, since we check for expiration first. */
+    certs_cell_cert_t *cert = certs_cell_get_certs(d->ccell, 4);
+    uint8_t *body = certs_cell_cert_getarray_body(cert);
+    /* The expiration field is bytes [32..35]. once again, HOURS. */
+    set_uint32(body+32, htonl(24)); /* Back to jan 2, 1970. */
+    REENCODE();
+  })
+
 CERTS_FAIL(wrong_labels_1,
   {
     require_failure_message = "The link certificate was not valid";
@@ -459,21 +870,26 @@ CERTS_FAIL(wrong_labels_2,
   })
 CERTS_FAIL(wrong_labels_3,
            {
-             require_failure_message = "The certs we wanted were missing";
+             require_failure_message =
+               "The certs we wanted (ID, Link) were missing";
              certs_cell_get_certs(d->ccell, 0)->cert_type = 2;
              certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
              REENCODE();
            })
 CERTS_FAIL(server_missing_certs,
            {
-             require_failure_message = "The certs we wanted were missing";
+             require_failure_message =
+               "The certs we wanted (ID, Auth) were missing";
              d->c->handshake_state->started_here = 0;
+             d->c->handshake_state->certs->started_here = 0;
+
            })
 CERTS_FAIL(server_wrong_labels_1,
            {
              require_failure_message =
                "The authentication certificate was not valid";
              d->c->handshake_state->started_here = 0;
+             d->c->handshake_state->certs->started_here = 0;
              certs_cell_get_certs(d->ccell, 0)->cert_type = 2;
              certs_cell_get_certs(d->ccell, 1)->cert_type = 3;
              REENCODE();
@@ -487,6 +903,11 @@ test_link_handshake_send_authchallenge(void *arg)
   or_connection_t *c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   var_cell_t *cell1=NULL, *cell2=NULL;
 
+  crypto_pk_t *rsa0 = pk_generate(0), *rsa1 = pk_generate(1);
+  tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
+                                 rsa0, rsa1, 86400), ==, 0);
+  init_mock_ed_keys(rsa0);
+
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
 
   tt_int_op(connection_init_or_handshake_state(c1, 0), ==, 0);
@@ -496,15 +917,15 @@ test_link_handshake_send_authchallenge(void *arg)
   cell1 = mock_got_var_cell;
   tt_int_op(0, ==, connection_or_send_auth_challenge_cell(c1));
   cell2 = mock_got_var_cell;
-  tt_int_op(36, ==, cell1->payload_len);
-  tt_int_op(36, ==, cell2->payload_len);
+  tt_int_op(38, ==, cell1->payload_len);
+  tt_int_op(38, ==, cell2->payload_len);
   tt_int_op(0, ==, cell1->circ_id);
   tt_int_op(0, ==, cell2->circ_id);
   tt_int_op(CELL_AUTH_CHALLENGE, ==, cell1->command);
   tt_int_op(CELL_AUTH_CHALLENGE, ==, cell2->command);
 
-  tt_mem_op("\x00\x01\x00\x01", ==, cell1->payload + 32, 4);
-  tt_mem_op("\x00\x01\x00\x01", ==, cell2->payload + 32, 4);
+  tt_mem_op("\x00\x02\x00\x01\x00\x03", ==, cell1->payload + 32, 6);
+  tt_mem_op("\x00\x02\x00\x01\x00\x03", ==, cell2->payload + 32, 6);
   tt_mem_op(cell1->payload, !=, cell2->payload, 32);
 
  done:
@@ -512,6 +933,8 @@ test_link_handshake_send_authchallenge(void *arg)
   connection_free_(TO_CONN(c1));
   tor_free(cell1);
   tor_free(cell2);
+  crypto_pk_free(rsa0);
+  crypto_pk_free(rsa1);
 }
 
 typedef struct authchallenge_data_s {
@@ -556,9 +979,9 @@ recv_authchallenge_setup(const struct testcase_t *test)
   d->c->handshake_state->received_certs_cell = 1;
   d->cell = var_cell_new(128);
   d->cell->payload_len = 38;
-  d->cell->payload[33] = 2;
-  d->cell->payload[35] = 7;
-  d->cell->payload[37] = 1;
+  d->cell->payload[33] = 2; /* 2 methods */
+  d->cell->payload[35] = 7; /* This one isn't real */
+  d->cell->payload[37] = 1; /* This is the old RSA one. */
   d->cell->command = CELL_AUTH_CHALLENGE;
 
   get_options_mutable()->ORPort_set = 1;
@@ -566,7 +989,6 @@ recv_authchallenge_setup(const struct testcase_t *test)
   MOCK(connection_or_close_for_error, mock_close_for_err);
   MOCK(connection_or_send_netinfo, mock_send_netinfo);
   MOCK(connection_or_send_authenticate_cell, mock_send_authenticate);
-
   tt_int_op(0, ==, d->c->handshake_state->received_auth_challenge);
   tt_int_op(0, ==, mock_send_authenticate_called);
   tt_int_op(0, ==, mock_send_netinfo_called);
@@ -592,6 +1014,26 @@ test_link_handshake_recv_authchallenge_ok(void *arg)
   tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
   tt_int_op(1, ==, mock_send_authenticate_called);
   tt_int_op(1, ==, mock_send_netinfo_called);
+  tt_int_op(1, ==, mock_send_authenticate_called_with_type); /* RSA */
+ done:
+  ;
+}
+
+static void
+test_link_handshake_recv_authchallenge_ok_ed25519(void *arg)
+{
+  authchallenge_data_t *d = arg;
+
+  /* Add the ed25519 authentication mechanism here. */
+  d->cell->payload[33] = 3; /* 3 types are supported now. */
+  d->cell->payload[39] = 3;
+  d->cell->payload_len += 2;
+  channel_tls_process_auth_challenge_cell(d->cell, d->chan);
+  tt_int_op(0, ==, mock_close_called);
+  tt_int_op(1, ==, d->c->handshake_state->received_auth_challenge);
+  tt_int_op(1, ==, mock_send_authenticate_called);
+  tt_int_op(1, ==, mock_send_netinfo_called);
+  tt_int_op(3, ==, mock_send_authenticate_called_with_type); /* Ed25519 */
  done:
   ;
 }
@@ -655,7 +1097,8 @@ AUTHCHALLENGE_FAIL(badproto,
 AUTHCHALLENGE_FAIL(as_server,
                    require_failure_message = "We didn't originate this "
                      "connection";
-                   d->c->handshake_state->started_here = 0;)
+                   d->c->handshake_state->started_here = 0;
+                   d->c->handshake_state->certs->started_here = 0;)
 AUTHCHALLENGE_FAIL(duplicate,
                    require_failure_message = "We already received one";
                    d->c->handshake_state->received_auth_challenge = 1)
@@ -673,15 +1116,6 @@ AUTHCHALLENGE_FAIL(nonzero_circid,
                    require_failure_message = "It had a nonzero circuit ID";
                    d->cell->circ_id = 1337)
 
-static tor_x509_cert_t *mock_peer_cert = NULL;
-
-static tor_x509_cert_t *
-mock_get_peer_cert(tor_tls_t *tls)
-{
-  (void)tls;
-  return tor_x509_cert_dup(mock_peer_cert);
-}
-
 static int
 mock_get_tlssecrets(tor_tls_t *tls, uint8_t *secrets_out)
 {
@@ -701,6 +1135,7 @@ mock_set_circid_type(channel_t *chan,
 }
 
 typedef struct authenticate_data_s {
+  int is_ed;
   or_connection_t *c1, *c2;
   channel_tls_t *chan2;
   var_cell_t *cell;
@@ -717,11 +1152,12 @@ authenticate_data_cleanup(const struct testcase_t *test, void *arg)
   UNMOCK(tor_tls_get_tlssecrets);
   UNMOCK(connection_or_close_for_error);
   UNMOCK(channel_set_circid_type);
+  UNMOCK(tor_tls_export_key_material);
   authenticate_data_t *d = arg;
   if (d) {
     tor_free(d->cell);
-    connection_or_remove_from_identity_map(d->c1);
-    connection_or_remove_from_identity_map(d->c2);
+    connection_or_clear_identity(d->c1);
+    connection_or_clear_identity(d->c2);
     connection_free_(TO_CONN(d->c1));
     connection_free_(TO_CONN(d->c2));
     circuitmux_free(d->chan2->base_.cmux);
@@ -742,6 +1178,7 @@ static void *
 authenticate_data_setup(const struct testcase_t *test)
 {
   authenticate_data_t *d = tor_malloc_zero(sizeof(*d));
+  int is_ed = d->is_ed = (test->setup_data == (void*)3);
 
   scheduler_init();
 
@@ -751,6 +1188,7 @@ authenticate_data_setup(const struct testcase_t *test)
   MOCK(tor_tls_get_tlssecrets, mock_get_tlssecrets);
   MOCK(connection_or_close_for_error, mock_close_for_err);
   MOCK(channel_set_circid_type, mock_set_circid_type);
+  MOCK(tor_tls_export_key_material, mock_export_key_material);
   d->c1 = or_connection_new(CONN_TYPE_OR, AF_INET);
   d->c2 = or_connection_new(CONN_TYPE_OR, AF_INET);
   tor_addr_from_ipv4h(&d->c1->base_.addr, 0x01020304);
@@ -761,6 +1199,8 @@ authenticate_data_setup(const struct testcase_t *test)
   tt_int_op(tor_tls_context_init(TOR_TLS_CTX_IS_PUBLIC_SERVER,
                                  d->key1, d->key2, 86400), ==, 0);
 
+  init_mock_ed_keys(d->key2);
+
   d->c1->base_.state = OR_CONN_STATE_OR_HANDSHAKING_V3;
   d->c1->link_proto = 3;
   tt_int_op(connection_init_or_handshake_state(d->c1, 1), ==, 0);
@@ -791,21 +1231,37 @@ authenticate_data_setup(const struct testcase_t *test)
   const uint8_t *der;
   size_t sz;
   tor_x509_cert_get_der(id_cert, &der, &sz);
-  d->c1->handshake_state->id_cert = tor_x509_cert_decode(der, sz);
-  d->c2->handshake_state->id_cert = tor_x509_cert_decode(der, sz);
+  d->c1->handshake_state->certs->id_cert = tor_x509_cert_decode(der, sz);
+  d->c2->handshake_state->certs->id_cert = tor_x509_cert_decode(der, sz);
+
+  if (is_ed) {
+    d->c1->handshake_state->certs->ed_id_sign =
+      tor_cert_dup(get_master_signing_key_cert());
+    d->c2->handshake_state->certs->ed_id_sign =
+      tor_cert_dup(get_master_signing_key_cert());
+    d->c2->handshake_state->certs->ed_sign_auth =
+      tor_cert_dup(get_current_auth_key_cert());
+  } else {
+    tt_assert(! tor_tls_get_my_certs(0, &auth_cert, &id_cert));
+    tor_x509_cert_get_der(auth_cert, &der, &sz);
+    d->c2->handshake_state->certs->auth_cert = tor_x509_cert_decode(der, sz);
+  }
 
   tor_x509_cert_get_der(link_cert, &der, &sz);
   mock_peer_cert = tor_x509_cert_decode(der, sz);
   tt_assert(mock_peer_cert);
+
   mock_own_cert = tor_x509_cert_decode(der, sz);
   tt_assert(mock_own_cert);
-  tt_assert(! tor_tls_get_my_certs(0, &auth_cert, &id_cert));
-  tor_x509_cert_get_der(auth_cert, &der, &sz);
-  d->c2->handshake_state->auth_cert = tor_x509_cert_decode(der, sz);
 
   /* Make an authenticate cell ... */
-  tt_int_op(0, ==, connection_or_send_authenticate_cell(d->c1,
-                                             AUTHTYPE_RSA_SHA256_TLSSECRET));
+  int authtype;
+  if (is_ed)
+    authtype = AUTHTYPE_ED25519_SHA256_RFC5705;
+  else
+    authtype = AUTHTYPE_RSA_SHA256_TLSSECRET;
+  tt_int_op(0, ==, connection_or_send_authenticate_cell(d->c1, authtype));
+
   tt_assert(mock_got_var_cell);
   d->cell = mock_got_var_cell;
   mock_got_var_cell = NULL;
@@ -831,42 +1287,64 @@ test_link_handshake_auth_cell(void *arg)
   /* Is the cell well-formed on the outer layer? */
   tt_int_op(d->cell->command, ==, CELL_AUTHENTICATE);
   tt_int_op(d->cell->payload[0], ==, 0);
-  tt_int_op(d->cell->payload[1], ==, 1);
+  if (d->is_ed)
+    tt_int_op(d->cell->payload[1], ==, 3);
+  else
+    tt_int_op(d->cell->payload[1], ==, 1);
   tt_int_op(ntohs(get_uint16(d->cell->payload + 2)), ==,
             d->cell->payload_len - 4);
 
   /* Check it out for plausibility... */
   auth_ctx_t ctx;
-  ctx.is_ed = 0;
+  ctx.is_ed = d->is_ed;
   tt_int_op(d->cell->payload_len-4, ==, auth1_parse(&auth1,
                                              d->cell->payload+4,
                                              d->cell->payload_len - 4, &ctx));
   tt_assert(auth1);
 
-  tt_mem_op(auth1->type, ==, "AUTH0001", 8);
+  if (d->is_ed) {
+    tt_mem_op(auth1->type, ==, "AUTH0003", 8);
+  } else {
+    tt_mem_op(auth1->type, ==, "AUTH0001", 8);
+  }
   tt_mem_op(auth1->tlssecrets, ==, "int getRandomNumber(){return 4;}", 32);
-  tt_int_op(auth1_getlen_sig(auth1), >, 120);
 
   /* Is the signature okay? */
-  uint8_t sig[128];
-  uint8_t digest[32];
-
-  auth_pubkey = tor_tls_cert_get_key(d->c2->handshake_state->auth_cert);
-  int n = crypto_pk_public_checksig(
+  const uint8_t *start = d->cell->payload+4, *end = auth1->end_of_signed;
+  if (d->is_ed) {
+    ed25519_signature_t sig;
+    tt_int_op(auth1_getlen_sig(auth1), ==, ED25519_SIG_LEN);
+    memcpy(&sig.sig, auth1_getarray_sig(auth1), ED25519_SIG_LEN);
+    tt_assert(!ed25519_checksig(&sig, start, end-start,
+                                &get_current_auth_keypair()->pubkey));
+  } else {
+    uint8_t sig[128];
+    uint8_t digest[32];
+    tt_int_op(auth1_getlen_sig(auth1), >, 120);
+    auth_pubkey = tor_tls_cert_get_key(
+                                d->c2->handshake_state->certs->auth_cert);
+    int n = crypto_pk_public_checksig(
               auth_pubkey,
               (char*)sig, sizeof(sig), (char*)auth1_getarray_sig(auth1),
               auth1_getlen_sig(auth1));
-  tt_int_op(n, ==, 32);
-  const uint8_t *start = d->cell->payload+4, *end = auth1->end_of_signed;
-  crypto_digest256((char*)digest,
-                   (const char*)start, end-start, DIGEST_SHA256);
-  tt_mem_op(sig, ==, digest, 32);
+    tt_int_op(n, ==, 32);
+    crypto_digest256((char*)digest,
+                     (const char*)start, end-start, DIGEST_SHA256);
+    tt_mem_op(sig, ==, digest, 32);
+  }
 
   /* Then feed it to c2. */
   tt_int_op(d->c2->handshake_state->authenticated, ==, 0);
   channel_tls_process_authenticate_cell(d->cell, d->chan2);
   tt_int_op(mock_close_called, ==, 0);
   tt_int_op(d->c2->handshake_state->authenticated, ==, 1);
+  if (d->is_ed) {
+    tt_int_op(d->c2->handshake_state->authenticated_ed25519, ==, 1);
+    tt_int_op(d->c2->handshake_state->authenticated_rsa, ==, 1);
+  } else {
+    tt_int_op(d->c2->handshake_state->authenticated_ed25519, ==, 0);
+    tt_int_op(d->c2->handshake_state->authenticated_rsa, ==, 1);
+  }
 
  done:
   auth1_free(auth1);
@@ -900,7 +1378,8 @@ AUTHENTICATE_FAIL(badproto,
                   d->c2->link_proto = 2)
 AUTHENTICATE_FAIL(atclient,
                   require_failure_message = "We originated this connection";
-                  d->c2->handshake_state->started_here = 1)
+                  d->c2->handshake_state->started_here = 1;
+                  d->c2->handshake_state->certs->started_here = 1;)
 AUTHENTICATE_FAIL(duplicate,
                   require_failure_message = "We already got one";
                   d->c2->handshake_state->received_authenticate = 1)
@@ -924,13 +1403,13 @@ AUTHENTICATE_FAIL(nocerts,
 AUTHENTICATE_FAIL(noidcert,
                   require_failure_message = "We never got an identity "
                     "certificate";
-                  tor_x509_cert_free(d->c2->handshake_state->id_cert);
-                  d->c2->handshake_state->id_cert = NULL)
+                  tor_x509_cert_free(d->c2->handshake_state->certs->id_cert);
+                  d->c2->handshake_state->certs->id_cert = NULL)
 AUTHENTICATE_FAIL(noauthcert,
-                  require_failure_message = "We never got an authentication "
-                    "certificate";
-                  tor_x509_cert_free(d->c2->handshake_state->auth_cert);
-                  d->c2->handshake_state->auth_cert = NULL)
+                  require_failure_message = "We never got an RSA "
+                    "authentication certificate";
+                  tor_x509_cert_free(d->c2->handshake_state->certs->auth_cert);
+                  d->c2->handshake_state->certs->auth_cert = NULL)
 AUTHENTICATE_FAIL(tooshort,
                   require_failure_message = "Cell was way too short";
                   d->cell->payload_len = 3)
@@ -954,11 +1433,33 @@ AUTHENTICATE_FAIL(badcontent,
                     "cell body was not as expected";
                   d->cell->payload[10] ^= 0xff)
 AUTHENTICATE_FAIL(badsig_1,
-                  require_failure_message = "Signature wasn't valid";
+                  if (d->is_ed)
+                    require_failure_message = "Ed25519 signature wasn't valid";
+                  else
+                    require_failure_message = "RSA signature wasn't valid";
                   d->cell->payload[d->cell->payload_len - 5] ^= 0xff)
-
-#define TEST(name, flags)                                       \
-  { #name , test_link_handshake_ ## name, (flags), NULL, NULL }
+AUTHENTICATE_FAIL(missing_ed_id,
+                {
+                  tor_cert_free(d->c2->handshake_state->certs->ed_id_sign);
+                  d->c2->handshake_state->certs->ed_id_sign = NULL;
+                  require_failure_message = "Ed authenticate without Ed ID "
+                    "cert from peer";
+                })
+AUTHENTICATE_FAIL(missing_ed_auth,
+                {
+                  tor_cert_free(d->c2->handshake_state->certs->ed_sign_auth);
+                  d->c2->handshake_state->certs->ed_sign_auth = NULL;
+                  require_failure_message = "We never got an Ed25519 "
+                    "authentication certificate";
+                })
+
+#define TEST_RSA(name, flags)                                           \
+  { #name , test_link_handshake_ ## name, (flags),                      \
+      &passthrough_setup, (void*)"RSA" }
+
+#define TEST_ED(name, flags)                                            \
+  { #name "_ed25519" , test_link_handshake_ ## name, (flags),           \
+      &passthrough_setup, (void*)"Ed25519" }
 
 #define TEST_RCV_AUTHCHALLENGE(name)                            \
   { "recv_authchallenge/" #name ,                               \
@@ -968,17 +1469,34 @@ AUTHENTICATE_FAIL(badsig_1,
 #define TEST_RCV_CERTS(name)                                    \
   { "recv_certs/" #name ,                                       \
       test_link_handshake_recv_certs_ ## name, TT_FORK,         \
-      &setup_recv_certs, NULL }
+      &setup_recv_certs, (void*)"RSA-Link" }
+
+#define TEST_RCV_CERTS_RSA(name,type)                           \
+  { "recv_certs/" #name ,                                       \
+      test_link_handshake_recv_certs_ ## name, TT_FORK,         \
+      &setup_recv_certs, (void*)type }
+
+#define TEST_RCV_CERTS_ED(name, type)                           \
+  { "recv_certs/" #name "_ed25519",                             \
+      test_link_handshake_recv_certs_ ## name, TT_FORK,         \
+      &setup_recv_certs, (void*)type }
 
 #define TEST_AUTHENTICATE(name)                                         \
   { "authenticate/" #name , test_link_handshake_auth_ ## name, TT_FORK, \
       &setup_authenticate, NULL }
 
+#define TEST_AUTHENTICATE_ED(name)                                      \
+  { "authenticate/" #name "_ed25519" , test_link_handshake_auth_ ## name, \
+      TT_FORK, &setup_authenticate, (void*)3 }
+
 struct testcase_t link_handshake_tests[] = {
-  TEST(certs_ok, TT_FORK),
-  //TEST(certs_bad, TT_FORK),
+  TEST_RSA(certs_ok, TT_FORK),
+  TEST_ED(certs_ok, TT_FORK),
+
   TEST_RCV_CERTS(ok),
-  TEST_RCV_CERTS(ok_server),
+  TEST_RCV_CERTS_ED(ok, "Ed25519-Link"),
+  TEST_RCV_CERTS_RSA(ok_server, "RSA-Auth"),
+  TEST_RCV_CERTS_ED(ok_server, "Ed25519-Auth"),
   TEST_RCV_CERTS(badstate),
   TEST_RCV_CERTS(badproto),
   TEST_RCV_CERTS(duplicate),
@@ -988,18 +1506,41 @@ struct testcase_t link_handshake_tests[] = {
   TEST_RCV_CERTS(truncated_1),
   TEST_RCV_CERTS(truncated_2),
   TEST_RCV_CERTS(truncated_3),
+  TEST_RCV_CERTS_ED(truncated_4, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_5, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_6, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(truncated_7, "Ed25519-Link"),
   TEST_RCV_CERTS(not_x509),
   TEST_RCV_CERTS(both_link),
   TEST_RCV_CERTS(both_id_rsa),
   TEST_RCV_CERTS(both_auth),
+  TEST_RCV_CERTS_ED(duplicate_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(duplicate_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(duplicate_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_signing_key, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(missing_auth, "Ed25519-Auth"),
+  TEST_RCV_CERTS_ED(missing_rsa_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(link_mismatch, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_ed_sig, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_rsa_id_cert, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(bad_crosscert, "Ed25519-Link"),
+  TEST_RCV_CERTS_RSA(expired_rsa_id, "RSA-Link"),
+  TEST_RCV_CERTS_ED(expired_rsa_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_ed_id, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_ed_link, "Ed25519-Link"),
+  TEST_RCV_CERTS_ED(expired_crosscert, "Ed25519-Link"),
   TEST_RCV_CERTS(wrong_labels_1),
   TEST_RCV_CERTS(wrong_labels_2),
   TEST_RCV_CERTS(wrong_labels_3),
   TEST_RCV_CERTS(server_missing_certs),
   TEST_RCV_CERTS(server_wrong_labels_1),
 
-  TEST(send_authchallenge, TT_FORK),
+  TEST_RSA(send_authchallenge, TT_FORK),
   TEST_RCV_AUTHCHALLENGE(ok),
+  TEST_RCV_AUTHCHALLENGE(ok_ed25519),
   TEST_RCV_AUTHCHALLENGE(ok_noserver),
   TEST_RCV_AUTHCHALLENGE(ok_unrecognized),
   TEST_RCV_AUTHCHALLENGE(badstate),
@@ -1012,6 +1553,7 @@ struct testcase_t link_handshake_tests[] = {
   TEST_RCV_AUTHCHALLENGE(nonzero_circid),
 
   TEST_AUTHENTICATE(cell),
+  TEST_AUTHENTICATE_ED(cell),
   TEST_AUTHENTICATE(badstate),
   TEST_AUTHENTICATE(badproto),
   TEST_AUTHENTICATE(atclient),
@@ -1027,6 +1569,9 @@ struct testcase_t link_handshake_tests[] = {
   TEST_AUTHENTICATE(tooshort_1),
   TEST_AUTHENTICATE(badcontent),
   TEST_AUTHENTICATE(badsig_1),
+  TEST_AUTHENTICATE_ED(badsig_1),
+  TEST_AUTHENTICATE_ED(missing_ed_id),
+  TEST_AUTHENTICATE_ED(missing_ed_auth),
   //TEST_AUTHENTICATE(),
 
   END_OF_TESTCASES
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 15471e46d0..94b3e4ea68 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 2ae605b8db..c78fda3b69 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -14,12 +14,6 @@
 
 #include "test.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
-#include <openssl/rsa.h>
-#include <openssl/bn.h>
-#include <openssl/pem.h>
-ENABLE_GCC_WARNING(redundant-decls)
-
 #ifdef _WIN32
 /* For mkdir() */
 #include <direct.h>
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index d58f8a7fca..256354415c 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2016, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c
index a560e5fc5e..d0eea85d6f 100644
--- a/src/test/test_ntor_cl.c
+++ b/src/test/test_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index 6102af01f5..f03a504d1d 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOM handling logic */
@@ -15,6 +15,7 @@
 #include "config.h"
 #include "relay.h"
 #include "test.h"
+#include "test_helpers.h"
 
 /* small replacement mock for circuit_mark_for_close_ to avoid doing all
  * the other bookkeeping that comes with marking circuits. */
@@ -58,24 +59,6 @@ dummy_or_circuit_new(int n_p_cells, int n_n_cells)
   return TO_CIRCUIT(circ);
 }
 
-static circuit_t *
-dummy_origin_circuit_new(int n_cells)
-{
-  origin_circuit_t *circ = origin_circuit_new();
-  int i;
-  cell_t cell;
-
-  for (i=0; i < n_cells; ++i) {
-    crypto_rand((void*)&cell, sizeof(cell));
-    cell_queue_append_packed_copy(TO_CIRCUIT(circ),
-                                  &TO_CIRCUIT(circ)->n_chan_cells,
-                                  1, &cell, 1, 0);
-  }
-
-  TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  return TO_CIRCUIT(circ);
-}
-
 static void
 add_bytes_to_buf(buf_t *buf, size_t n_bytes)
 {
diff --git a/src/test/test_oos.c b/src/test/test_oos.c
index db06625116..9fd6bce5ae 100644
--- a/src/test/test_oos.c
+++ b/src/test/test_oos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOS handler */
diff --git a/src/test/test_options.c b/src/test/test_options.c
index e85e11805b..ad735b72a6 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
@@ -18,6 +18,7 @@
 #include "sandbox.h"
 #include "memarea.h"
 #include "policies.h"
+#include "test_helpers.h"
 
 #define NS_MODULE test_options
 
@@ -104,11 +105,71 @@ clear_log_messages(void)
   "EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212:80 7EA6 EAD6 FD83" \
   " 083C 538F 4403 8BBF A077 587D D755\n"
 
+static int
+test_options_checklog(const char *configuration, int expect_log_severity,
+                      const char *expect_log)
+{
+  int found = 0, ret = -1;
+  char *actual_log = NULL;
+
+  if (messages) {
+    SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) {
+      if (m->severity == expect_log_severity &&
+          strstr(m->msg, expect_log)) {
+        found = 1;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+  if (!found) {
+    actual_log = dump_logs();
+    TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.",
+            log_level_to_string(expect_log_severity), expect_log,
+            configuration, actual_log));
+  }
+  ret = 0;
+
+ done:
+  tor_free(actual_log);
+  return ret;
+}
+
+static int
+test_options_checkmsgs(const char *configuration,
+                       const char *expect_errmsg,
+                       int expect_log_severity,
+                       const char *expect_log,
+                       char *msg)
+{
+  if (expect_errmsg && !msg) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got none.",
+            expect_errmsg, configuration));
+  } else if (expect_errmsg && !strstr(msg, expect_errmsg)) {
+    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
+            expect_errmsg, configuration, msg));
+  } else if (!expect_errmsg && msg) {
+    TT_DIE(("Expected no error message from <%s> but got <%s>.",
+            configuration, msg));
+  }
+  if (expect_log) {
+    return test_options_checklog(configuration, expect_log_severity,
+                                 expect_log);
+  }
+  return 0;
+
+ done:
+  return -1;
+}
+
+/* Which phases of config parsing/validation to check for messages/logs */
+enum { PH_GETLINES, PH_ASSIGN, PH_VALIDATE };
+
 static void
 test_options_validate_impl(const char *configuration,
                            const char *expect_errmsg,
                            int expect_log_severity,
-                           const char *expect_log)
+                           const char *expect_log,
+                           int phase)
 {
   or_options_t *opt=NULL;
   or_options_t *dflt;
@@ -119,43 +180,34 @@ test_options_validate_impl(const char *configuration,
   setup_options(opt, dflt);
 
   r = config_get_lines(configuration, &cl, 1);
-  tt_int_op(r, OP_EQ, 0);
+  if (phase == PH_GETLINES) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
+  }
+  if (r)
+    goto done;
 
   r = config_assign(&options_format, opt, cl, 0, &msg);
-  tt_int_op(r, OP_EQ, 0);
-
-  r = options_validate(NULL, opt, dflt, 0, &msg);
-  if (expect_errmsg && !msg) {
-    TT_DIE(("Expected error message <%s> from <%s>, but got none.",
-            expect_errmsg, configuration));
-  } else if (expect_errmsg && !strstr(msg, expect_errmsg)) {
-    TT_DIE(("Expected error message <%s> from <%s>, but got <%s>.",
-            expect_errmsg, configuration, msg));
-  } else if (!expect_errmsg && msg) {
-    TT_DIE(("Expected no error message from <%s> but got <%s>.",
-            configuration, msg));
+  if (phase == PH_ASSIGN) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
   }
   tt_int_op((r == 0), OP_EQ, (msg == NULL));
+  if (r)
+    goto done;
 
-  if (expect_log) {
-    int found = 0;
-    if (messages) {
-      SMARTLIST_FOREACH_BEGIN(messages, logmsg_t *, m) {
-        if (m->severity == expect_log_severity &&
-            strstr(m->msg, expect_log)) {
-          found = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(m);
-    }
-    if (!found) {
-      tor_free(msg);
-      msg = dump_logs();
-      TT_DIE(("Expected log message [%s] %s from <%s>, but got <%s>.",
-              log_level_to_string(expect_log_severity), expect_log,
-              configuration, msg));
-    }
+  r = options_validate(NULL, opt, dflt, 0, &msg);
+  if (phase == PH_VALIDATE) {
+    if (test_options_checkmsgs(configuration, expect_errmsg,
+                               expect_log_severity,
+                               expect_log, msg))
+      goto done;
   }
+  tt_int_op((r == 0), OP_EQ, (msg == NULL));
 
  done:
   escaped(NULL);
@@ -167,14 +219,14 @@ test_options_validate_impl(const char *configuration,
   clear_log_messages();
 }
 
-#define WANT_ERR(config, msg)                           \
-  test_options_validate_impl((config), (msg), 0, NULL)
-#define WANT_LOG(config, severity, msg)                         \
-  test_options_validate_impl((config), NULL, (severity), (msg))
-#define WANT_ERR_LOG(config, msg, severity, logmsg)                     \
-  test_options_validate_impl((config), (msg), (severity), (logmsg))
-#define OK(config)                                      \
-  test_options_validate_impl((config), NULL, 0, NULL)
+#define WANT_ERR(config, msg, ph)                               \
+  test_options_validate_impl((config), (msg), 0, NULL, (ph))
+#define WANT_LOG(config, severity, msg, ph)                             \
+  test_options_validate_impl((config), NULL, (severity), (msg), (ph))
+#define WANT_ERR_LOG(config, msg, severity, logmsg, ph)                 \
+  test_options_validate_impl((config), (msg), (severity), (logmsg), (ph))
+#define OK(config, ph)                                          \
+  test_options_validate_impl((config), NULL, 0, NULL, (ph))
 
 static void
 test_options_validate(void *arg)
@@ -183,21 +235,39 @@ test_options_validate(void *arg)
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
 
-  WANT_ERR("ExtORPort 500000", "Invalid ExtORPort");
+  WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
   WANT_ERR_LOG("ServerTransportOptions trebuchet",
                "ServerTransportOptions did not parse",
-               LOG_WARN, "Too few arguments");
-  OK("ServerTransportOptions trebuchet sling=snappy");
-  OK("ServerTransportOptions trebuchet sling=");
+               LOG_WARN, "Too few arguments", PH_VALIDATE);
+  OK("ServerTransportOptions trebuchet sling=snappy", PH_VALIDATE);
+  OK("ServerTransportOptions trebuchet sling=", PH_VALIDATE);
   WANT_ERR_LOG("ServerTransportOptions trebuchet slingsnappy",
                "ServerTransportOptions did not parse",
-               LOG_WARN, "\"slingsnappy\" is not a k=v");
+               LOG_WARN, "\"slingsnappy\" is not a k=v", PH_VALIDATE);
 
   WANT_ERR("DirPort 8080\nDirCache 0",
-           "DirPort configured but DirCache disabled.");
+           "DirPort configured but DirCache disabled.", PH_VALIDATE);
   WANT_ERR("BridgeRelay 1\nDirCache 0",
-           "We're a bridge but DirCache is disabled.");
+           "We're a bridge but DirCache is disabled.", PH_VALIDATE);
+
+  WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
+               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
+               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               PH_ASSIGN);
+  WANT_ERR_LOG("LogTimeGranularity 21 snarks",
+               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
+               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               PH_ASSIGN);
+  OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
+  OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
+
+  WANT_LOG("ControlSocket \"string with trailing garbage\" bogus", LOG_WARN,
+           "Error while parsing configuration: "
+           "Excess data after quoted string", PH_GETLINES);
+  WANT_LOG("ControlSocket \"bogus escape \\@\"", LOG_WARN,
+           "Error while parsing configuration: "
+           "Invalid escape sequence in quoted string", PH_GETLINES);
 
   close_temp_logs();
   clear_log_messages();
@@ -332,7 +402,8 @@ fixed_get_uname(void)
   "VirtualAddrNetworkIPv4 127.192.0.0/10\n"                             \
   "VirtualAddrNetworkIPv6 [FE80::]/10\n"                                \
   "SchedulerHighWaterMark__ 42\n"                                       \
-  "SchedulerLowWaterMark__ 10\n"
+  "SchedulerLowWaterMark__ 10\n"                                        \
+  "UseEntryGuards 1\n"
 
 typedef struct {
   or_options_t *old_opt;
@@ -352,6 +423,12 @@ get_options_test_data(const char *conf)
   result->opt = options_new();
   result->old_opt = options_new();
   result->def_opt = options_new();
+
+  // XXX: Really, all of these options should be set to defaults
+  // with options_init(), but about a dozen tests break when I do that.
+  // Being kinda lame and just fixing the immedate breakage for now..
+  result->opt->ConnectionPadding = -1; // default must be "auto"
+
   rv = config_get_lines(conf, &cl, 1);
   tt_assert(rv == 0);
   rv = config_assign(&options_format, result->opt, cl, 0, &msg);
@@ -400,7 +477,7 @@ test_options_validate__uname_for_server(void *ignored)
   (void)ignored;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                      "ORListenAddress 127.0.0.1:5555");
+                                      "ORPort 127.0.0.1:5555");
   setup_capture_of_logs(LOG_WARN);
 
   MOCK(get_uname, fixed_get_uname);
@@ -534,7 +611,7 @@ test_options_validate__contactinfo(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                "ORListenAddress 127.0.0.1:5555\nORPort 955");
+                                "ORPort 127.0.0.1:5555");
   setup_capture_of_logs(LOG_DEBUG);
   tdata->opt->ContactInfo = NULL;
 
@@ -547,7 +624,7 @@ test_options_validate__contactinfo(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("ORListenAddress 127.0.0.1:5555\nORPort 955\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "ContactInfo hella@example.org");
   mock_clean_saved_logs();
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -650,16 +727,18 @@ test_options_validate__authdir(void *ignored)
   setup_capture_of_logs(LOG_INFO);
   options_test_data_t *tdata = get_options_test_data(
                                  "AuthoritativeDirectory 1\n"
-                                 "Address this.should.not_exist.example.org");
+                                 "Address this.should.not!exist!.example.org");
 
   sandbox_disable_getaddrinfo_cache();
 
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to resolve/guess local address. See logs for"
             " details.");
   expect_log_msg("Could not resolve local Address "
-            "'this.should.not_exist.example.org'. Failing.\n");
+            "'this.should.not!exist!.example.org'. Failing.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
@@ -953,8 +1032,7 @@ test_options_validate__relay_with_hidden_services(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_DEBUG);
   options_test_data_t *tdata = get_options_test_data(
-                                  "ORListenAddress 127.0.0.1:5555\n"
-                                  "ORPort 955\n"
+                                  "ORPort 127.0.0.1:5555\n"
                                   "HiddenServiceDir "
                                   "/Library/Tor/var/lib/tor/hidden_service/\n"
                                   "HiddenServicePort 80 127.0.0.1:8080\n"
@@ -1018,13 +1096,13 @@ test_options_validate__transproxy(void *ignored)
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
-#if !defined(__OpenBSD__) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined( DARWIN )
   tt_str_op(msg, OP_EQ,
           "pf-divert is a OpenBSD-specific and OS X/Darwin-specific feature.");
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_PF_DIVERT);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without "
-            "any valid TransPort or TransListenAddress.");
+            "any valid TransPort.");
 #endif
   tor_free(msg);
 
@@ -1039,7 +1117,7 @@ test_options_validate__transproxy(void *ignored)
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_TPROXY);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without any valid "
-            "TransPort or TransListenAddress.");
+            "TransPort.");
 #endif
   tor_free(msg);
 
@@ -1055,7 +1133,7 @@ test_options_validate__transproxy(void *ignored)
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_IPFW);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without any valid "
-            "TransPort or TransListenAddress.");
+            "TransPort.");
 #endif
   tor_free(msg);
 
@@ -1087,7 +1165,7 @@ test_options_validate__transproxy(void *ignored)
   if (msg) {
     TT_DIE(("Expected NULL but got '%s'", msg));
   }
-#elif defined(__OpenBSD__)
+#elif defined(OpenBSD)
   tdata = get_options_test_data("TransProxyType pf-divert\n"
                                 "TransPort 127.0.0.1:123\n");
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -1113,8 +1191,7 @@ test_options_validate__transproxy(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "TransPort and TransListenAddress are disabled in "
-            "this build.");
+  tt_str_op(msg, OP_EQ, "TransPort is disabled in this build.");
   tor_free(msg);
 #endif
 
@@ -1309,54 +1386,6 @@ test_options_validate__node_families(void *ignored)
 }
 
 static void
-test_options_validate__tlsec(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_DEBUG);
-  options_test_data_t *tdata = get_options_test_data(
-                                 "TLSECGroup ed25519\n"
-                                 "SchedulerHighWaterMark__ 42\n"
-                                 "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_log_msg("Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(!tdata->opt->TLSECGroup);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("TLSECGroup P224\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg(
-            "Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(tdata->opt->TLSECGroup);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("TLSECGroup P256\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg(
-            "Unrecognized TLSECGroup: Falling back to the default.\n");
-  tt_assert(tdata->opt->TLSECGroup);
-  tor_free(msg);
-
- done:
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__token_bucket(void *ignored)
 {
   (void)ignored;
@@ -1738,8 +1767,7 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
                                 "ConnLimit 1\n"
                                 "SchedulerHighWaterMark__ 42\n"
@@ -1752,8 +1780,7 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableORAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
                                 "ConnLimit 1\n"
                                 "SchedulerHighWaterMark__ 42\n"
@@ -1766,8 +1793,7 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableDirAddresses *:82\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
                                 "ConnLimit 1\n"
                                 "SchedulerHighWaterMark__ 42\n"
@@ -1780,8 +1806,7 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ClientUseIPv4 0\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "MaxClientCircuitsPending 1\n"
                                 "ConnLimit 1\n"
                                 "SchedulerHighWaterMark__ 42\n"
@@ -1794,14 +1819,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   /* Test IPv4-only clients setting IPv6 preferences */
 
-#define WARN_PLEASE_USE_IPV6_OR_LOG_MSG \
-        "ClientPreferIPv6ORPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges.\n"
-
-#define WARN_PLEASE_USE_IPV6_DIR_LOG_MSG \
-        "ClientPreferIPv6DirPort 1 is ignored unless tor is using IPv6. " \
-        "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges.\n"
-
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ClientUseIPv4 1\n"
@@ -1811,7 +1828,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg(WARN_PLEASE_USE_IPV6_OR_LOG_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
@@ -1823,7 +1839,6 @@ test_options_validate__reachable_addresses(void *ignored)
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg(WARN_PLEASE_USE_IPV6_DIR_LOG_MSG);
   tor_free(msg);
 
   /* Now test an IPv4/IPv6 client setting IPv6 preferences */
@@ -1891,8 +1906,7 @@ test_options_validate__use_bridges(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                    "UseBridges 1\n"
                                    "ClientUseIPv4 1\n"
-                                   "ORListenAddress 127.0.0.1:5555\n"
-                                   "ORPort 955\n"
+                                   "ORPort 127.0.0.1:5555\n"
                                    "MaxClientCircuitsPending 1\n"
                                    "ConnLimit 1\n"
                                    "SchedulerHighWaterMark__ 42\n"
@@ -1945,6 +1959,19 @@ test_options_validate__use_bridges(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
+                                "UseEntryGuards 0\n"
+                                );
+
+  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "Setting UseBridges requires also setting UseEntryGuards.");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+                                "UseBridges 1\n"
+                                "Bridge 10.0.0.1\n"
                                 "Bridge !!!\n"
                                 );
 
@@ -2001,56 +2028,6 @@ test_options_validate__entry_nodes(void *ignored)
 }
 
 static void
-test_options_validate__invalid_nodes(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                  "AllowInvalidNodes something_stupid\n"
-                                  "MaxClientCircuitsPending 1\n"
-                                  "ConnLimit 1\n"
-                                  "SchedulerHighWaterMark__ 42\n"
-                                  "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Unrecognized value 'something_stupid' in AllowInvalidNodes");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("AllowInvalidNodes entry, middle, exit\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_int_op(tdata->opt->AllowInvalid_, OP_EQ, ALLOW_INVALID_ENTRY |
-            ALLOW_INVALID_EXIT | ALLOW_INVALID_MIDDLE);
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data("AllowInvalidNodes introduction, rendezvous\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n"
-                                "SchedulerHighWaterMark__ 42\n"
-                                "SchedulerLowWaterMark__ 10\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_int_op(tdata->opt->AllowInvalid_, OP_EQ, ALLOW_INVALID_INTRODUCTION |
-            ALLOW_INVALID_RENDEZVOUS);
-  tor_free(msg);
-
- done:
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__safe_logging(void *ignored)
 {
   (void)ignored;
@@ -2320,30 +2297,6 @@ test_options_validate__hidserv(void *ignored)
 }
 
 static void
-test_options_validate__predicted_ports(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-
-  options_test_data_t *tdata = get_options_test_data(
-                                     "PredictedPortsRelevanceTime 100000000\n"
-                                     TEST_OPTIONS_DEFAULT_VALUES);
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("PredictedPortsRelevanceTime is too "
-            "large; clipping to 3600s.\n");
-  tt_int_op(tdata->opt->PredictedPortsRelevanceTime, OP_EQ, 3600);
-
- done:
-  teardown_capture_of_logs();
-  policies_free_all();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__path_bias(void *ignored)
 {
   (void)ignored;
@@ -2489,8 +2442,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 1\n"
                                 );
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -2501,8 +2453,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 30000\n"
                                 );
@@ -2514,8 +2465,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "RelayBandwidthRate 1\n"
                                 "MaxAdvertisedBandwidth 38400\n"
@@ -2528,8 +2478,7 @@ test_options_validate__bandwidth(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "BandwidthBurst 76800\n"
                                 "RelayBandwidthRate 76800\n"
@@ -2819,8 +2768,8 @@ test_options_validate__single_onion(void *ignored)
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode is incompatible with "
             "using Tor as an anonymous client. Please set "
-            "Socks/Trans/NATD/DNSPort to 0, or HiddenServiceNonAnonymousMode "
-            "to 0, or use the non-anonymous Tor2webMode.");
+            "Socks/Trans/NATD/DNSPort to 0, or revert "
+            "HiddenServiceNonAnonymousMode to 0.");
   tor_free(msg);
   free_options_test_data(tdata);
 
@@ -2967,8 +2916,7 @@ test_options_validate__accounting(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(
            TEST_OPTIONS_DEFAULT_VALUES
-           "ORListenAddress 127.0.0.1:5555\n"
-           "ORPort 955\n"
+           "ORPort 127.0.0.1:5555\n"
            "BandwidthRate 76800\n"
            "BandwidthBurst 76800\n"
            "MaxAdvertisedBandwidth 38400\n"
@@ -3037,6 +2985,7 @@ test_options_validate__proxy(void *ignored)
   options_test_data_t *tdata = NULL;
   sandbox_disable_getaddrinfo_cache();
   setup_capture_of_logs(LOG_WARN);
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
@@ -3057,6 +3006,7 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
+
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "HttpProxy not_so_valid!\n"
                                 );
@@ -3357,6 +3307,7 @@ test_options_validate__proxy(void *ignored)
   policies_free_all();
   // sandbox_free_getaddrinfo_cache();
   tor_free(msg);
+  UNMOCK(tor_addr_lookup);
 }
 
 static void
@@ -3599,8 +3550,7 @@ test_options_validate__families(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "MyFamily home\n"
                                 "BridgeRelay 1\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 51300\n"
                                 "BandwidthBurst 51300\n"
                                 "MaxAdvertisedBandwidth 25700\n"
@@ -3829,8 +3779,7 @@ test_options_validate__transport(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ServerTransportPlugin foo exec bar\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
@@ -3872,8 +3821,7 @@ test_options_validate__transport(void *ignored)
   tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
                                 "ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 "ServerTransportPlugin foo exec bar\n"
-                                "ORListenAddress 127.0.0.1:5555\n"
-                                "ORPort 955\n"
+                                "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
@@ -4230,48 +4178,6 @@ test_options_validate__virtual_addr(void *ignored)
 }
 
 static void
-test_options_validate__exits(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  options_test_data_t *tdata = NULL;
-  setup_capture_of_logs(LOG_WARN);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AllowSingleHopExits 1"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("You have set AllowSingleHopExits; "
-            "now your relay will allow others to make one-hop exits. However,"
-            " since by default most clients avoid relays that set this option,"
-            " most clients will ignore you.\n");
-  tor_free(msg);
-
-  free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AllowSingleHopExits 1\n"
-                                VALID_DIR_AUTH
-                                );
-  mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_no_log_msg("You have set AllowSingleHopExits; "
-            "now your relay will allow others to make one-hop exits. However,"
-            " since by default most clients avoid relays that set this option,"
-            " most clients will ignore you.\n");
-  tor_free(msg);
-
- done:
-  policies_free_all();
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__testing_options(void *ignored)
 {
   (void)ignored;
@@ -4509,7 +4415,6 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(exclude_nodes),
   LOCAL_VALIDATE_TEST(scheduler),
   LOCAL_VALIDATE_TEST(node_families),
-  LOCAL_VALIDATE_TEST(tlsec),
   LOCAL_VALIDATE_TEST(token_bucket),
   LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
@@ -4520,12 +4425,10 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(reachable_addresses),
   LOCAL_VALIDATE_TEST(use_bridges),
   LOCAL_VALIDATE_TEST(entry_nodes),
-  LOCAL_VALIDATE_TEST(invalid_nodes),
   LOCAL_VALIDATE_TEST(safe_logging),
   LOCAL_VALIDATE_TEST(publish_server_descriptor),
   LOCAL_VALIDATE_TEST(testing),
   LOCAL_VALIDATE_TEST(hidserv),
-  LOCAL_VALIDATE_TEST(predicted_ports),
   LOCAL_VALIDATE_TEST(path_bias),
   LOCAL_VALIDATE_TEST(bandwidth),
   LOCAL_VALIDATE_TEST(circuits),
@@ -4543,7 +4446,6 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(constrained_sockets),
   LOCAL_VALIDATE_TEST(v3_auth),
   LOCAL_VALIDATE_TEST(virtual_addr),
-  LOCAL_VALIDATE_TEST(exits),
   LOCAL_VALIDATE_TEST(testing_options),
   LOCAL_VALIDATE_TEST(accel),
   END_OF_TESTCASES              /*  */
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 1ffdc2cd51..1b2fac4325 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2016, The Tor Project, Inc. */
+/* Copyright (c) 2013-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -1031,10 +1031,10 @@ test_policies_general(void *arg)
   {
     char *policy_strng = NULL;
     smartlist_t *chunks = smartlist_new();
-    smartlist_add(chunks, tor_strdup("accept "));
+    smartlist_add_strdup(chunks, "accept ");
     for (i=1; i<10000; ++i)
       smartlist_add_asprintf(chunks, "%d,", i);
-    smartlist_add(chunks, tor_strdup("20000"));
+    smartlist_add_strdup(chunks, "20000");
     policy_strng = smartlist_join_strings(chunks, "", 0, NULL);
     SMARTLIST_FOREACH(chunks, char *, ch, tor_free(ch));
     smartlist_free(chunks);
@@ -1048,9 +1048,9 @@ test_policies_general(void *arg)
   for (i=1; i<2000; i+=2) {
     char buf[POLICY_BUF_LEN];
     tor_snprintf(buf, sizeof(buf), "reject *:%d", i);
-    smartlist_add(sm, tor_strdup(buf));
+    smartlist_add_strdup(sm, buf);
   }
-  smartlist_add(sm, tor_strdup("accept *:*"));
+  smartlist_add_strdup(sm, "accept *:*");
   policy_str = smartlist_join_strings(sm, ",", 0, NULL);
   test_policy_summary_helper( policy_str,
     "accept 2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,"
@@ -1587,8 +1587,12 @@ test_policies_getinfo_helper_policies(void *arg)
   append_exit_policy_string(&mock_my_routerinfo.exit_policy, "reject *6:*");
 
   mock_options.IPv6Exit = 1;
-  tor_addr_from_ipv4h(&mock_options.OutboundBindAddressIPv4_, TEST_IPV4_ADDR);
-  tor_addr_parse(&mock_options.OutboundBindAddressIPv6_, TEST_IPV6_ADDR);
+  tor_addr_from_ipv4h(
+      &mock_options.OutboundBindAddresses[OUTBOUND_ADDR_EXIT][0],
+      TEST_IPV4_ADDR);
+  tor_addr_parse(
+      &mock_options.OutboundBindAddresses[OUTBOUND_ADDR_EXIT][1],
+      TEST_IPV6_ADDR);
 
   mock_options.ExitPolicyRejectPrivate = 1;
   mock_options.ExitPolicyRejectLocalInterfaces = 1;
diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c
index 9e63fc006d..5c52af8693 100644
--- a/src/test/test_procmon.c
+++ b/src/test/test_procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROCMON_PRIVATE
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index 92ead3ca37..c093e69968 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROTOVER_PRIVATE
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index e5cdc5f3cd..79b03171bc 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -155,9 +155,9 @@ test_pt_get_transport_options(void *arg)
   opt_str = get_transport_options_for_server_proxy(mp);
   tt_ptr_op(opt_str, OP_EQ, NULL);
 
-  smartlist_add(mp->transports_to_launch, tor_strdup("gruyere"));
-  smartlist_add(mp->transports_to_launch, tor_strdup("roquefort"));
-  smartlist_add(mp->transports_to_launch, tor_strdup("stnectaire"));
+  smartlist_add_strdup(mp->transports_to_launch, "gruyere");
+  smartlist_add_strdup(mp->transports_to_launch, "roquefort");
+  smartlist_add_strdup(mp->transports_to_launch, "stnectaire");
 
   tt_assert(options);
 
@@ -284,13 +284,13 @@ test_pt_get_extrainfo_string(void *arg)
 }
 
 #ifdef _WIN32
-#define STDIN_HANDLE HANDLE
+#define STDIN_HANDLE HANDLE*
 #else
-#define STDIN_HANDLE FILE
+#define STDIN_HANDLE int
 #endif
 
 static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle,
+tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
                                       enum stream_status *stream_status_out)
 {
   static int times_called = 0;
@@ -305,7 +305,7 @@ tor_get_lines_from_handle_replacement(STDIN_HANDLE *handle,
     smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
                            times_called, times_called);
   } else {
-    smartlist_add(retval_sl, tor_strdup("SMETHODS DONE"));
+    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
   }
 
   return retval_sl;
diff --git a/src/test/test_pubsub.c b/src/test/test_pubsub.c
index 547d6c6b32..2f047d9f2c 100644
--- a/src/test/test_pubsub.c
+++ b/src/test/test_pubsub.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016, The Tor Project, Inc. */
+/* Copyright (c) 2016-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/test/test_relay.c b/src/test/test_relay.c
index 4713c79ea5..238d4c5baf 100644
--- a/src/test/test_relay.c
+++ b/src/test/test_relay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index fb6748965a..eea1f5dc80 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for handling different kinds of relay cell */
diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c
index a5d3f351f8..feba8f664e 100644
--- a/src/test/test_rendcache.c
+++ b/src/test/test_rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -10,7 +10,7 @@
 #include "router.h"
 #include "routerlist.h"
 #include "config.h"
-#include <openssl/rsa.h>
+#include "hs_common.h"
 #include "rend_test_helpers.h"
 #include "log_test_helpers.h"
 
@@ -24,15 +24,16 @@ static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 60;
 static rend_data_t *
 mock_rend_data(const char *onion_address)
 {
-  rend_data_t *rend_query = tor_malloc_zero(sizeof(rend_data_t));
+  rend_data_v2_t *v2_data = tor_malloc_zero(sizeof(*v2_data));
+  rend_data_t *rend_query = &v2_data->base_;
+  rend_query->version = 2;
 
-  strlcpy(rend_query->onion_address, onion_address,
-          sizeof(rend_query->onion_address));
-  rend_query->auth_type = REND_NO_AUTH;
+  strlcpy(v2_data->onion_address, onion_address,
+          sizeof(v2_data->onion_address));
+  v2_data->auth_type = REND_NO_AUTH;
   rend_query->hsdirs_fp = smartlist_new();
   smartlist_add(rend_query->hsdirs_fp, tor_memdup("aaaaaaaaaaaaaaaaaaaaaaaa",
                                                  DIGEST_LEN));
-
   return rend_query;
 }
 
@@ -144,7 +145,8 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   // Test mismatch between service ID and onion address
   rend_cache_init();
-  strncpy(mock_rend_query->onion_address, "abc", REND_SERVICE_ID_LEN_BASE32+1);
+  strncpy(TO_REND_DATA_V2(mock_rend_query)->onion_address, "abc",
+          REND_SERVICE_ID_LEN_BASE32+1);
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
                                            desc_id_base32,
                                            mock_rend_query, NULL);
@@ -155,12 +157,16 @@ test_rend_cache_store_v2_desc_as_client(void *data)
   // Test incorrect descriptor ID
   rend_cache_init();
   mock_rend_query = mock_rend_data(service_id);
-  desc_id_base32[0]++;
+  char orig = desc_id_base32[0];
+  if (desc_id_base32[0] == 'a')
+    desc_id_base32[0] = 'b';
+  else
+    desc_id_base32[0] = 'a';
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
                                            desc_id_base32, mock_rend_query,
                                            NULL);
   tt_int_op(ret, OP_EQ, -1);
-  desc_id_base32[0]--;
+  desc_id_base32[0] = orig;
   rend_cache_free_all();
 
   // Test too old descriptor
@@ -230,9 +236,9 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   generate_desc(RECENT_TIME, &desc_holder, &service_id, 3);
   mock_rend_query = mock_rend_data(service_id);
-  mock_rend_query->auth_type = REND_BASIC_AUTH;
+  TO_REND_DATA_V2(mock_rend_query)->auth_type = REND_BASIC_AUTH;
   client_cookie[0] = 'A';
-  memcpy(mock_rend_query->descriptor_cookie, client_cookie,
+  memcpy(TO_REND_DATA_V2(mock_rend_query)->descriptor_cookie, client_cookie,
          REND_DESC_COOKIE_LEN);
   base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_holder->desc_id,
                 DIGEST_LEN);
@@ -250,7 +256,7 @@ test_rend_cache_store_v2_desc_as_client(void *data)
 
   generate_desc(RECENT_TIME, &desc_holder, &service_id, 3);
   mock_rend_query = mock_rend_data(service_id);
-  mock_rend_query->auth_type = REND_BASIC_AUTH;
+  TO_REND_DATA_V2(mock_rend_query)->auth_type = REND_BASIC_AUTH;
   base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_holder->desc_id,
                 DIGEST_LEN);
   ret = rend_cache_store_v2_desc_as_client(desc_holder->desc_str,
@@ -1078,9 +1084,10 @@ static void
 test_rend_cache_clean_v2_descs_as_dir(void *data)
 {
   rend_cache_entry_t *e;
-  time_t now;
+  time_t now, cutoff;
   rend_service_descriptor_t *desc;
   now = time(NULL);
+  cutoff = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW);
   const char key[DIGEST_LEN] = "abcde";
 
   (void)data;
@@ -1088,7 +1095,7 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   rend_cache_init();
 
   // Test running with an empty cache
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  rend_cache_clean_v2_descs_as_dir(cutoff);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
 
   // Test with only one new entry
@@ -1100,38 +1107,15 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   e->parsed = desc;
   digestmap_set(rend_cache_v2_dir, key, e);
 
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  /* Set the cutoff to minus 10 seconds. */
+  rend_cache_clean_v2_descs_as_dir(cutoff - 10);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 1);
 
   // Test with one old entry
-  desc->timestamp = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW + 1000);
-  rend_cache_clean_v2_descs_as_dir(now, 0);
-  tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
-
-  // Test with one entry that has an old last served
-  e = tor_malloc_zero(sizeof(rend_cache_entry_t));
-  e->last_served = now - (REND_CACHE_MAX_AGE + REND_CACHE_MAX_SKEW + 1000);
-  desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  desc->timestamp = now;
-  desc->pk = pk_generate(0);
-  e->parsed = desc;
-  digestmap_set(rend_cache_v2_dir, key, e);
-
-  rend_cache_clean_v2_descs_as_dir(now, 0);
+  desc->timestamp = cutoff - 1000;
+  rend_cache_clean_v2_descs_as_dir(cutoff);
   tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 0);
 
-  // Test a run through asking for a large force_remove
-  e = tor_malloc_zero(sizeof(rend_cache_entry_t));
-  e->last_served = now;
-  desc = tor_malloc_zero(sizeof(rend_service_descriptor_t));
-  desc->timestamp = now;
-  desc->pk = pk_generate(0);
-  e->parsed = desc;
-  digestmap_set(rend_cache_v2_dir, key, e);
-
-  rend_cache_clean_v2_descs_as_dir(now, 20000);
-  tt_int_op(digestmap_size(rend_cache_v2_dir), OP_EQ, 1);
-
  done:
   rend_cache_free_all();
 }
diff --git a/src/test/test_replay.c b/src/test/test_replay.c
index e882bc6164..80e7203716 100644
--- a/src/test/test_replay.c
+++ b/src/test/test_replay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2016, The Tor Project, Inc. */
+/* Copyright (c) 2012-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define REPLAYCACHE_PRIVATE
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 24b0da1c46..db6b9b3872 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -450,8 +450,8 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
   options->DataDirectory = dir;
 
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
+  tt_int_op(1, ==, load_ed_keys(options, now));
+  tt_int_op(0, ==, generate_ed_link_cert(options, now, 0));
   tt_assert(get_master_identity_key());
   tt_assert(get_master_identity_key());
   tt_assert(get_master_signing_keypair());
@@ -466,7 +466,7 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
   /* Call load_ed_keys again, but nothing has changed. */
   tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
+  tt_int_op(0, ==, generate_ed_link_cert(options, now, 0));
   tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
   tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
   tt_mem_op(&auth, ==, get_current_auth_keypair(), sizeof(auth));
@@ -474,8 +474,8 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
   /* Force a reload: we make new link/auth keys. */
   routerkeys_free_all();
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
+  tt_int_op(1, ==, load_ed_keys(options, now));
+  tt_int_op(0, ==, generate_ed_link_cert(options, now, 0));
   tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
   tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
   tt_assert(tor_cert_eq(link_cert, get_current_link_cert_cert()));
@@ -489,7 +489,7 @@ test_routerkeys_ed_keys_init_all(void *arg)
 
   /* Force a link/auth-key regeneration by advancing time. */
   tt_int_op(0, ==, load_ed_keys(options, now+3*86400));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now+3*86400));
+  tt_int_op(0, ==, generate_ed_link_cert(options, now+3*86400, 0));
   tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
   tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
@@ -502,8 +502,8 @@ test_routerkeys_ed_keys_init_all(void *arg)
   memcpy(&auth, get_current_auth_keypair(), sizeof(auth));
 
   /* Force a signing-key regeneration by advancing time. */
-  tt_int_op(0, ==, load_ed_keys(options, now+100*86400));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now+100*86400));
+  tt_int_op(1, ==, load_ed_keys(options, now+100*86400));
+  tt_int_op(0, ==, generate_ed_link_cert(options, now+100*86400, 0));
   tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
   tt_mem_op(&sign, !=, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
@@ -520,8 +520,8 @@ test_routerkeys_ed_keys_init_all(void *arg)
   routerkeys_free_all();
   unlink(get_fname("test_ed_keys_init_all/keys/"
                    "ed25519_master_id_secret_key"));
-  tt_int_op(0, ==, load_ed_keys(options, now));
-  tt_int_op(0, ==, generate_ed_link_cert(options, now));
+  tt_int_op(1, ==, load_ed_keys(options, now));
+  tt_int_op(0, ==, generate_ed_link_cert(options, now, 0));
   tt_mem_op(&id, ==, get_master_identity_key(), sizeof(id));
   tt_mem_op(&sign, ==, get_master_signing_keypair(), sizeof(sign));
   tt_assert(! tor_cert_eq(link_cert, get_current_link_cert_cert()));
@@ -614,6 +614,66 @@ test_routerkeys_cross_certify_tap(void *args)
   crypto_pk_free(onion_key);
 }
 
+static void
+test_routerkeys_rsa_ed_crosscert(void *arg)
+{
+  (void)arg;
+  ed25519_public_key_t ed;
+  crypto_pk_t *rsa = pk_generate(2);
+
+  uint8_t *cc = NULL;
+  ssize_t cc_len;
+  time_t expires_in = 1470846177;
+
+  tt_int_op(0, OP_EQ, ed25519_public_from_base64(&ed,
+                        "ThisStringCanContainAnythingSoNoKeyHereNowX"));
+  cc_len = tor_make_rsa_ed25519_crosscert(&ed, rsa, expires_in, &cc);
+
+  tt_int_op(cc_len, OP_GT, 0);
+  tt_int_op(cc_len, OP_GT, 37); /* key, expires, siglen */
+  tt_mem_op(cc, OP_EQ, ed.pubkey, 32);
+  time_t expires_out = 3600 * ntohl(get_uint32(cc+32));
+  tt_int_op(expires_out, OP_GE, expires_in);
+  tt_int_op(expires_out, OP_LE, expires_in + 3600);
+
+  tt_int_op(cc_len, OP_EQ, 37 + get_uint8(cc+36));
+
+  tt_int_op(0, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+
+  /* Now try after it has expired */
+  tt_int_op(-4, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_out + 1));
+
+  /* Truncated object */
+  tt_int_op(-2, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len - 2, rsa, &ed,
+                                                  expires_in - 10));
+
+  /* Key not as expected */
+  cc[0] ^= 3;
+  tt_int_op(-3, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+  cc[0] ^= 3;
+
+  /* Bad signature */
+  cc[40] ^= 3;
+  tt_int_op(-5, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                   expires_in - 10));
+  cc[40] ^= 3;
+
+  /* Signature of wrong data */
+  cc[0] ^= 3;
+  ed.pubkey[0] ^= 3;
+  tt_int_op(-6, OP_EQ, rsa_ed25519_crosscert_check(cc, cc_len, rsa, &ed,
+                                                  expires_in - 10));
+  cc[0] ^= 3;
+  ed.pubkey[0] ^= 3;
+
+ done:
+  crypto_pk_free(rsa);
+  tor_free(cc);
+}
+
 #define TEST(name, flags)                                       \
   { #name , test_routerkeys_ ## name, (flags), NULL, NULL }
 
@@ -626,6 +686,7 @@ struct testcase_t routerkeys_tests[] = {
   TEST(ed_keys_init_all, TT_FORK),
   TEST(cross_certify_ntor, 0),
   TEST(cross_certify_tap, 0),
+  TEST(rsa_ed_crosscert, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 088bd257c3..0b4b6c5c44 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -15,6 +15,7 @@
 #include "container.h"
 #include "directory.h"
 #include "dirvote.h"
+#include "entrynodes.h"
 #include "microdesc.h"
 #include "networkstatus.h"
 #include "nodelist.h"
@@ -335,30 +336,6 @@ test_router_pick_directory_server_impl(void *arg)
   node_router1->is_valid = 1;
   node_router3->is_valid = 1;
 
-  flags |= PDS_FOR_GUARD;
-  node_router1->using_as_guard = 1;
-  node_router2->using_as_guard = 1;
-  node_router3->using_as_guard = 1;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs == NULL);
-  node_router1->using_as_guard = 0;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
-  tt_assert(tor_memeq(rs->identity_digest, router1_id, DIGEST_LEN));
-  rs = NULL;
-  node_router2->using_as_guard = 0;
-  node_router3->using_as_guard = 0;
-
-  /* One not valid, one guard. This should leave one remaining */
-  node_router1->is_valid = 0;
-  node_router2->using_as_guard = 1;
-  rs = router_pick_directory_server_impl(V3_DIRINFO, flags, NULL);
-  tt_assert(rs != NULL);
-  tt_assert(tor_memeq(rs->identity_digest, router3_id, DIGEST_LEN));
-  rs = NULL;
-  node_router1->is_valid = 1;
-  node_router2->using_as_guard = 0;
-
   /* Manipulate overloaded */
 
   node_router2->rs->last_dir_503_at = now;
@@ -420,6 +397,7 @@ test_router_pick_directory_server_impl(void *arg)
 
  done:
   UNMOCK(usable_consensus_flavor);
+
   if (router1_id)
     tor_free(router1_id);
   if (router2_id)
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index 1b526d430b..7efd042ed5 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -623,7 +623,7 @@ NS(test_main)(void *arg)
   (void)arg;
 
   tgt = routerset_new();
-  smartlist_add(src->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(src->list, "{xx}");
   routerset_union(tgt, src);
 
   tt_int_op(smartlist_len(tgt->list), OP_NE, 0);
@@ -745,7 +745,7 @@ NS(test_main)(void *arg)
   tt_int_op(is_empty, OP_NE, 0);
 
   set = routerset_new();
-  smartlist_add(set->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->list, "{xx}");
   is_empty = routerset_is_empty(set);
   routerset_free(set);
   set = NULL;
@@ -1616,7 +1616,7 @@ NS(test_main)(void *arg)
   NS_MOCK(node_get_by_nickname);
 
   NS(mock_nickname) = "foo";
-  smartlist_add(set->list, tor_strdup(NS(mock_nickname)));
+  smartlist_add_strdup(set->list, NS(mock_nickname));
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1667,7 +1667,7 @@ NS(test_main)(void *arg)
 
   NS(mock_node).is_running = 0;
   NS(mock_nickname) = "foo";
-  smartlist_add(set->list, tor_strdup(NS(mock_nickname)));
+  smartlist_add_strdup(set->list, NS(mock_nickname));
 
   routerset_get_all_nodes(out, set, NULL, 1);
   out_len = smartlist_len(out);
@@ -1766,7 +1766,7 @@ NS(test_main)(void *arg)
 
   NS_MOCK(nodelist_get_list);
 
-  smartlist_add(set->country_names, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->country_names, "{xx}");
   NS(mock_smartlist) = smartlist_new();
 
   routerset_get_all_nodes(out, set, NULL, 1);
@@ -1813,7 +1813,7 @@ NS(test_main)(void *arg)
 
   NS_MOCK(nodelist_get_list);
 
-  smartlist_add(set->country_names, tor_strdup("{xx}"));
+  smartlist_add_strdup(set->country_names, "{xx}");
   NS(mock_smartlist) = smartlist_new();
   NS(mock_node).is_running = 0;
   smartlist_add(NS(mock_smartlist), (void *)&NS(mock_node));
@@ -1985,7 +1985,7 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(b->list, tor_strdup("{xx}"));
+  smartlist_add_strdup(b->list, "{xx}");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2010,9 +2010,9 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("{aa}"));
-  smartlist_add(b->list, tor_strdup("{b1}"));
-  smartlist_add(b->list, tor_strdup("{b2}"));
+  smartlist_add_strdup(a->list, "{aa}");
+  smartlist_add_strdup(b->list, "{b1}");
+  smartlist_add_strdup(b->list, "{b2}");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2037,8 +2037,8 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("foo"));
-  smartlist_add(b->list, tor_strdup("bar"));
+  smartlist_add_strdup(a->list, "foo");
+  smartlist_add_strdup(b->list, "bar");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
@@ -2063,8 +2063,8 @@ NS(test_main)(void *arg)
   int r;
   (void)arg;
 
-  smartlist_add(a->list, tor_strdup("foo"));
-  smartlist_add(b->list, tor_strdup("foo"));
+  smartlist_add_strdup(a->list, "foo");
+  smartlist_add_strdup(b->list, "foo");
   r = routerset_equal(a, b);
   routerset_free(a);
   routerset_free(b);
diff --git a/src/test/test_rust.c b/src/test/test_rust.c
new file mode 100644
index 0000000000..6ad57d6fcb
--- /dev/null
+++ b/src/test/test_rust.c
@@ -0,0 +1,31 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "compat_rust.h"
+#include "test.h"
+#include "util.h"
+
+static void
+test_welcome_string(void *arg)
+{
+  (void)arg;
+  rust_str_t s = rust_welcome_string();
+  const char *c_str = rust_str_get(s);
+  tt_assert(c_str);
+  size_t len = strlen(c_str);
+#ifdef HAVE_RUST
+  tt_assert(len > 0);
+#else
+  tt_assert(len == 0);
+#endif
+
+ done:
+  rust_str_free(s);
+}
+
+struct testcase_t rust_tests[] = {
+  { "welcome_string", test_welcome_string, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
new file mode 100755
index 0000000000..d559f94ce0
--- /dev/null
+++ b/src/test/test_rust.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Test all the Rust crates we're using
+
+crates=tor_util
+
+exitcode=0
+
+for crate in $crates; do
+    cd "${abs_top_srcdir:-.}/src/rust/${crate}"
+    CARGO_TARGET_DIR="${abs_top_builddir}/src/rust/target" CARGO_HOME="${abs_top_builddir}/src/rust" "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} || exitcode=1
+done
+
+exit $exitcode
diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c
index 05ea8e86e8..4c536b0905 100644
--- a/src/test/test_scheduler.c
+++ b/src/test/test_scheduler.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2016, The Tor Project, Inc. */
+/* Copyright (c) 2014-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index 6a8c1abaff..8cb35be6a1 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -48,7 +48,7 @@ init_authority_state(void)
   mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
-  tt_int_op(0, ==, load_ed_keys(options, time(NULL)));
+  tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
   sr_state_init(0, 0);
   /* It's possible a commit has been generated in our state depending on
    * the phase we are currently in which uses "now" as the starting
@@ -286,7 +286,7 @@ test_sr_commit(void *arg)
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
-    tt_int_op(0, ==, load_ed_keys(options, now));
+    tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
   }
 
   /* Generate our commit object and validate it has the appropriate field
@@ -348,12 +348,12 @@ test_sr_commit(void *arg)
   /* We'll build a list of values from our commit that our parsing function
    * takes from a vote line and see if we can parse it correctly. */
   {
-    smartlist_add(args, tor_strdup("1"));
-    smartlist_add(args,
-               tor_strdup(crypto_digest_algorithm_get_name(our_commit->alg)));
-    smartlist_add(args, tor_strdup(sr_commit_get_rsa_fpr(our_commit)));
-    smartlist_add(args, tor_strdup(our_commit->encoded_commit));
-    smartlist_add(args, tor_strdup(our_commit->encoded_reveal));
+    smartlist_add_strdup(args, "1");
+    smartlist_add_strdup(args,
+               crypto_digest_algorithm_get_name(our_commit->alg));
+    smartlist_add_strdup(args, sr_commit_get_rsa_fpr(our_commit));
+    smartlist_add_strdup(args, our_commit->encoded_commit);
+    smartlist_add_strdup(args, our_commit->encoded_reveal);
     parsed_commit = sr_parse_commit(args);
     tt_assert(parsed_commit);
     /* That parsed commit should be _EXACTLY_ like our original commit (we
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index 7c9f0b1cc2..e640702499 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 62ff12fe15..bb1be11f2b 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c
new file mode 100644
index 0000000000..19e5de4ea3
--- /dev/null
+++ b/src/test/test_storagedir.c
@@ -0,0 +1,375 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "or.h"
+#include "storagedir.h"
+#include "test.h"
+
+#ifdef HAVE_UTIME_H
+#include <utime.h>
+#endif
+
+static void
+test_storagedir_empty(void *arg)
+{
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  (void)arg;
+
+  tt_int_op(FN_NOENT, OP_EQ, file_status(dirname));
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tt_int_op(FN_DIR, OP_EQ, file_status(dirname));
+
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+  storage_dir_free(d);
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tt_int_op(FN_DIR, OP_EQ, file_status(dirname));
+
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+}
+
+static void
+test_storagedir_basic(void *arg)
+{
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *junk = NULL, *bytes = NULL;
+  const size_t junklen = 1024;
+  char *fname1 = NULL, *fname2 = NULL;
+  const char hello_str[] = "then what are we but cold, alone ... ?";
+  tor_mmap_t *mapping = NULL;
+  (void)arg;
+
+  junk = tor_malloc(junklen);
+  crypto_rand((void*)junk, junklen);
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+  int r;
+  r = storage_dir_save_string_to_file(d, hello_str, 1, &fname1);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(fname1, OP_NE, NULL);
+  tt_u64_op(strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+
+  r = storage_dir_save_bytes_to_file(d, junk, junklen, 1, &fname2);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(fname2, OP_NE, NULL);
+
+  tt_str_op(fname1, OP_NE, fname2);
+
+  tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname1));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname2));
+
+  storage_dir_free(d);
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+  tt_int_op(2, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(junklen + strlen(hello_str), OP_EQ, storage_dir_get_usage(d));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname1));
+  tt_assert(smartlist_contains_string(storage_dir_list(d), fname2));
+
+  size_t n;
+  bytes = storage_dir_read(d, fname2, 1, &n);
+  tt_assert(bytes);
+  tt_u64_op(n, OP_EQ, junklen);
+  tt_mem_op(bytes, OP_EQ, junk, junklen);
+
+  mapping = storage_dir_map(d, fname1);
+  tt_assert(mapping);
+  tt_u64_op(mapping->size, OP_EQ, strlen(hello_str));
+  tt_mem_op(mapping->data, OP_EQ, hello_str, strlen(hello_str));
+
+ done:
+  tor_free(dirname);
+  tor_free(junk);
+  tor_free(bytes);
+  tor_munmap_file(mapping);
+  storage_dir_free(d);
+  tor_free(fname1);
+  tor_free(fname2);
+}
+
+static void
+test_storagedir_deletion(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  char *fn1 = NULL, *fn2 = NULL;
+  char *bytes = NULL;
+  int r;
+  const char str1[] = "There are nine and sixty ways to disguise communiques";
+  const char str2[] = "And rather more than one of them is right";
+
+  // Make sure the directory is there. */
+  d = storage_dir_new(dirname, 10);
+  storage_dir_free(d);
+  d = NULL;
+
+  tor_asprintf(&fn1, "%s/1007", dirname);
+  r = write_str_to_file(fn1, str1, 0);
+  tt_int_op(r, OP_EQ, 0);
+
+  tor_asprintf(&fn2, "%s/1003.tmp", dirname);
+  r = write_str_to_file(fn2, str2, 0);
+  tt_int_op(r, OP_EQ, 0);
+
+  // The tempfile should be deleted the next time we list the directory.
+  d = storage_dir_new(dirname, 10);
+  tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d));
+  tt_int_op(FN_FILE, OP_EQ, file_status(fn1));
+  tt_int_op(FN_NOENT, OP_EQ, file_status(fn2));
+
+  bytes = (char*) storage_dir_read(d, "1007", 1, NULL);
+  tt_str_op(bytes, OP_EQ, str1);
+
+  // Should have no effect; file already gone.
+  storage_dir_remove_file(d, "1003.tmp");
+  tt_int_op(1, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(strlen(str1), OP_EQ, storage_dir_get_usage(d));
+
+  // Actually remove a file.
+  storage_dir_remove_file(d, "1007");
+  tt_int_op(FN_NOENT, OP_EQ, file_status(fn1));
+  tt_int_op(0, OP_EQ, smartlist_len(storage_dir_list(d)));
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  tor_free(fn1);
+  tor_free(fn2);
+  storage_dir_free(d);
+  tor_free(bytes);
+}
+
+static void
+test_storagedir_full(void *arg)
+{
+  (void)arg;
+
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  const char str[] = "enemies of the peephole";
+  int r;
+
+  d = storage_dir_new(dirname, 3);
+  tt_assert(d);
+
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  // These should fail!
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, -1);
+  r = storage_dir_save_string_to_file(d, str, 1, NULL);
+  tt_int_op(r, OP_EQ, -1);
+
+  tt_u64_op(strlen(str) * 3, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  storage_dir_free(d);
+}
+
+static void
+test_storagedir_cleaning(void *arg)
+{
+  (void)arg;
+
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  const char str[] =
+    "On a mountain halfway between Reno and Rome / "
+    "We have a machine in a plexiglass dome / "
+    "Which listens and looks into everyone's home."
+    " -- Dr. Seuss";
+  char *fns[8];
+  int r, i;
+
+  memset(fns, 0, sizeof(fns));
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  for (i = 0; i < 8; ++i) {
+    r = storage_dir_save_string_to_file(d, str+i*2, 1, &fns[i]);
+    tt_int_op(r, OP_EQ, 0);
+  }
+
+  /* Now we're going to make sure all the files have distinct mtimes. */
+  time_t now = time(NULL);
+  struct utimbuf ub;
+  ub.actime = now;
+  ub.modtime = now - 1000;
+  for (i = 0; i < 8; ++i) {
+    char *f = NULL;
+    tor_asprintf(&f, "%s/%s", dirname, fns[i]);
+    r = utime(f, &ub);
+    tor_free(f);
+    tt_int_op(r, OP_EQ, 0);
+    ub.modtime += 5;
+  }
+
+  const uint64_t usage_orig = storage_dir_get_usage(d);
+  /* No changes needed if we are already under target. */
+  storage_dir_shrink(d, 1024*1024, 0);
+  tt_u64_op(usage_orig, OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of at least one byte.  This will delete fns[0]. */
+  storage_dir_shrink(d, usage_orig - 1, 0);
+  tt_u64_op(usage_orig, OP_GT, storage_dir_get_usage(d));
+  tt_u64_op(usage_orig - strlen(str), OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of at least two files.  This will delete fns[1] and fns[2]. */
+  storage_dir_shrink(d, 1024*1024, 2);
+  tt_u64_op(usage_orig - strlen(str)*3 + 6, OP_EQ, storage_dir_get_usage(d));
+
+  /* Get rid of everything. */
+  storage_dir_remove_all(d);
+  tt_u64_op(0, OP_EQ, storage_dir_get_usage(d));
+
+ done:
+  tor_free(dirname);
+  storage_dir_free(d);
+  for (i = 0; i < 8; ++i) {
+    tor_free(fns[i]);
+  }
+}
+
+static void
+test_storagedir_save_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL;
+  char *fname = NULL;
+  uint8_t *saved = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  crypto_rand((char *)inp, 8192);
+
+  config_line_append(&labels, "Foo", "bar baz");
+  config_line_append(&labels, "quux", "quuzXxz");
+  const char expected[] =
+    "Foo bar baz\n"
+    "quux quuzXxz\n";
+
+  int r = storage_dir_save_labeled_to_file(d, labels, inp, 8192, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  size_t n;
+  saved = storage_dir_read(d, fname, 1, &n);
+  tt_assert(memchr(saved, '\0', n));
+  tt_str_op((char*)saved, OP_EQ, expected); /* NUL guarantees strcmp works */
+  tt_mem_op(saved+strlen(expected)+1, OP_EQ, inp, 8192);
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  tor_free(saved);
+}
+
+static void
+test_storagedir_read_labeled(void *arg)
+{
+  (void)arg;
+  char *dirname = tor_strdup(get_fname_rnd("store_dir"));
+  storage_dir_t *d = NULL;
+  uint8_t *inp = tor_malloc_zero(8192);
+  config_line_t *labels = NULL, *labels2 = NULL;
+  char *fname = NULL;
+  tor_mmap_t *map = NULL;
+  uint8_t *as_read = NULL;
+
+  d = storage_dir_new(dirname, 10);
+  tt_assert(d);
+
+  tor_snprintf((char*)inp, 8192,
+               "Hello world\n"
+               "This is a test\n"
+               "Yadda yadda.\n");
+  size_t bodylen = 8192 - strlen((char*)inp) - 1;
+  crypto_rand((char *)inp+strlen((char*)inp)+1, bodylen);
+
+  int r = storage_dir_save_bytes_to_file(d, inp, 8192, 1, &fname);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* Try mapping */
+  const uint8_t *datap = NULL;
+  size_t sz = 0;
+  map = storage_dir_map_labeled(d, fname, &labels, &datap, &sz);
+  tt_assert(map);
+  tt_assert(datap);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(datap, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(labels);
+  tt_str_op(labels->key, OP_EQ, "Hello");
+  tt_str_op(labels->value, OP_EQ, "world");
+  tt_assert(labels->next);
+  tt_str_op(labels->next->key, OP_EQ, "This");
+  tt_str_op(labels->next->value, OP_EQ, "is a test");
+  tt_assert(labels->next->next);
+  tt_str_op(labels->next->next->key, OP_EQ, "Yadda");
+  tt_str_op(labels->next->next->value, OP_EQ, "yadda.");
+  tt_assert(labels->next->next->next == NULL);
+
+  /* Try reading this time. */
+  sz = 0;
+  as_read = storage_dir_read_labeled(d, fname, &labels2, &sz);
+  tt_assert(as_read);
+  tt_u64_op(sz, OP_EQ, bodylen);
+  tt_mem_op(as_read, OP_EQ, inp+strlen((char*)inp)+1, bodylen);
+  tt_assert(config_lines_eq(labels, labels2));
+
+ done:
+  storage_dir_free(d);
+  tor_free(dirname);
+  tor_free(inp);
+  tor_free(fname);
+  config_free_lines(labels);
+  config_free_lines(labels2);
+  tor_munmap_file(map);
+  tor_free(as_read);
+}
+
+#define ENT(name)                                               \
+  { #name, test_storagedir_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t storagedir_tests[] = {
+  ENT(empty),
+  ENT(basic),
+  ENT(deletion),
+  ENT(full),
+  ENT(cleaning),
+  ENT(save_labeled),
+  ENT(read_labeled),
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index e12205bb2e..53de793fe8 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2016, The Tor Project, Inc. */
+/* Copyright (c) 2015-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
diff --git a/src/test/test_threads.c b/src/test/test_threads.c
index ebbc95c7ca..18a9407ff7 100644
--- a/src/test/test_threads.c
+++ b/src/test/test_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 47455cff83..7aa3051464 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -1,7 +1,8 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
+#define TORTLS_OPENSSL_PRIVATE
 #define LOG_PRIVATE
 #include "orconfig.h"
 
@@ -1091,13 +1092,13 @@ test_tortls_check_lifetime(void *ignored)
   time_t now = time(NULL);
 
   tls = tor_malloc_zero(sizeof(tor_tls_t));
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, 0);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   tls->ssl = tor_malloc_zero(sizeof(SSL));
   tls->ssl->session = tor_malloc_zero(sizeof(SSL_SESSION));
   tls->ssl->session->peer = validCert;
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, 0);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   ASN1_STRING_free(validCert->cert_info->validity->notBefore);
@@ -1105,10 +1106,10 @@ test_tortls_check_lifetime(void *ignored)
   ASN1_STRING_free(validCert->cert_info->validity->notAfter);
   validCert->cert_info->validity->notAfter = ASN1_TIME_set(NULL, now+60);
 
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, 0, -1000);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), 0, -1000);
   tt_int_op(ret, OP_EQ, -1);
 
-  ret = tor_tls_check_lifetime(LOG_WARN, tls, -1000, 0);
+  ret = tor_tls_check_lifetime(LOG_WARN, tls, time(NULL), -1000, 0);
   tt_int_op(ret, OP_EQ, -1);
 
  done:
@@ -2658,18 +2659,18 @@ test_tortls_cert_is_valid(void *ignored)
   tor_x509_cert_t *cert = NULL, *scert = NULL;
 
   scert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
 
   cert = tor_malloc_zero(sizeof(tor_x509_cert_t));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(scert);
   tor_free(cert);
 
   cert = tor_x509_cert_new(read_cert_from(validCertString));
   scert = tor_x509_cert_new(read_cert_from(caCertString));
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 1);
 
 #ifndef OPENSSL_OPAQUE
@@ -2680,7 +2681,7 @@ test_tortls_cert_is_valid(void *ignored)
   ASN1_TIME_free(cert->cert->cert_info->validity->notAfter);
   cert->cert->cert_info->validity->notAfter =
     ASN1_TIME_set(NULL, time(NULL)-1000000);
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
 
   tor_x509_cert_free(cert);
@@ -2689,7 +2690,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   X509_PUBKEY_free(cert->cert->cert_info->key);
   cert->cert->cert_info->key = NULL;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
   tt_int_op(ret, OP_EQ, 0);
 #endif
 
@@ -2700,7 +2701,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   /* This doesn't actually change the key in the cert. XXXXXX */
   BN_one(EVP_PKEY_get1_RSA(X509_get_pubkey(cert->cert))->n);
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
   tt_int_op(ret, OP_EQ, 0);
 
   tor_x509_cert_free(cert);
@@ -2709,7 +2710,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   /* This doesn't actually change the key in the cert. XXXXXX */
   X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 1);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 1);
   tt_int_op(ret, OP_EQ, 0);
 
   tor_x509_cert_free(cert);
@@ -2718,7 +2719,7 @@ test_tortls_cert_is_valid(void *ignored)
   scert = tor_x509_cert_new(read_cert_from(caCertString));
   /* This doesn't actually change the key in the cert. XXXXXX */
   X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 1);
 
   tor_x509_cert_free(cert);
@@ -2728,7 +2729,7 @@ test_tortls_cert_is_valid(void *ignored)
   /* This doesn't actually change the key in the cert. XXXXXX */
   X509_get_pubkey(cert->cert)->type = EVP_PKEY_EC;
   X509_get_pubkey(cert->cert)->ameth = NULL;
-  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, 0);
+  ret = tor_tls_cert_is_valid(LOG_WARN, cert, scert, time(NULL), 0);
   tt_int_op(ret, OP_EQ, 0);
 #endif
 
diff --git a/src/test/test_util.c b/src/test/test_util.c
index 0b707caeeb..fa6ce1dc85 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -9,6 +9,7 @@
 #define CONTROL_PRIVATE
 #define UTIL_PRIVATE
 #include "or.h"
+#include "buffers.h"
 #include "config.h"
 #include "control.h"
 #include "test.h"
@@ -1059,6 +1060,23 @@ test_util_time(void *arg)
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04 00:48:22.100", &t_res));
   tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04 00:48:22XYZ", &t_res));
 
+  /* but... that _is_ acceptable if we aren't being strict. */
+  t_res = 0;
+  i = parse_iso_time_("2004-08-04 00:48:22XYZ", &t_res, 0, 0);
+  tt_int_op(0,OP_EQ, i);
+  tt_int_op(t_res,OP_EQ, (time_t)1091580502UL);
+
+  /* try nospace variant. */
+  t_res = 0;
+  i = parse_iso_time_nospace("2004-08-04T00:48:22", &t_res);
+  tt_int_op(0,OP_EQ, i);
+  tt_int_op(t_res,OP_EQ, (time_t)1091580502UL);
+
+  tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04T00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time_nospace("2004-08-04 00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time("2004-08-04x00:48:22", &t_res));
+  tt_int_op(-1,OP_EQ, parse_iso_time_nospace("2004-08-04x00:48:22", &t_res));
+
   /* Test tor_gettimeofday */
 
   end.tv_sec = 4;
@@ -2224,114 +2242,220 @@ test_util_pow2(void *arg)
   ;
 }
 
-/** Run unit tests for compression functions */
 static void
-test_util_gzip(void *arg)
+test_util_compress_impl(compress_method_t method)
 {
-  char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
-  const char *ccp2;
+  char *buf1=NULL, *buf2=NULL, *buf3=NULL;
   size_t len1, len2;
-  tor_zlib_state_t *state = NULL;
-
-  (void)arg;
-  buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
-  tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
 
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                               GZIP_METHOD));
-  tt_assert(buf2);
-  tt_assert(len1 < strlen(buf1));
-  tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD);
+  tt_assert(tor_compress_supports_method(method));
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1,
-                                 GZIP_METHOD, 1, LOG_INFO));
-  tt_assert(buf3);
-  tt_int_op(strlen(buf1) + 1,OP_EQ, len2);
-  tt_str_op(buf1,OP_EQ, buf3);
+  if (method != NO_METHOD) {
+    tt_assert(tor_compress_version_str(method) != NULL);
+    tt_assert(tor_compress_header_version_str(method) != NULL);
+  }
 
-  tor_free(buf2);
-  tor_free(buf3);
+  buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ");
+  tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD);
 
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                                 ZLIB_METHOD));
-  tt_assert(buf2);
-  tt_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD);
+  tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, method));
+  tt_assert(buf2 != NULL);
+  if (method == NO_METHOD) {
+    // The identity transform doesn't actually compress, and it isn't
+    // detectable as "the identity transform."
+    tt_int_op(len1, OP_EQ, strlen(buf1)+1);
+    tt_int_op(detect_compression_method(buf2, len1), OP_EQ, UNKNOWN_METHOD);
+  } else {
+    tt_int_op(len1, OP_LT, strlen(buf1));
+    tt_int_op(detect_compression_method(buf2, len1), OP_EQ, method);
+  }
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1,
-                                   ZLIB_METHOD, 1, LOG_INFO));
-  tt_assert(buf3);
-  tt_int_op(strlen(buf1) + 1,OP_EQ, len2);
-  tt_str_op(buf1,OP_EQ, buf3);
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, method, 1, LOG_INFO));
+  tt_assert(buf3 != NULL);
+  tt_int_op(strlen(buf1) + 1, OP_EQ, len2);
+  tt_str_op(buf1, OP_EQ, buf3);
+  tt_int_op(buf3[len2], OP_EQ, 0);
 
   /* Check whether we can uncompress concatenated, compressed strings. */
   tor_free(buf3);
   buf2 = tor_reallocarray(buf2, len1, 2);
   memcpy(buf2+len1, buf2, len1);
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2,
-                                   ZLIB_METHOD, 1, LOG_INFO));
-  tt_int_op((strlen(buf1)+1)*2,OP_EQ, len2);
-  tt_mem_op(buf3,OP_EQ,
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, method, 1, LOG_INFO));
+  tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2);
+  tt_mem_op(buf3, OP_EQ,
              "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0"
              "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0",
              (strlen(buf1)+1)*2);
+  tt_int_op(buf3[len2], OP_EQ, 0);
+
+  /* Check whether we can uncompress partial strings */
 
   tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
 
-  /* Check whether we can uncompress partial strings. */
-  buf1 =
-    tor_strdup("String with low redundancy that won't be compressed much.");
-  tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1,
-                                 ZLIB_METHOD));
-  tt_assert(len1>16);
-  /* when we allow an incomplete string, we should succeed.*/
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16,
-                                  ZLIB_METHOD, 0, LOG_INFO));
-  tt_assert(len2 > 5);
-  buf3[len2]='\0';
-  tt_assert(!strcmpstart(buf1, buf3));
-
-  /* when we demand a complete string, this must fail. */
+  size_t b1len = 1<<10;
+  if (method == ZSTD_METHOD) {
+    // zstd needs a big input before it starts generating output that it
+    // can partially decompress.
+    b1len = 1<<18;
+  }
+  buf1 = tor_malloc(b1len);
+  crypto_rand(buf1, b1len);
+  tt_assert(!tor_compress(&buf2, &len1, buf1, b1len, method));
+  tt_int_op(len1, OP_GT, 16);
+  /* when we allow an incomplete output we should succeed.*/
+  tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16,
+                            method, 0, LOG_INFO));
+  tt_int_op(len2, OP_GT, 5);
+  tt_int_op(len2, OP_LE, len1);
+  tt_assert(fast_memeq(buf1, buf3, len2));
+  tt_int_op(buf3[len2], OP_EQ, 0);
+
+  /* when we demand a complete output from a real compression method, this
+   * must fail. */
   tor_free(buf3);
-  tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16,
-                                 ZLIB_METHOD, 1, LOG_INFO));
-  tt_assert(!buf3);
+  if (method != NO_METHOD) {
+    tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16,
+                             method, 1, LOG_INFO));
+    tt_assert(buf3 == NULL);
+  }
 
-  /* Now, try streaming compression. */
+ done:
   tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
-  state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION);
+}
+
+static void
+test_util_compress_stream_impl(compress_method_t method,
+                               compression_level_t level)
+{
+  char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2;
+  const char *ccp2;
+  size_t len1, len2;
+
+  tor_compress_state_t *state = NULL;
+  state = tor_compress_new(1, method, level);
   tt_assert(state);
   cp1 = buf1 = tor_malloc(1024);
   len1 = 1024;
   ccp2 = "ABCDEFGHIJABCDEFGHIJ";
   len2 = 21;
-  tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0)
-              == TOR_ZLIB_OK);
-  tt_int_op(0,OP_EQ, len2); /* Make sure we compressed it all. */
+  tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0),
+            OP_EQ, TOR_COMPRESS_OK);
+  tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */
   tt_assert(cp1 > buf1);
 
   len2 = 0;
   cp2 = cp1;
-  tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1)
-              == TOR_ZLIB_DONE);
-  tt_int_op(0,OP_EQ, len2);
-  tt_assert(cp1 > cp2); /* Make sure we really added something. */
+  tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1),
+            OP_EQ, TOR_COMPRESS_DONE);
+  tt_int_op(0, OP_EQ, len2);
+  if (method == NO_METHOD) {
+    tt_ptr_op(cp1, OP_EQ, cp2);
+  } else {
+    tt_assert(cp1 > cp2); /* Make sure we really added something. */
+  }
 
-  tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1,
-                                  ZLIB_METHOD, 1, LOG_WARN));
+  tt_int_op(tor_compress_state_size(state), OP_GT, 0);
+
+  tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1,
+                            method, 1, LOG_WARN));
   /* Make sure it compressed right. */
   tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ");
-  tt_int_op(21,OP_EQ, len2);
+  tt_int_op(21, OP_EQ, len2);
 
  done:
   if (state)
-    tor_zlib_free(state);
+    tor_compress_free(state);
+  tor_free(buf1);
   tor_free(buf2);
   tor_free(buf3);
-  tor_free(buf1);
+}
+
+/** Run unit tests for compression functions */
+static void
+test_util_compress(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  compression_level_t levels[] = {
+    BEST_COMPRESSION,
+    HIGH_COMPRESSION,
+    MEDIUM_COMPRESSION,
+    LOW_COMPRESSION
+  };
+
+  test_util_compress_impl(method);
+
+  for (unsigned l = 0; l < ARRAY_LENGTH(levels); ++l) {
+    compression_level_t level = levels[l];
+    test_util_compress_stream_impl(method, level);
+  }
+ done:
+  ;
+}
+
+static void
+test_util_decompress_concatenated_impl(compress_method_t method)
+{
+  char input[4096];
+  char *c1 = NULL, *c2 = NULL, *c3 = NULL;
+  char *result = NULL;
+  size_t sz1, sz2, sz3, szr;
+  int r;
+
+  crypto_rand(input, sizeof(input));
+
+  /* Compress the input in two chunks. */
+  r = tor_compress(&c1, &sz1, input, 2048, method);
+  tt_int_op(r, OP_EQ, 0);
+  r = tor_compress(&c2, &sz2, input+2048, 2048, method);
+  tt_int_op(r, OP_EQ, 0);
+
+  /* concatenate the chunks. */
+  sz3 = sz1 + sz2;
+  c3 = tor_malloc(sz3);
+  memcpy(c3, c1, sz1);
+  memcpy(c3+sz1, c2, sz2);
+
+  /* decompress the concatenated result */
+  r = tor_uncompress(&result, &szr, c3, sz3, method, 0, LOG_WARN);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(szr, OP_EQ, sizeof(input));
+  tt_mem_op(result, OP_EQ, input, sizeof(input));
+
+ done:
+  tor_free(c1);
+  tor_free(c2);
+  tor_free(c3);
+  tor_free(result);
+}
+
+static void
+test_util_decompress_concatenated(void *arg)
+{
+  const char *methodname = arg;
+  tt_assert(methodname);
+
+  compress_method_t method = compression_method_get_by_name(methodname);
+  tt_int_op(method, OP_NE, UNKNOWN_METHOD);
+  if (! tor_compress_supports_method(method)) {
+    tt_skip();
+  }
+
+  test_util_decompress_concatenated_impl(method);
+ done:
+  ;
 }
 
 static void
@@ -2346,44 +2470,44 @@ test_util_gzip_compression_bomb(void *arg)
   char *one_mb = tor_malloc_zero(one_million);
   char *result = NULL;
   size_t result_len = 0;
-  tor_zlib_state_t *state = NULL;
+  tor_compress_state_t *state = NULL;
 
   /* Make sure we can't produce a compression bomb */
   setup_full_capture_of_logs(LOG_WARN);
-  tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len,
-                                         one_mb, one_million,
-                                         ZLIB_METHOD));
+  tt_int_op(-1, OP_EQ, tor_compress(&result, &result_len,
+                                    one_mb, one_million,
+                                    ZLIB_METHOD));
   expect_single_log_msg_containing(
          "We compressed something and got an insanely high "
          "compression factor; other Tors would think this "
-         "was a zlib bomb.");
+         "was a compression bomb.");
   teardown_capture_of_logs();
 
   /* Here's a compression bomb that we made manually. */
   const char compression_bomb[1039] =
     { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2,
       0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ };
-  tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len,
-                                           compression_bomb, 1039,
-                                           ZLIB_METHOD, 0, LOG_WARN));
+  tt_int_op(-1, OP_EQ, tor_uncompress(&result, &result_len,
+                                      compression_bomb, 1039,
+                                      ZLIB_METHOD, 0, LOG_WARN));
 
   /* Now try streaming that. */
-  state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
-  tor_zlib_output_t r;
+  state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION);
+  tor_compress_output_t r;
   const char *inp = compression_bomb;
   size_t inlen = 1039;
   do {
     char *outp = one_mb;
     size_t outleft = 4096; /* small on purpose */
-    r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0);
+    r = tor_compress_process(state, &outp, &outleft, &inp, &inlen, 0);
     tt_int_op(inlen, OP_NE, 0);
-  } while (r == TOR_ZLIB_BUF_FULL);
+  } while (r == TOR_COMPRESS_BUFFER_FULL);
 
-  tt_int_op(r, OP_EQ, TOR_ZLIB_ERR);
+  tt_int_op(r, OP_EQ, TOR_COMPRESS_ERROR);
 
  done:
   tor_free(one_mb);
-  tor_zlib_free(state);
+  tor_compress_free(state);
 }
 
 /** Run unit tests for mmap() wrapper functionality. */
@@ -3333,6 +3457,13 @@ test_util_memarea(void *arg)
   void *malloced_ptr = NULL;
   int i;
 
+#ifdef DISABLE_MEMORY_SENTINELS
+  /* If memory sentinels are disabled, this whole module is just an alias for
+     malloc(), which is free to lay out memory most any way it wants. */
+  if (1)
+    tt_skip();
+#endif
+
   (void)arg;
   tt_assert(area);
 
@@ -3926,17 +4057,13 @@ test_util_exit_status(void *ptr)
 #endif
 
 #ifndef _WIN32
-/* Check that fgets with a non-blocking pipe returns partial lines and sets
- * EAGAIN, returns full lines and sets no error, and returns NULL on EOF and
- * sets no error */
 static void
-test_util_fgets_eagain(void *ptr)
+test_util_string_from_pipe(void *ptr)
 {
   int test_pipe[2] = {-1, -1};
-  int retval;
+  int retval = 0;
+  enum stream_status status = IO_STREAM_TERM;
   ssize_t retlen;
-  char *retptr;
-  FILE *test_stream = NULL;
   char buf[4] = { 0 };
 
   (void)ptr;
@@ -3947,91 +4074,115 @@ test_util_fgets_eagain(void *ptr)
   retval = pipe(test_pipe);
   tt_int_op(retval, OP_EQ, 0);
 
-  /* Set up the read-end to be non-blocking */
-  retval = fcntl(test_pipe[0], F_SETFL, O_NONBLOCK);
-  tt_int_op(retval, OP_EQ, 0);
+  /* Send in a string. */
+  retlen = write(test_pipe[1], "ABC", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "ABC");
+  errno = 0;
 
-  /* Open it as a stdio stream */
-  test_stream = fdopen(test_pipe[0], "r");
-  tt_ptr_op(test_stream, OP_NE, NULL);
+  /* Send in a string that contains a nul. */
+  retlen = write(test_pipe[1], "AB\0", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
+  errno = 0;
 
-  /* Send in a partial line */
-  retlen = write(test_pipe[1], "A", 1);
+  /* Send in a string that contains a nul only. */
+  retlen = write(test_pipe[1], "\0", 1);
   tt_int_op(retlen, OP_EQ, 1);
-  retptr = fgets(buf, sizeof(buf), test_stream);
-  tt_int_op(errno, OP_EQ, EAGAIN);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "A");
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "");
   errno = 0;
 
-  /* Send in the rest */
-  retlen = write(test_pipe[1], "B\n", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  /* Send in a string that contains a trailing newline. */
+  retlen = write(test_pipe[1], "AB\n", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
+  errno = 0;
+
+  /* Send in a string that contains a newline only. */
+  retlen = write(test_pipe[1], "\n", 1);
+  tt_int_op(retlen, OP_EQ, 1);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "B\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "");
   errno = 0;
 
-  /* Send in a full line */
-  retlen = write(test_pipe[1], "CD\n", 3);
+  /* Send in a string and check that we nul terminate return values. */
+  retlen = write(test_pipe[1], "AAA", 3);
   tt_int_op(retlen, OP_EQ, 3);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "CD\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AAA");
+  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
   errno = 0;
 
-  /* Send in a partial line */
-  retlen = write(test_pipe[1], "E", 1);
+  retlen = write(test_pipe[1], "B", 1);
   tt_int_op(retlen, OP_EQ, 1);
-  retptr = fgets(buf, sizeof(buf), test_stream);
-  tt_int_op(errno, OP_EQ, EAGAIN);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "E");
+
+  memset(buf, '\xff', sizeof(buf));
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
+  tt_int_op(errno, OP_EQ, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "B");
+  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
   errno = 0;
 
-  /* Send in the rest */
-  retlen = write(test_pipe[1], "F\n", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  /* Send in multiple lines. */
+  retlen = write(test_pipe[1], "A\nB", 3);
+  tt_int_op(retlen, OP_EQ, 3);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "F\n");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "A\nB");
   errno = 0;
 
-  /* Send in a full line and close */
-  retlen = write(test_pipe[1], "GH", 2);
+  /* Send in a line and close */
+  retlen = write(test_pipe[1], "AB", 2);
   tt_int_op(retlen, OP_EQ, 2);
   retval = close(test_pipe[1]);
   tt_int_op(retval, OP_EQ, 0);
   test_pipe[1] = -1;
-  retptr = fgets(buf, sizeof(buf), test_stream);
+
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, buf);
-  tt_str_op(buf, OP_EQ, "GH");
+  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
+  tt_str_op(buf, OP_EQ, "AB");
   errno = 0;
 
   /* Check for EOF */
-  retptr = fgets(buf, sizeof(buf), test_stream);
+  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
   tt_int_op(errno, OP_EQ, 0);
-  tt_ptr_op(retptr, OP_EQ, NULL);
-  retval = feof(test_stream);
-  tt_int_op(retval, OP_NE, 0);
+  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
   errno = 0;
 
-  /* Check that buf is unchanged according to C99 and C11 */
-  tt_str_op(buf, OP_EQ, "GH");
-
  done:
-  if (test_stream != NULL)
-    fclose(test_stream);
   if (test_pipe[0] != -1)
     close(test_pipe[0]);
   if (test_pipe[1] != -1)
     close(test_pipe[1]);
 }
-#endif
+
+#endif // _WIN32
 
 /**
  * Test for format_hex_number_sigsafe()
@@ -5085,7 +5236,8 @@ test_util_socket(void *arg)
 
   fd1 = tor_open_socket_with_extensions(domain, SOCK_STREAM, 0, 0, 0);
   int err = tor_socket_errno(fd1);
-  if (fd1 < 0 && err == SOCK_ERRNO(EPROTONOSUPPORT)) {
+  if (fd1 < 0 && (err == SOCK_ERRNO(EPROTONOSUPPORT) ||
+                  err == SOCK_ERRNO(EAFNOSUPPORT))) {
     /* Assume we're on an IPv4-only or IPv6-only system, and give up now. */
     goto done;
   }
@@ -5473,26 +5625,26 @@ test_util_calloc_check(void *arg)
 {
   (void) arg;
   /* Easy cases that are good. */
-  tt_assert(size_mul_check__(0,0));
-  tt_assert(size_mul_check__(0,100));
-  tt_assert(size_mul_check__(100,0));
-  tt_assert(size_mul_check__(100,100));
+  tt_assert(size_mul_check(0,0));
+  tt_assert(size_mul_check(0,100));
+  tt_assert(size_mul_check(100,0));
+  tt_assert(size_mul_check(100,100));
 
   /* Harder cases that are still good. */
-  tt_assert(size_mul_check__(SIZE_MAX, 1));
-  tt_assert(size_mul_check__(1, SIZE_MAX));
-  tt_assert(size_mul_check__(SIZE_MAX / 10, 9));
-  tt_assert(size_mul_check__(11, SIZE_MAX / 12));
+  tt_assert(size_mul_check(SIZE_MAX, 1));
+  tt_assert(size_mul_check(1, SIZE_MAX));
+  tt_assert(size_mul_check(SIZE_MAX / 10, 9));
+  tt_assert(size_mul_check(11, SIZE_MAX / 12));
   const size_t sqrt_size_max_p1 = ((size_t)1) << (sizeof(size_t) * 4);
-  tt_assert(size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1 - 1));
+  tt_assert(size_mul_check(sqrt_size_max_p1, sqrt_size_max_p1 - 1));
 
   /* Cases that overflow */
-  tt_assert(! size_mul_check__(SIZE_MAX, 2));
-  tt_assert(! size_mul_check__(2, SIZE_MAX));
-  tt_assert(! size_mul_check__(SIZE_MAX / 10, 11));
-  tt_assert(! size_mul_check__(11, SIZE_MAX / 10));
-  tt_assert(! size_mul_check__(SIZE_MAX / 8, 9));
-  tt_assert(! size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1));
+  tt_assert(! size_mul_check(SIZE_MAX, 2));
+  tt_assert(! size_mul_check(2, SIZE_MAX));
+  tt_assert(! size_mul_check(SIZE_MAX / 10, 11));
+  tt_assert(! size_mul_check(11, SIZE_MAX / 10));
+  tt_assert(! size_mul_check(SIZE_MAX / 8, 9));
+  tt_assert(! size_mul_check(sqrt_size_max_p1, sqrt_size_max_p1));
 
  done:
   ;
@@ -5623,12 +5775,126 @@ test_util_monotonic_time_ratchet(void *arg)
   ;
 }
 
+static void
+test_util_htonll(void *arg)
+{
+  (void)arg;
+#ifdef WORDS_BIGENDIAN
+  const uint64_t res_be = 0x8877665544332211;
+#else
+  const uint64_t res_le = 0x1122334455667788;
+#endif
+
+  tt_u64_op(0, OP_EQ, tor_htonll(0));
+  tt_u64_op(0, OP_EQ, tor_ntohll(0));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_htonll(UINT64_MAX));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_ntohll(UINT64_MAX));
+
+#ifdef WORDS_BIGENDIAN
+  tt_u64_op(res_be, OP_EQ, tor_htonll(0x8877665544332211));
+  tt_u64_op(res_be, OP_EQ, tor_ntohll(0x8877665544332211));
+#else
+  tt_u64_op(res_le, OP_EQ, tor_htonll(0x8877665544332211));
+  tt_u64_op(res_le, OP_EQ, tor_ntohll(0x8877665544332211));
+#endif
+
+ done:
+  ;
+}
+
+static void
+test_util_get_unquoted_path(void *arg)
+{
+  (void)arg;
+
+  char *r = NULL;
+
+  r = get_unquoted_path("\""); // "
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"\"\""); // """
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\\\""); // \"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\\\"\\\""); // \"\"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\C\""); // A\B\C"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C"); // "A\B\C
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\"C\""); // "A\B"C"
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\"C"); // A\B"C
+  tt_ptr_op(r, OP_EQ, NULL);
+  tor_free(r);
+
+  r = get_unquoted_path("");
+  tt_str_op(r, OP_EQ, "");
+  tor_free(r);
+
+  r = get_unquoted_path("\"\""); // ""
+  tt_str_op(r, OP_EQ, "");
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\C"); // A\B\C
+  tt_str_op(r, OP_EQ, "A\\B\\C"); // A\B\C
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C\""); // "A\B\C"
+  tt_str_op(r, OP_EQ, "A\\B\\C"); // A\B\C
+  tor_free(r);
+
+  r = get_unquoted_path("\"\\\""); // "\"
+  tt_str_op(r, OP_EQ, "\\"); // \ /* comment to prevent line continuation */
+  tor_free(r);
+
+  r = get_unquoted_path("\"\\\"\""); // "\""
+  tt_str_op(r, OP_EQ, "\""); // "
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\C\\\"\""); // "A\B\C\""
+  tt_str_op(r, OP_EQ, "A\\B\\C\""); // A\B\C"
+  tor_free(r);
+
+  r = get_unquoted_path("A\\B\\\"C"); // A\B\"C
+  tt_str_op(r, OP_EQ, "A\\B\"C"); // A\B"C
+  tor_free(r);
+
+  r = get_unquoted_path("\"A\\B\\\"C\""); // "A\B\"C"
+  tt_str_op(r, OP_EQ, "A\\B\"C"); // A\B"C
+
+ done:
+  tor_free(r);
+}
+
 #define UTIL_LEGACY(name)                                               \
   { #name, test_util_ ## name , 0, NULL, NULL }
 
 #define UTIL_TEST(name, flags)                          \
   { #name, test_util_ ## name, flags, NULL, NULL }
 
+#define COMPRESS(name, identifier)              \
+  { "compress/" #name, test_util_compress, 0, &passthrough_setup,       \
+    (char*)(identifier) }
+
+#define COMPRESS_CONCAT(name, identifier)                               \
+  { "compress_concat/" #name, test_util_decompress_concatenated, 0,     \
+    &passthrough_setup,                                                 \
+    (char*)(identifier) }
+
 #ifdef _WIN32
 #define UTIL_TEST_NO_WIN(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_TEST_WIN_ONLY(n, f) UTIL_TEST(n, (f))
@@ -5653,7 +5919,16 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(strmisc),
   UTIL_TEST(parse_integer, 0),
   UTIL_LEGACY(pow2),
-  UTIL_LEGACY(gzip),
+  COMPRESS(zlib, "deflate"),
+  COMPRESS(gzip, "gzip"),
+  COMPRESS(lzma, "x-tor-lzma"),
+  COMPRESS(zstd, "x-zstd"),
+  COMPRESS(none, "identity"),
+  COMPRESS_CONCAT(zlib, "deflate"),
+  COMPRESS_CONCAT(gzip, "gzip"),
+  COMPRESS_CONCAT(lzma, "x-tor-lzma"),
+  COMPRESS_CONCAT(zstd, "x-zstd"),
+  COMPRESS_CONCAT(none, "identity"),
   UTIL_TEST(gzip_compression_bomb, TT_FORK),
   UTIL_LEGACY(datadir),
   UTIL_LEGACY(memarea),
@@ -5677,7 +5952,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
   UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(fgets_eagain, 0),
+  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
   UTIL_TEST(join_win_cmdline, 0),
@@ -5716,6 +5991,8 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(calloc_check, 0),
   UTIL_TEST(monotonic_time, 0),
   UTIL_TEST(monotonic_time_ratchet, TT_FORK),
+  UTIL_TEST(htonll, 0),
+  UTIL_TEST(get_unquoted_path, 0),
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index 63a668238c..ea0a86499f 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,25 +11,14 @@
 
 #define NS_MODULE util_format
 
-#if !defined(HAVE_HTONLL) && !defined(htonll)
-#ifdef WORDS_BIGENDIAN
-#define htonll(x) (x)
-#else
-static uint64_t
-htonll(uint64_t a)
-{
-  return htonl((uint32_t)(a>>32)) | (((uint64_t)htonl((uint32_t)a))<<32);
-}
-#endif
-#endif
-
 static void
 test_util_format_unaligned_accessors(void *ignored)
 {
   (void)ignored;
   char buf[9] = "onionsoup"; // 6f6e696f6e736f7570
 
-  tt_u64_op(get_uint64(buf+1), OP_EQ, htonll(U64_LITERAL(0x6e696f6e736f7570)));
+  tt_u64_op(get_uint64(buf+1), OP_EQ,
+      tor_htonll(U64_LITERAL(0x6e696f6e736f7570)));
   tt_uint_op(get_uint32(buf+1), OP_EQ, htonl(0x6e696f6e));
   tt_uint_op(get_uint16(buf+1), OP_EQ, htons(0x6e69));
   tt_uint_op(get_uint8(buf+1), OP_EQ, 0x6e);
@@ -43,7 +32,7 @@ test_util_format_unaligned_accessors(void *ignored)
   set_uint32(buf+1, htonl(0x78696465));
   tt_mem_op(buf, OP_EQ, "oxidestop", 9);
 
-  set_uint64(buf+1, htonll(U64_LITERAL(0x6266757363617465)));
+  set_uint64(buf+1, tor_htonll(U64_LITERAL(0x6266757363617465)));
   tt_mem_op(buf, OP_EQ, "obfuscate", 9);
  done:
   ;
@@ -144,48 +133,54 @@ test_util_format_base64_encode(void *ignored)
 }
 
 static void
-test_util_format_base64_decode_nopad(void *ignored)
+test_util_format_base64_decode_oddsize(void *ignored)
 {
   (void)ignored;
   int res;
   int i;
   char *src;
-  uint8_t *dst, *real_dst;
-  uint8_t expected[] = {0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65};
+  char *dst, real_dst[7];
+  char expected[] = {0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65};
   char real_src[] = "ZXhhbXBsZQ";
+  char expected40[] = "testing40characteroddsizebase64encoding!";
+  char src40[] = "dGVzdGluZzQwY2hhcmFjdGVyb2Rkc2l6ZWJhc2U2NGVuY29kaW5nIQ";
+  char pad40[] = "dGVzdGluZzQwY2hhcmFjdGVyb2Rkc2l6ZWJhc2U2NGVuY29kaW5nIQ==";
 
   src = tor_malloc_zero(256);
   dst = tor_malloc_zero(1000);
-  real_dst = tor_malloc_zero(10);
 
   for (i=0;i<256;i++) {
     src[i] = (char)i;
   }
 
-  res = base64_decode_nopad(dst, 1, src, SIZE_T_CEILING);
-  tt_int_op(res, OP_EQ, -1);
-
-  res = base64_decode_nopad(dst, 1, src, 5);
+  res = base64_decode(dst, 1, src, 5);
   tt_int_op(res, OP_EQ, -1);
 
   const char *s = "SGVsbG8gd29ybGQ";
-  res = base64_decode_nopad(dst, 1000, s, strlen(s));
+  res = base64_decode(dst, 1000, s, strlen(s));
   tt_int_op(res, OP_EQ, 11);
   tt_mem_op(dst, OP_EQ, "Hello world", 11);
 
   s = "T3BhIG11bmRv";
-  res = base64_decode_nopad(dst, 9, s, strlen(s));
+  res = base64_decode(dst, 9, s, strlen(s));
   tt_int_op(res, OP_EQ, 9);
   tt_mem_op(dst, OP_EQ, "Opa mundo", 9);
 
-  res = base64_decode_nopad(real_dst, 10, real_src, 10);
+  res = base64_decode(real_dst, sizeof(real_dst), real_src, 10);
   tt_int_op(res, OP_EQ, 7);
   tt_mem_op(real_dst, OP_EQ, expected, 7);
 
+  res = base64_decode(dst, 40, src40, strlen(src40));
+  tt_int_op(res, OP_EQ, 40);
+  tt_mem_op(dst, OP_EQ, expected40, 40);
+
+  res = base64_decode(dst, 40, pad40, strlen(pad40));
+  tt_int_op(res, OP_EQ, 40);
+  tt_mem_op(dst, OP_EQ, expected40, 40);
+
  done:
   tor_free(src);
   tor_free(dst);
-  tor_free(real_dst);
 }
 
 static void
@@ -207,10 +202,10 @@ test_util_format_base64_decode(void *ignored)
     src[i] = (char)i;
   }
 
-  res = base64_decode(dst, 1, src, SIZE_T_CEILING);
+  res = base64_decode(dst, 1, src, 100);
   tt_int_op(res, OP_EQ, -1);
 
-  res = base64_decode(dst, SIZE_T_CEILING+1, src, 10);
+  res = base64_decode(dst, 1, real_src, 10);
   tt_int_op(res, OP_EQ, -1);
 
   const char *s = "T3BhIG11bmRv";
@@ -378,11 +373,39 @@ test_util_format_base32_decode(void *arg)
   tor_free(dst);
 }
 
+static void
+test_util_format_encoded_size(void *arg)
+{
+  (void)arg;
+  uint8_t inbuf[256];
+  char outbuf[1024];
+  unsigned i;
+
+  crypto_rand((char *)inbuf, sizeof(inbuf));
+  for (i = 0; i <= sizeof(inbuf); ++i) {
+    /* XXXX (Once the return values are consistent, check them too.) */
+
+    base32_encode(outbuf, sizeof(outbuf), (char *)inbuf, i);
+    /* The "+ 1" below is an API inconsistency. */
+    tt_int_op(strlen(outbuf) + 1, OP_EQ, base32_encoded_size(i));
+
+    base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
+    tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
+                  BASE64_ENCODE_MULTILINE);
+    tt_int_op(strlen(outbuf), OP_EQ,
+              base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+  }
+
+ done:
+  ;
+}
+
 struct testcase_t util_format_tests[] = {
   { "unaligned_accessors", test_util_format_unaligned_accessors, 0,
     NULL, NULL },
   { "base64_encode", test_util_format_base64_encode, 0, NULL, NULL },
-  { "base64_decode_nopad", test_util_format_base64_decode_nopad, 0,
+  { "base64_decode_oddsize", test_util_format_base64_decode_oddsize, 0,
     NULL, NULL },
   { "base64_decode", test_util_format_base64_decode, 0, NULL, NULL },
   { "base16_decode", test_util_format_base16_decode, 0, NULL, NULL },
@@ -390,6 +413,7 @@ struct testcase_t util_format_tests[] = {
     NULL, NULL },
   { "base32_decode", test_util_format_base32_decode, 0,
     NULL, NULL },
+  { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
 
diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c
index 4e75b97f3d..70292f2287 100644
--- a/src/test/test_util_process.c
+++ b/src/test/test_util_process.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2016, The Tor Project, Inc. */
+/* Copyright (c) 2010-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define UTIL_PROCESS_PRIVATE
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
index 1e7160598c..3e5d78948d 100644
--- a/src/test/test_util_slow.c
+++ b/src/test/test_util_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -242,7 +242,7 @@ test_util_spawn_background_partial_read_impl(int exit_early)
 #else
     /* Check that we didn't read the end of file last time */
     tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_handle, stdout_buf,
+    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
                               sizeof(stdout_buf) - 1, NULL, &eof);
 #endif
     log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
@@ -273,7 +273,7 @@ test_util_spawn_background_partial_read_impl(int exit_early)
 #else
   if (!eof) {
     /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_handle, stdout_buf,
+    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
                               sizeof(stdout_buf) - 1,
                               process_handle, &eof);
     tt_int_op(0,OP_EQ, pos);
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index ccb8d0c8ca..6fa46f90d4 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "or.h"
@@ -200,7 +200,9 @@ add_work(threadpool_t *tp)
     crypto_rand((char*)w->msg, 20);
     w->msglen = 20;
     ++rsa_sent;
-    return threadpool_queue_work(tp, workqueue_do_rsa, handle_reply, w);
+    return threadpool_queue_work_priority(tp,
+                                          WQ_PRI_MED,
+                                          workqueue_do_rsa, handle_reply, w);
   } else {
     ecdh_work_t *w = tor_malloc_zero(sizeof(*w));
     w->serial = n_sent++;
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index 9c6580f788..d7e36edbc0 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2016, The Tor Project, Inc. */
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 extern const char tor_git_revision[];
@@ -21,6 +21,7 @@ const char tor_git_revision[] = "";
 #include "rephist.h"
 #include "backtrace.h"
 #include "test.h"
+#include "channelpadding.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -38,7 +39,6 @@ const char tor_git_revision[] = "";
 
 #ifdef USE_DMALLOC
 #include <dmalloc.h>
-#include <openssl/crypto.h>
 #include "main.h"
 #endif
 
@@ -178,65 +178,6 @@ remove_directory(void)
   rm_rf(temp_dir);
 }
 
-/** Define this if unit tests spend too much time generating public keys*/
-#define CACHE_GENERATED_KEYS
-
-#define N_PREGEN_KEYS 11
-static crypto_pk_t *pregen_keys[N_PREGEN_KEYS];
-static int next_key_idx;
-
-/** Generate and return a new keypair for use in unit tests.  If we're using
- * the key cache optimization, we might reuse keys. "idx" is ignored.
- * Our only guarantee is that we won't reuse a key till this function has been
- * called several times. The order in which keys are returned is slightly
- * randomized, so that tests that depend on a particular order will not be
- * reliable. */
-crypto_pk_t *
-pk_generate(int idx)
-{
-  (void) idx;
-#ifdef CACHE_GENERATED_KEYS
-  /* Either skip 1 or 2 keys. */
-  next_key_idx += crypto_rand_int_range(1,3);
-  next_key_idx %= N_PREGEN_KEYS;
-  return crypto_pk_dup_key(pregen_keys[next_key_idx]);
-#else
-  crypto_pk_t *result;
-  int res;
-  result = crypto_pk_new();
-  res = crypto_pk_generate_key__real(result);
-  tor_assert(!res);
-  return result;
-#endif
-}
-
-#ifdef CACHE_GENERATED_KEYS
-static int
-crypto_pk_generate_key_with_bits__get_cached(crypto_pk_t *env, int bits)
-{
-  if (bits != 1024)
-    return crypto_pk_generate_key_with_bits__real(env, bits);
-
-  crypto_pk_t *newkey = pk_generate(0);
-  crypto_pk_assign_(env, newkey);
-  crypto_pk_free(newkey);
-  return 0;
-}
-#endif
-
-/** Free all storage used for the cached key optimization. */
-static void
-free_pregenerated_keys(void)
-{
-  unsigned idx;
-  for (idx = 0; idx < N_PREGEN_KEYS; ++idx) {
-    if (pregen_keys[idx]) {
-      crypto_pk_free(pregen_keys[idx]);
-      pregen_keys[idx] = NULL;
-    }
-  }
-}
-
 static void *
 passthrough_test_setup(const struct testcase_t *testcase)
 {
@@ -297,14 +238,15 @@ main(int c, const char **v)
 
 #ifdef USE_DMALLOC
   {
-    int r = CRYPTO_set_mem_ex_functions(tor_malloc_, tor_realloc_, tor_free_);
-    tor_assert(r);
+    int r = crypto_use_tor_alloc_functions();
+    tor_assert(r == 0);
   }
 #endif
 
   update_approx_time(time(NULL));
   options = options_new();
   tor_threads_init();
+  tor_compress_init();
 
   network_init();
 
@@ -363,17 +305,14 @@ main(int c, const char **v)
     tor_free(errmsg);
     return 1;
   }
+
   tor_set_failed_assertion_callback(an_assertion_failed);
 
-#ifdef CACHE_GENERATED_KEYS
-  for (i = 0; i < N_PREGEN_KEYS; ++i) {
-    pregen_keys[i] = crypto_pk_new();
-    int r = crypto_pk_generate_key(pregen_keys[i]);
-    tor_assert(r == 0);
-  }
-  MOCK(crypto_pk_generate_key_with_bits,
-       crypto_pk_generate_key_with_bits__get_cached);
-#endif
+  init_pregenerated_keys();
+
+  channelpadding_new_consensus_params(NULL);
+
+  predicted_ports_init();
 
   atexit(remove_directory);
 
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
new file mode 100644
index 0000000000..5dff233a69
--- /dev/null
+++ b/src/test/testing_rsakeys.c
@@ -0,0 +1,546 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "or.h"
+#include "test.h"
+
+/** Define this if unit tests spend too much time generating public keys.
+ * This module is meant to save time by using a bunch of pregenerated RSA
+keys among */
+#define USE_PREGENERATED_RSA_KEYS
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+
+static const char *PREGEN_KEYS_1024[] = {
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCZa39BCgq7KWBWFSjGYHhqmTCHvQ7WNEFAb9Mujb6Xn/Zy01fu\n"
+"WIpVvqmAKeLNEziItUm/gB8GwAN+/ZLwL9pufjIp2Ar+yqVXKySioZQxuCgTP2wm\n"
+"Ku0OfmAra1Xbtrkc2OCJllxkyNPrJ/kxfwjWR96UP0+VMbOlkBoEH1FtvwIDAQAB\n"
+"AoGAUXoygeMIYe+OdwkTt48CRHKIwH3aRE5KHSOGPyIOB05vvvmYqD8jcHgqYqNc\n"
+"DNdZXdkRin9LevU8phObFq4DTXp08XggUx4Kk4AdsFKubQtJ8gHm3xlSKbZXX2m/\n"
+"ZF0GRaZtVDQ3TRGh+OBLILt/2jT+BaFKGAyJ7al76F2nprECQQDJyLlteLDFBmrd\n"
+"0kAjNBE50S5YskBCQeQACROfyTKW8lG1J57UBeYjXvbrDFBR4alIS9DEexGai9Gz\n"
+"wxpgKg2nAkEAwqQmPstjHxvqGQRi41uXO026MLxY7dhEqs1aSw3tuT8v17pW3OEa\n"
+"Qxv7JINePZ3+sNN+Ic+3RXBR0QuD7lSSKQJAZjVSF21GvMXfY7SX4D0DbLHUNAE2\n"
+"I1mUz5/JXOpgwazETmpfPS4vwELd93kpRhBz2rbsbFmaNRoVgmSU+5jRiQJAZ1bV\n"
+"g2NilgKxEGU2x3U6Xt8Oqo9lO6omEvUCKnUTsNWuZf/l3FGbKuQxO5qPr3Ex5tny\n"
+"zqrEqBZRKgbOHfxCuQJAbJY5C3Nm5koemr031r00MY2YD1b6+hyKZyPdZ21HpyY8\n"
+"z1kWShL0POjYPX/BnKE1FkpklWcKBb7wkK7dvAKkEQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQCyqMM2TfFGV5tVBTVabxLVln8146nDavIdR6q78DCUMh8Zfzkk\n"
+"h9Lbl1NX4RU+AmrCZMPq21/EjIRxRQyRdgPYJVLdp96eGeYnEzmMkqvXiswXvDg/\n"
+"tXqsjyJeYsoHMQWDTpCLfjYo4K1ol1sg8VIs4wQeq5og6QSdmhBoz7MyqQIDAQAB\n"
+"AoGBAIJekey7nZeV8Bxva4ptSRIg+v0I/2VBUiG5nUX9NIW/uV/yrXERx/VDjKaw\n"
+"8b5JJzxpKWnk4RJc83xwRYaT1qMYHiQfybxEI0K9SjhtaThAjtXkQGtZgLJILl3t\n"
+"yh3LPTh1ocwafsKjU6eGYAe/DYn9/QwYHbtyaimcigu4etp9AkEA2DgC+HndoP1i\n"
+"np26Lx+4TG0vAfrVYGSLT9FXwf2iBV3oJvdKqu6wr8ipb1SbshRPcOQd31/mCh6+\n"
+"2BR+d4ddcwJBANOHrlBbGZdHnoEu6kKbPwwkc31IZYqyfSpkqm0Lb2oWZ9SInKfc\n"
+"cz0qpH91p610XUpYmycaJr4K+N8jgrz86HMCQQCoqGBg1Ca2OpCf66bctWB8dTqS\n"
+"z8d7rlIhC8npr1+f0hWRt5pN5Wx7YgoQpq3gZgllpPtMT7DQOhVh1fKkaDnTAkA4\n"
+"XuskPPLX7t0dvhvtviOSH9CrLXTp/mD+wC7uumJpmij3aaSd01DelxOZaAhUYDNQ\n"
+"UcafKAf1E0V5aaQ4qwljAkA9NVN6CtpzzcLrstTKxrx5P1Ylt/0UYQDo1lIaqwrT\n"
+"aOFbXmOungiC9+p/4U7RbX0MEzjFDHCWlaHASviGVgta\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQDDt2V63APj3JSqaRgofUzhtB+prm0wII4uHyxfOxnpYIELOW5z\n"
+"3UHmkr+B4D+Nif5jIp0i6W4OS4S+YHewKsDsXvXKRIW78KzOt6Le4JI9rSarNjy5\n"
+"aJKksWQRALLCmxP/BdolaBFqF3fIPD5+Zxu8ESgxhkEQI4p7awUp3E730QIDAQAB\n"
+"AoGAZktfAR4p8lkCYydW9yK2ommQ+xEuBK+fYL/uYz/yxSYpjIJSFsEYhrlA21Mo\n"
+"JIRxr8MRuoOjgFk8YnztUeimuHpslDlZDaCBzjRjBRFCMepZNG9xqSEL0u7C+SH6\n"
+"KU5f2x2P6PneBj6WaHZM+6Lf2xHlOoeuaVSUfq2Pk2VBF9kCQQDtawWWNwP0+xea\n"
+"oCAQpanaLzYPjlqZfHJQ1AAI5eSkdf1qmlypIHwOtjAEa6XuEO/Or8RNkNy4nQdw\n"
+"qhcQ7PXDAkEA0wjT6Z+Lrt67FnwPgoSvl4Nukcqw4OWHbBKhaQPsO9+oc3PAXLdD\n"
+"SclUUqDF6NX1yONTV1KrPdz4zElmEua+2wJABm4inZnp2oW+cuqpU6oY+pbSwQMb\n"
+"AxMyyWukgJkxYx7q+SsrHU2K7p8Sl9wOh28f/5oVGAC3aayfGfcRXtz8HwJAIqeO\n"
+"dQzYGU1GF7kjquEzHIRewd4xEZ1fkaW1j9MvFd3ygZL+gbsud41yJWd1WHjaNbTu\n"
+"2KYgrLX+vT1IX844hQJAbg0V7iHlttQqXL7yN09jIjQLprqVhDZCUHS9s9Dxe7fz\n"
+"Ac0ZZD0D6EVNmSmBB71q7kLUWX/W/10d447TLnnfew==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDhCAjPEockl4lqkvoIb5O3NJJG8NWD31c63e/cPWY6MX5nOM/q\n"
+"avof2eWJxFOk0HQ2BRVwIgNex6kLxtsdw7XE0A5uZorTp9DbRCGMqUqHNhHH9ci2\n"
+"mMPP9jptq3ieWg310bH4Tad8h3WE2npSCDBvxyV6EmuH2rlQW9ZlHNoiRQIDAQAB\n"
+"AoGBAI4PgWggPTqng7PJF5mNvsYQpSutzE0VCL977nmuNUQVjMPjRLarVD4ZU+QW\n"
+"EevhQQv9R5xjjJcgGqL5pchzjeKDm0/LA+AygnZoDMs2O68Neieqvr7cPqr5ALGs\n"
+"WuZvSn+bRJTenvV9sUh2ii0/u3GQbL1v7GWDkIdD7itDbmRhAkEA8iijuEY+W67w\n"
+"7JusjY2MQ2Cm6xxxR0YcnYPzT6UDm+Z7NNJwKscQ6AjayNmxmXGpbUdukzLzXf8y\n"
+"fccI9t6iHQJBAO3kx9nZay0Ktl51QP5o2gwoqRIbnogGfR06KJOlzIPGR0aPn8cg\n"
+"uKq2SiyjewEaSBM6S/4UlxYUmvc3VKnxCEkCQQDpTjg2YQ7RPGIIRA/iLV7Wx3bq\n"
+"C/QjjCwjoi44LK6mdE9928WPoUzrkSRg4EQYpwZqL6kcDrmkdSuLPMipOGQNAkA3\n"
+"KtzlujPOiDNuiEaAORSHyU4b8ue6p7aP9pK+Wq6oyGxzAo+NABuTCx78ZxT5Vnzs\n"
+"aJKC44d+CV0+g0hQ+KJxAkEAqFYzNWIzTHX8DVDdK9BpUaBg1DFxIeP5Kk+/X3FF\n"
+"5BafG08B6OiLf8qIGGsxLXNRjIE0GVp3Sy23FUKtUymP+A==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDMDk01VwPxQq/BAwOBmfGUP/x5BQn+uxI0Aat6bdWuz/2CsjbS\n"
+"CWD/YLCaPm+DpHp9RMwk4HONJaw4B2XOw3ELPx7y9DEgdC1wZ9wRkJmqr2IJZoZR\n"
+"C7x43nNv+/IXTiRkkljCcMpoL1Tld+L2VbmWR29PdZwvspWRILkEZu1mNwIDAQAB\n"
+"AoGANvFK3KfXSei4xfF3yjeXEmHAKx2uOUZJenNQpqBYPr+F9ODjXd5knZ59LqrM\n"
+"/9cTnBMgHHXK5yBTpKppQSjikLeQ2BF04Ktff9oGqVcS9x/rKo0CREuxsEfawZOW\n"
+"OzOWENp4YcDKGP1I/Ctr185QzStaWrXVQftxmYQ53T77ShECQQDnhabwtqW7rfe4\n"
+"+MfkWEJ9Y2s6iMs3JWnwPOX9G9R39PiAD4vAghHJyHHttS9Ipxmvp0hThu0x7a4g\n"
+"8BfUpqgjAkEA4aFAmzarWKigREAACVTYH2RHpXbuk05vF9WqfMPiEvQUd5a1q6vc\n"
+"xkGZsE3v/TExLjPRZP4FeUNV5sD7THzA3QJBAJxPoRlNx3GCEAlDdfnWGPX9JI09\n"
+"hC40RWUcSI7ttjJTI1+an1kWuBnLChhaRpU/tFjikTNLmmMmPHUihIRfDI8CQG7g\n"
+"3WzpKr8A7vFbOilbxnF2yDaqAYfmTXW7DHMPl/OUetJh/5kDdhT/e9VGF5+nIvH/\n"
+"iPFGW85Bpt8lCtmFnQkCQQDjpp9iy2qesE7KKX4Kv3++QfCJ2w3g7lwg4iyncoDd\n"
+"JrM53p29HROM21R6eekvqeWIe9tEX754b+E/N60ZjpGm\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDdDn3H+Eu0AW5GKohqDBntw6ubnd3VaJwZGzZyga4J2kLg8peP\n"
+"RAW6GDD6pcHzW+KZbFWHtRk70FSwvmyGcf+DY0r5tfyCHyDGmbJyPR0o6OVCgSFl\n"
+"ccf4eDvbyszzMdlx3uL05ABIpCShoKtEUqvyIQla3Jon+QBwuVkizMzyVwIDAQAB\n"
+"AoGACoKh4Fwh3VEkGRn0mnYw1Wk0Q5Xh8j+jDF6K3C7mQ3mpLGDca+dkDlEQIxq2\n"
+"egeoYnsQJf+qT3m8TRsAtfO9nj7+7IX4BfCtdIi4RNcorbs5YMWtFyaywnM6SQjS\n"
+"+1qf74aL4On9WRO2FtvnTMjFAAkiWNbQp7mWwTmB59i620ECQQDwde6/PwhUzvZh\n"
+"dyslKJdna5RjkDQyDIuh0zD/tFZ0Iko7Luec8q6n52ev/n0OiTLGetUh8goePsPP\n"
+"HVZHidNJAkEA61eMCmmu+GCAg2vJRtL5sDakAXsbP5M9Bf/QVHXtc4EVXHC6T2ld\n"
+"bldOJriNbBThBuPNmlQbssn9FApkyWT4nwJBAIuHIv3+CUuMvBJaH8L0BsaP+g67\n"
+"wk24Ud2Yujnl3rSMoR4uXV8IwqfS8quAs/gXTEs3QyzrUUuzh9NKZqIkK2ECQQCz\n"
+"vivBEDKIlPvSZBJYO25kfXcJgoKvLb9fw5/TwjXXD/HGpnpFiI3JZnjT7gRlVhT/\n"
+"9CDmC/MTvF3EXqPXhXy1AkEAo3a2me23Ljmub21jycSKaCk09dK85QTRRMe9c/hs\n"
+"i+pcGi9ZZW0Mm7cyQo47oXjNurkkv0fEvXIobVTEXAGU7w==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQCv8R1IbfYnE3R3kNeezJ7m02XnyCBDDy0YfrQldQ+urdg1CFye\n"
+"bO0iPniJb8fmV8NW7x6nUZTDznCg+igroKXtK/w0WYmJJiH4A7Oi5xNjAfRIPvJ/\n"
+"J5GI8szS8rH8tp8pW1h8k/kNg2pnBjwQ2U9omhp95RGaHDQSRYzzH/fEFQIDAQAB\n"
+"AoGAcy7+BcH/iZuB/xjzIIJDcUhqibCJ9n0D/+pLU85sYuZrCmUcBZe4M1gEn61v\n"
+"iExilRJc1hthskL/l1POYql8lk+aqeeDuh38fWJj60TCV/sENiuXOsTmoFVA5pNn\n"
+"lwlG8JlpBMsgr1fGqg1C/WLFfMmvXdKVGvpRqI06j7AYUa0CQQDfZ5rI+FhXBlxo\n"
+"PR5CM1LB90DuHUMW+Kqoj0c9d2esXEQM7UqQ/9BiBQbL6Py7Z3VwCxibOqyz7+V7\n"
+"2aGUMAKnAkEAyZy5Mu2tHs6YBBxPYam7huzMUYjddN7ixAZUyGwxQp9kTIF2NbSQ\n"
+"yVDjKrco3s2lO4qj4pSumwVe3GGlsi6G4wJAOOS3pIqqZK84BUvbUtyjLMZ9AKbv\n"
+"GQCG5ZpneB3ahyiQJAKiRL8BIJVLH87b3hYA8GHDCHUu2jwz4xCPd5+qbQJAV0TP\n"
+"pYvb9AnZI25drhiaY7z8dA6aTYxs/A0Bhf/PEteLwtIHKRgP1BR/QG4n8slxTGSm\n"
+"q91P9ypL9XkPECGzoQJBAIMvGEM7ZGevQHBjJ8HhU8IsgT4cYH/XEYb8jRy4F+Ui\n"
+"jKxHPxLuFK4urAZunNUNrqhT0PxbB7hRjtHZrmFkrcc=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDSpmV8ncLwc8gXzdFsZGPDtMO7C/IN9jKCIK13WIseMg1APlMt\n"
+"PB5lMQ9fa3m9ZRU0L8HzRo+u/Xdos3yIBI38X2Avy0laGKnQxiOKaDT/5ZHeiBBh\n"
+"nMZjP2WY5V1sgqNP9RD8enE6WaSvq1j0BM++mn9KEe//5+dWD8tboBKF4QIDAQAB\n"
+"AoGBALgVoerdE1Z+WAY1XyaSNHz6o3H6ZnW9CTaex/jb7/dbVikmThnhx842qXCB\n"
+"w8m3ZGhOs/edWkNaTde5wsI6+LhVGco/PWxN4v61jokxUU+5KvUvGacXhXIjzKwG\n"
+"DrNCYmle62QCI1z4+TLQW/Lq+jw2Wzk70NWEvoP58gt5SJoBAkEA9wubRKRs49LW\n"
+"5JNQZ9hjc+mAfP9YK/sMe4jkdloMMWXjSMlF3Z4mI9XQSpfbBqwWIBXsjU/15LIS\n"
+"ftmujZsMKQJBANpJEZI7UFoRdSP7AlM0YJuXWnVGyn/K+VIeEso5AlZdKXCTpxqp\n"
+"9blWq0UVC6jLesZ5UNPuBiAnrBaVwDA8YvkCQF+FQVfdK607TJO80g4VAP9EfcXX\n"
+"BUScIUtytsN8NdKzzpnKGRWDnMOmXI87ABkoWLW3RGuvSyhOIhCiInfmR2ECQASc\n"
+"FmroJcJBLCAeZOYs7P1cLOTdIdmhB7LcP7lVit8YCJAADj9Z536KfgNvdleSNH2M\n"
+"glB3blmvfMrdTrm2DMECQQDj6GJ/Tc2rCsq534xknasVjrgtJMQFxmQCTVgBx9pc\n"
+"gTflJAHAmNDvstacVqeObLCF2ZIvya8fSXGbDOJYeGDv\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDGgUJAm7vf/3focNGwzv4TkzYF2XwpAirnb61dyxvfug1zKv2k\n"
+"AUg3qACiurR7JrI+kAbmxEnNaKV7ts7uO763wP9KE8YAuFZsp7NFA295rEZhw38T\n"
+"rUlWHMCeaZ3mqW2q8gA14C/ZJCG4gS91SIHLjNGsbHwr2Jvri2ItwIP8FQIDAQAB\n"
+"AoGAONceb32oiHWQkkBr6uL6ogRPPdGO2fdC7c5uqCLWsnOGEmpHAsVTNoym0fIA\n"
+"aBsmgv+e2klukKDccdZg3prA+z7lHcc2a4bIFguF6ei80hLIis/dds66fFXofCzy\n"
+"DMlkncSbJwIvQHG9gblxp9qSKElZF7XjABZEImarfUlakGkCQQD//msGy5N0ZhMI\n"
+"yGMXkwXRJXfmRrIrOqHx6u1eUp4OuqDW+hBz4KCHnWfuRJkNGQIammSf18jPasP5\n"
+"YHyr/LifAkEAxoJ8R8Vusexo9ZjuU44qXCSvJQ26UBV7mn6TGEAn2DRK1RWKDaHv\n"
+"j2vnRjt3CO9WPDQL7SB/1HNAy+dIMPyqywJBAIB6tESIz8zPniX+TJ18UKMTZwXP\n"
+"3YQMvVKpUdDRLjq+OBMtFizSRD9MJOlUzGvibUfkzTPcHRDcyNbUMj4vbIkCQBx4\n"
+"6sqAjvgGKKfRX52sbnb47AYsieSisC/gp8h6qzxfg7w8cqix6WJw36M7ND+b1Iqe\n"
+"DHfeiXc3cLvOWJRuKTECQCEYkujtSjXWb26xaESFWGtUI/nEvCyqYPQAFBpaGzQ3\n"
+"tiTDeKHzypesWYoTxOiNQWCQMLrFGuUbDpYOuDOVNjw=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQCcwSAfytnspSSDX/sKmCPOMnpuCYeWA4wbz1wLyb63a8/KXhhG\n"
+"6o2W0kt3x1vnGZkeWwZOeBFUqwoc+xHhoNcZFsMOyqbqA3UMZW5cx27MsexRTQHs\n"
+"Go1newu/E+8NNCohY51G7z1Hdo0L6mi/Tldh7puuGsMwKqNG/Vvo/GQDgwIDAQAB\n"
+"AoGBAIUdpBAbjXDe1OET0vYuOMnUKA/l29RS8tpy/zGrg1/0GCM8QNWIPfEEaL4w\n"
+"+CSKonMazYI5iE4kaZQuygKXOdFqKxX8nrGK2hR0DIEUHhhiqyGMUKrf4ELkAJzK\n"
+"tHtcO64OFEU2EGa72wCmyk2MhqhLxWxA7E00x24uvW6pen6xAkEAzHhbzlRgLZ+K\n"
+"QuXmQHEqkGaS2Ccf6c9TA5Bf5S2/5zBl+OqVyJJQH0yrbPYR6Nn1NeSv3R4IDJYg\n"
+"fSZLaVzWHQJBAMRCU6QtTnZoQ97pLvXCSKRYKJF+CnE3zDFTyoJrpK0W1FSnb1EE\n"
+"DWjjdSdMLynf/InX+VOaLk3Gxwjme4NKjh8CQQCg2b4/HplayrsVzY3I/D2jw02Z\n"
+"xY2RfYusrhMCU284DBbsLn8OfiuRs9rXqOyF5ZDFiNXgeROT8zYzvcBtbp7xAkBU\n"
+"ZET9IvJLXjhZISItUXbVHIeNUIqC9sBaMbKx9EGioF97a2gliT2O7cgRtuPM+ODq\n"
+"ETHILlNc5G3vuNRBt4x3AkBV98Y1SZA3TQlUVTsjGraxkFTfU1IlomiOdOwTQ+xZ\n"
+"x+JxhhgZwZ+kgI3PidEufFCTZJ3WO6Wk9gk18Bx7CLjm\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXgIBAAKBgQDq/K7wNW3fcTbaRTjNZlM4W0G7tKeO+X0bca4+9uin3ML3ogNJ\n"
+"6qT/B0QAZB6Vyi9kKa3E8plQkjmPuX8Q27zj2QjEuDZ12RGFnikeOosUhOYiDh3Z\n"
+"T9CHnr6stozzgk79Xd6VI7bqRcgRwbY0uc9QVr6vwddyIfSploSpVcgspQIDAQAB\n"
+"AoGBAJfUpo/sZc6uzxtfCKGmkPTj+ef3hSBbUZuu60AhtxfnC06HrwpOg0eJAUYj\n"
+"aqOsHMziJTYQ7kDiCjE0UMaqxDNS5hueumznq2xM2mSN0nYoktU00kpANVkW4VPA\n"
+"33TB16DyqlKq2/21Rs1g8/8+IKkKDbRLTC//1WqNHASQVoGNAkEA/+z4hxTVXZkr\n"
+"9hz29tAHKURlqzxUEKLnS0eL+XGJRNfGJ+65eXL+gFiIbTnpVeidL1+lKWkZyYzl\n"
+"75cNRdUHhwJBAOsOJ9mUOqTbLW5tzh18ewZGOa1JcxhOvf2E1d56N8tDK6lvoqkF\n"
+"oUUb8kIweDxPLCVLCl8qFrbjn619fxDInXMCQAfEZGKNIlCd5nSoumIRPDZnagKB\n"
+"aTe8CfMB7+CZLoZVWiE6IIzsDYdNqI5QFKHT1nlqmLOiCfNRAGV+GxwEdB8CQQDE\n"
+"sHu4HclU2fMSTOAE3H01qt3om2WsGXfyBI3SNQMrG3IVvkymkwd4BQKbUGPMU5Pl\n"
+"QP3U1CtdruuXCUSijrzxAkEAoqYub6+0zM8fakSQZcZ01TG9Fuo2xVFDCQsvqR3m\n"
+"ZhRT/oinIvOxSh4fQs40bmt1RBmc2L1Is6YB2NTVQEBZDQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCrf0rPvHYaGYQrc1ciRwaONs8TUvSVmUU98HMYXoFEkBL4CAGH\n"
+"4oNHFk8kXHEOsBED0eccSYegWhqKHSz7PbjmJaXloExWrtx5ea3Twf8VTgcfDWQP\n"
+"0TzD3G1TYjAFPQ1/LAZCpQFmwpMmTGGxegUhOzkpEWXdLVEVc9Uw4C4L2QIDAQAB\n"
+"AoGAZXAJZA5pHM7y6nBynYe9TOkGWru6h7H8zsImkcd0VoWRcrvpi+JjG+0KKsuy\n"
+"46kop0XEmWq0mhgxknfnX0QG1MKTqGMIUGN4qCaezOabIpCOdA4d/pr/mWoNgOWw\n"
+"9Kc/tNCrKxPKsQMAlWP6ktHN30XRSlHgAjSeUVUiNHztvTECQQDUNin2nyIvj8ZA\n"
+"QAsFW9qW+TiTkeUK6yiZ9Gvgf20gwZRWOe5/xnMxVvtN6v7Av1ew/l4VhBoj/w5g\n"
+"ydIZk+2LAkEAzuJwdt+ccllG19qmEcbo9XFafgi2PvlEjPJmT1rHV2ns/7HIMu27\n"
+"PJY36GgExSfFco6VmicaoOt+RKg+5acgqwJBAKQxAEjcGWQ5VsgRhTVxO3DChX7Q\n"
+"TColhrWPwwPhM/s7K92HVzwvvKL5TNmdr9xMb7n3Ja56FouxZVuH6/J0XT8CQAat\n"
+"Mhnz/3WFQg8HRGLAe5YoMVZt64u+uaKe1ARtlo9QoNBjqWVTXL6IzocWjEjcjrey\n"
+"uEtARdC5qNqIX3dD3H8CP3pVCPvpHOTxkUaktmLYowSA1HSfO9wkE6bMCHhkLwXF\n"
+"yTIJ+N7c5u5YN1B6hhVqpKbdnSv+K0MQ0xbfwOWNMw==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDGQmrKfO3WovoXkOTSh/shO9qjbX4izhg4pccVU3Tp45v/dgAE\n"
+"uDUuaa/clToyH5AhOtuazO/asC3ZNajg1ia5VPzmQU3gtqiIZIEXFaOovPlOrXru\n"
+"wyQnxaGORndJwfDXicG6bUwI+PDpNq8c4VOTujReeF0r74qMSc7TQLVlUQIDAQAB\n"
+"AoGAakR/aTm9YibJVohbnl00xoOGlcLCsXU2lmaFZ3DsYdGWdD+TkvQJzW7ozJtQ\n"
+"Lj2sy6L4wujGR7nXWW3hr2IaLpoc1UoyJpieAZM5os6bMN+N4MCqdcZMlazMtSWV\n"
+"UDO7O7xQGFpcvvZmnfKCyluFaJ5K/tWxP+2TnS1/m0BDRIECQQD5DYvToA0eKBt+\n"
+"7K4eEI8pzDot9NlcL21D86kNgpmuY4pifALU7GvXr299JpFFiYa2A1JVRfpQaoI3\n"
+"hZzz0ze1AkEAy8opWJP+T2q4reD5Qq5UjjrHUXFID23KeJEjh5YF40/bHqyVpWVR\n"
+"UMntNgAzs+13vRij48Zn6I8GRhStaQ3ArQJASPyFS8GN1paeaDXoWPs1WWR2cF1f\n"
+"DbsAZHeVxVXOv+J//ZimI8wdVpodLCoPTLee+NxEVqUpVEPCYY8QjgwKOQJAATmj\n"
+"6f5pxvxzQ8hYd0gpBfngfOLbdgxI7VSiDAyg2G8AeDy9YZMsW/n6zRpPNUO2NpLR\n"
+"WWs18LX7aaxyJnGIuQJBAPPfy9pd4XEFsRBIIe3N23Gua1XkS/407RJtAGm73Vrt\n"
+"QhtWh3i6D5gfpEApMoaE8aaQQ7H0z+0Uh1t8SWesy10=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICWwIBAAKBgQCc/M/X8etUqrxnmH3PyuAYLIPZhwNySch8qz9NB47izYjxzuBG\n"
+"GSls6H7WeKIrB8UJY1gW8TLkdOLcrI/0hTANNHEPaueOE0xdABFj7tAaiiGPIM25\n"
+"N0wc76me0ZAMYJrZTHk8JZK153y9wInYBwVZreXCVSVf11RuVwe+iFQa5QIDAQAB\n"
+"AoGAQC4XJtivdhDLL6snHFF7pkZkrQTGgu3pOhakrXA+mTigGQOTqvTUe8LdP/9X\n"
+"hTIK+tiTheWcAcxLhx5BSB0/VDKjYhS0ROpTc33Iq9KalOQaTJbBYGA4eagpQjwU\n"
+"jGwr9u2sUsM9WI/Jg0VvLSKhfnNwYIUzLpK3BbWb2qAdh+0CQQDQ2s/8DlibFSBK\n"
+"UsFK7lLpV8UgMk9CkaNM2BPzI8Hsjpp6s3pULVRd36m4YTSg15EEHv7bZ1N/+krX\n"
+"mXb9xUULAkEAwGy5wHsUSjTK+kntkNXjlCU/+9R+HFpzg9Bwm/PqXTBwEWeU24hV\n"
+"iRjPvqPtWFZrWi/nfcviuMaqtdliw1I1zwJAZ2mQxhtMYC2LuYFUWAe9YfClmJWQ\n"
+"jUOTef8bka5I3RqW/t5TWc7AEWMnpDXtWx6hnUrDolt9Cschu7MvKeQ9lQJAL18U\n"
+"46PpPNN+XNuyVoOxgRkihVasrUI/SeYYsuv7eHGiRUagyOLpW9T139LvbV3pE8zT\n"
+"So7VA/Q0towL2lX01QJAGcoBNNouSpum9+5NvGQK1XXsZweawE+pFR2BE5XcjG+n\n"
+"FnaLEUBX7nTxhTU2cSQET1PKRNp568a281NEna0nxw==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXAIBAAKBgQDFOqqGG/VtIScxayZYZ+BT+hcs5W1bD5qRxunbG9O36UVT18UE\n"
+"CWw9HUf0Q5sDMGvVmBxwZ4GjbR5FDPfhIXaRCzobnejJXq/0k+O5NAVkcSPtJvhK\n"
+"AaUqBrWA41vnjKOtJudTsZLfufKafzYwVonze7fXGyVsBRjVwHNS4iqq2QIDAQAB\n"
+"AoGAJCoStI6R3RXUKvKb0GATuTJFZ50WBTmCPTK9FMkwdCuY47vPy2Ky7y3cUMTI\n"
+"urf5PewrYs0H72CFyWGMXkKVi8aOYshsATEXMfGSqOcqXn+UDssRzvabZFlpnAUa\n"
+"WDVt/iN092AdakXNna7/DxrLisDpq8HHJfjtlWGPfkXRg4ECQQDpHeKimTvwJcPc\n"
+"iDa6Qb/n9gwLeRckfzhYtfX1luJYLIOHh+J9vjQN75thenBLQB/B6qlKtOn9ejxg\n"
+"5z+3zIOpAkEA2JbxXVTCOA802p9khvHxDtLHdKi3w/BjjJiC7Mgqo69ZI+s3PB9E\n"
+"F2HJA69kZqpGqvybWHDapjWsq7rcMlxrsQJBAME2yvR3y00VEAyGPc4M1vF8ZqlP\n"
+"uRW/+ETWtEDUyU/JvU6lGt2bu2tdkEyv/cjxIiFIzP4litdT7B1pLc+6S9kCQBwE\n"
+"usiWFGHoJbA6emiyl7qRLdg7kzo3uMkRWa6D3nA6WM+6t/SBHu/faH+fit91G5s2\n"
+"/mmcf8yMmP/GNoIVTqECQFl4Pt6yGiz/YVoYSp35ljY5n3JB6T8o2pOmIrRLuPmT\n"
+"6kgyygtJBAmx5nnQoeG8n08tl9QakWznKzkNJ0DIFKI=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDCaOqJ0lsSAEBcnNB6X7BvVcEcol+evi/nJsPe0uT1SbtW50Ch\n"
+"vYOHwK6aQR2C5x9VSs47cLynTL7tNt5d8oeryF3NpI8VTPLImDJCcvUZhS7p4bxn\n"
+"JO+Wm+D/e3TWfyjreuWtdL+Mfimw2gzwWuBEtmj51GzQ89eYm7fh11SB6QIDAQAB\n"
+"AoGAWaakMbZNxPlUtOCjyysBY/Y5vYira7rswD3CKak7aFn+CE9QIMYSN7IFUqEg\n"
+"iNMoQd7jR8nvVX8wtJeO5+gF48W13C3n8FZSrW7c5N3bmfMIgo0xa/TGfeXHP98o\n"
+"7vhH0I58j3ZZt0Q+3wTm7t7WPE/nJzgrCk30TqmoaEmstTkCQQDtV6YZ6juEK2Lp\n"
+"LGUiqohcS/WJxvFrF5+LNpk86Xdgomf6FphZlkq42KYkvl7qibKDcfDqLKTbHHle\n"
+"vQQeCgZ7AkEA0bFHi7F8o4iHtKleBvt4QCj1neA0q3CRDypCI5EqFSrNpxY4Krhh\n"
+"WYSVX+xT00QYaCpKKWfYQztCw7Anylv96wJACl86Mwe5ch0zRV1bThiFvQLUyCCZ\n"
+"jESMBFlueOr6/I4cXSF/puqaeVl+aTyoiTdbRcNE8/bffXPRGgLIm0d04QJBAJSY\n"
+"lmTN789Lby99Xh6AkaSV4ghw26Ip8QHYJmph8npxjK69Niw/4Oy44cnKBVUPSmR2\n"
+"o3tYFY7/Lb7S1D+4lOUCQQDbMQUGVsZT+ZjuOG1bAjIuXoAOfOd3mgH5VgQHjSgJ\n"
+"ourZtlJ4OUpNrq9IfWqPkM+zSE8+0Dk8/9MS5ngBA/SJ\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIICXQIBAAKBgQDNbHjwg+7tVNr9erMLowXRnIcttp4pUJbr3B7Jo/u+kD/Yo3F3\n"
+"4rIKhHpJl1uEHP1QmvAD+4ApFFI2hNG54xYI8dGflxL5HOs5xxyOPpkrwzQ8Qvnv\n"
+"LPg7Gf6PAW9zF4McG4wK0TkrV28G6NhqcPs5VFY6UyvfZ0fEdWAeoWTIfQIDAQAB\n"
+"AoGBAKOmkMp7MLLd8QAS6eSRYSdWHdLrMyES1MjduaFGBF4SKOr7en/Zl6ENXSaX\n"
+"cA7V0XCPnjpt9/HCAKTyNupx4LCeFWiqdu8VGXhlzX8bdb896OSR2brKbxgRY5tF\n"
+"36uL8akrZdrYgocykQCxmRARMB7/rHwDusiamjL6RUZ3+c45AkEA6UPTVmKZQRMr\n"
+"A7Qgg5nXrXo9117Lpqf3FdZ1wdni9V59Ptf5xrx9oGZNZzctJPXSAH4M4cumSJrV\n"
+"sZ1V8qE7AwJBAOFx+5luLrVKrdlG7MyOhTAdhKYUvKIvL4wvVSY6y+L2nNEx/cTx\n"
+"KYbxGC+H1RJbkCS09rYir3VfDRWQ3W1c1n8CQH+X4hn2hO3blkPIW6CgniD+JKWR\n"
+"7MOUTMtdK7yFemfM76VYbgAPSohabSxwOfllnSE30cQQqTw9tXYaIdE98BECQG+M\n"
+"QWxSS0QillB6unIgVqBPCrJOcmNhK4qWZPBMiVNcqI0Nyj2nAeAl7MyfzfqOWY0A\n"
+"CU5nbR+LD2NLUXRqSisCQQCN3IGv1WOWInmA5xhU6vCFDX5u48Dcji7VLJO/Nv/i\n"
+"b/zHKAgjHk5Js7bi5ZWEGaUgA4Jt6cKmGdERheqTMKxx\n"
+"-----END RSA PRIVATE KEY-----\n"
+};
+
+static const char *PREGEN_KEYS_2048[] = {
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEpAIBAAKCAQEAoksI1qIuIaFCqT4QbgDvOQCmr9Z9F0E7ku+U5Ep/5dWNANqB\n"
+"bSzAOq0+cxiisfF+H4desoqiWDUwlOwXH74qD3ZsbChhvFUD78cQBWQkF+whLVHb\n"
+"296QmF0LZqosqz9HMS9CdoMUc1brZb78Hb25QIOOjrg25KYHLZHaqcet1wfhHow6\n"
+"Uehc6QTuWgOWFhJnfiXzYgen2o8lnLixxZozhk7Lm7Aix9ur2ckXdQ2Wgny4xw70\n"
+"JW84Hapnd8oFUD98XXrExk4VFuIcA8qo7r7y18II6wx4Cw1suKru6bhW65cM/y51\n"
+"KC4lB7VkvuoJCelRFdM1PfKZLv2tJP63oAqJrQIDAQABAoIBAQCWc38PEqw3avqU\n"
+"UMAEaoNa0bq1Gd8/Nq8WqVnbRSFKHO2pk+cWIb1W6BITuwvgcGKesezdEV4s7apK\n"
+"9I7/U1hEm2Ep50mrwRh0KZM1nD9Fmharn851Bt//D4qpMytT2caS1yADI8NKpZJ1\n"
+"8VZh7+cT4qG+txHUaAIRgbw3VrBWvTIMu6SOSOZm+e3eOr5UU3du1KvjdJHJ2c2k\n"
+"TceHvUdKxV7OYt+BBSN1oBOhs3ajUSRge1v3twRDg3cmbwG0DeXvwHNhGUTcF8IH\n"
+"JO1RF5njbkFvyqdAi3ltjU41zYd4OMuPtrwzFOtxUjKT62Soz109HUXXE2CGKFPZ\n"
+"PVi5/BIhAoGBANN1xqS5BgHszIB0nXbw5ImYpTRmyhO0KsTblBT9+8Q/B7BCK7bM\n"
+"zl+dOPeyvEadSwE7RSMMt6CAlTakWIf3Quw/VZajvXy9C9/LHf52pEKXjxMFMPKE\n"
+"aGLHpQnwMtDi8/H8AEAXxI3hpxB2KVR7sAYHWihSGjRJ6oPGvEmKEkb5AoGBAMR6\n"
+"G2PKz0xk1vFrjfjSY+y13gH/t7xHaXUggjggUSGKaknQh2BDUllXjadeI0fi1eLW\n"
+"r98ZImZZgntAgjaIZ4bAlooTDk4gRHaz9jI+z8lsRwOKnWdiigM7txiXZTMVwMqj\n"
+"o5mMNGMA+A+ACkTViRHmkDI7S/9FqAvnbOqVwgFVAoGBALUcY6WDvwx5B3Jh7tgH\n"
+"XIYpEh3+h8c2gYcX1g3gtvkPTwN8uToY0gz8eOVV1YHZiHsmi4GIi+HRH3usaRMT\n"
+"COOVHzYlSc8Dj57+tdLTRL6wVl9hC9o647ju64DGlI9qQquYPZKniLZIdbFYsu9j\n"
+"/JA9Tc/I+h6czFpPJccKlbrpAoGAAPWXrKUQ3g6f/g3IY66jTkSVEO1uuDyhBzFh\n"
+"cWS3ALLsUe/yuUWa4VTMHEUZZwB0iucBdNVqlZVaTb/C4wFHgCDwmzv8leUScIHw\n"
+"cc5ctV8R+bJzkk2o3tsrybLzi4xPpK2n3tgQaWtXyruVUUC5qpy1l4kylcyBRY2b\n"
+"uomAqQECgYAiCNWtuWIDlRBcvtIB+kHguzcoFT3vTCCNhalTEn0zi/tbi+voQgVJ\n"
+"SDJNptZv+6vRwQ/HfcQtljKIPO6hUZPYaFWRNhgbh7Ay85lRXYXQOottE8ayReBk\n"
+"zZb0fl853Qah4DPsaOugAvhjjKeBmKg6bFWO1z6hj18I3UpDf2YnVQ==\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEpQIBAAKCAQEAssO0r37mSJNAkc/ISwXBsu9JjyLeWlsHPAhylQGkSAdp2rjz\n"
+"E6AT0Eh3wrocNO31I4pvHReAuh1QedGY6T1cQwO/WAAhQtRCBQDK12qWRgfbC11y\n"
+"Xu7zNYPd1Z7YIRy+FxhbL5f+lv3rEUv0HUG5c3CWhLtbANKg+jOieIDzA4Yp1s55\n"
+"ynodQBUkTZrwQiT0P8yDSjiasf+clgJRfA1k2XK12KSAMRgyDuPTE4OtBxBvUM3L\n"
+"Zvxs81PsmcOuAG4DLaFTg2a/QkCjt2VC1SYYuh/LVxpL41FFh3eMoK5g5deHkgRe\n"
+"tlywKjAHIDJu/qgNzNgNW7ymwn2CfBvry9h0/wIDAQABAoIBAEMZ4wDdCWPEokAZ\n"
+"Vn2Ss5qO53WrCPuxn42RPjFgZGIFJl7LfbKoK8fK6+lUIrJbf+DPXdX1tIQn7MVN\n"
+"P7CNL8yX44MMyW9kbUOjgIBLqgyvdjFV6lBoMTKtRN+iuE31lATnR5Md4pqaxVnA\n"
+"wOkaepoycM1x5j7w0SwZparF/HIdkYv0y/MysqT9ByupPA4Fqp/iRSrosHXahNtI\n"
+"KZYj1TyERYtuDXq91P4dr/pWq3FmDNI8O3upblkL0YouvG/ZlFLdiNy77XbAyWcX\n"
+"ps3YDddM+vECnXO3+sa3ZxgBYvXJdWrrIzM5A+jCkDRZQGsFAzK5I5/S7C2ljt6i\n"
+"SmzqvMECgYEA16bGy2XTi6KBPb8aev/OBgK9XuGLwUqK1m15mS9Y2qPHmuc22qaZ\n"
+"hw6zginPFrxAEtQWKanhZy4aVqlLkDPLwRnyeuMo1EZAc5B1gZ5ViSAKxBq99hA9\n"
+"eqyakdb+IUQsEnRDxSc2gqUQ0EagksUyw5wGG5Q/CVEALmS/r1SU3KUCgYEA1DYf\n"
+"6JYdzuRtule3vYeWXKf8sOJpdplgWV7tvLrKkQhdE564uwMCYB23HvYfwWqEdDYG\n"
+"fsYg/ur/stk9MDZ3wZKffTEM8V3sX1t1JXnC3ogSAgMGhLZ3ILOLqkoO4BEZJnsS\n"
+"dMdiNijlAtQkqs/BO/UVUAKysCtKP3v/+1775dMCgYEAvLjGFjApfnSbV/cK7IM6\n"
+"wEXbhdIqZOCgOeEaXjVyM/zKbMRVW+oaR3hVHd8KzSG3jQKv1oxFpu9Qu3ByoWLC\n"
+"uF3Ft0debs6ADuJoAyQWROeWpGGmxlUWCGpO5rxYL7KiQxAeUsXrTU+5NBvq4CbV\n"
+"MxwyuCX3OGb7mp4upfiGQcUCgYEAuhVsDYv1P4LXJVvd5viKRV2ZG5KuYC1Ga5fu\n"
+"aFxzXJI07At2eaa94oKsHR494mEBHNZzA5/BN0fiSHZuTWS1xqxH5oOokc6Gg2ez\n"
+"ZdVLp88x20nD4YQPGkHW6tBeEuVrZG7vVC+yU0Ow7bYRISdkjqrusWZsQkbzqI+X\n"
+"fFliEbkCgYEAu8x+47M1ordbI7NmbBGyiyP0r7nMRCZ+KEvGeCNYracWmsnCNnfV\n"
+"zR2UzmwtSainw3Ho8Jv/rWDC8RIDauyBRYEi2VqOnUzT2ca0iymQyLeBCudAQuio\n"
+"drOu4JU8RzZ3Ad6V3DNFnaqmX/7GA9Pa2GI8NJMyb8p1GAGv7Gi8nxc=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEowIBAAKCAQEAt01S8JuEwWy/Hzb90yO2O7oGWq3GfvfDpFOF4OQnwG3kQ/BP\n"
+"4MoPDCYHdqb3iI9aD3vykZA6Q8zpdfGwjm4+bHrgRdiSmZWv8NvRwuQ5Ji9xbiGn\n"
+"hA1XwqH9hvgFTiy6tRvirWSJ7kzH3Q/bEGpCbHUQkwMog4v6yCNKNrjlwjN++eCi\n"
+"gFK/0RMOJMLOs8BD3zY+lKjd/pd8LBRujkMyUF5SryeRueAFjD2sq4OXq8DPABGt\n"
+"zdR6vbTcsi4JwP1Q6y4x0/LIWEprzzewNU63I5E2zj0WnoRGAIM4aF+VuqcHjWUx\n"
+"VWnyLZldSen6lScZ4xj4seitiDbSFvtFkDF6VwIDAQABAoIBAGTP9im2ntDyyjqU\n"
+"uA0DuxomOZBtupniEouyFBOX5/UBe2WSKZxsBNKdp8UuFz3X+aRCeyprtF/NtyjT\n"
+"AFOVdmebPPWtIxOtK9LAUyFo+7VwqmXzxHnwDLBS/2jXx7MzDozFBWpvvRx+xf1i\n"
+"1wy0JEwaJj90oTeYKRkhr5NhJZwkX8zCNYaemBd3kHB3aGWGJasI1Y81UezeRKCn\n"
+"hSbn2CrWalI7pyJ4lsavM11nIq1Eu2ZthJiNCMghbYrHoBHd+iVWiCYchP2rNEWV\n"
+"sdHtaVHtQ9zdZ43bao3OzPu7lAjd6UAbxsuhUe+a2YdDz/+Up+6+BvQf1FCfYIjW\n"
+"KFUdCoECgYEA4t5O+u0V9gkMUhKsevYb0zgc7O/mo8ivN+V++EpAtL0mhiwxeO8p\n"
+"oef0szLyhdULQeLN9pJQDCeAbkGdwIe3L+AKU8o8BFGEWLFysZjMg9In/UTrp5MN\n"
+"mMDy2SRKKu5BqsvdYH302xpZfHq1T2cMNDWE8lrZffduH06Cgq/XEtECgYEAztbj\n"
+"bhFneADnrvk609VnOQvoQEjySeCQKFQFRRI6k/FguqMisL2IRXnMaWammosdeCAg\n"
+"m7eZchnszHIst9cwZUKXUFqmAqeDuWSNdTI7uKZH6nT/A6IDlgdjaHsqhvpK0Ac9\n"
+"ngycdHONitOZh0ZG74pdWjf828Dwzf+CuYjl9KcCgYEAmIvI6ZqvkJ8m5Kzfw1Jn\n"
+"BVCOypbJK8oOX3R2Orea6KzjEYb3wQx3nwFcHX6danYFOskpmqlpH7MT/Y8rZsEa\n"
+"4RsxdoPedTzm08iFiXtn0R9nejp0hlov402iPXXUVSedih3IflBTa1w9XaEY9wog\n"
+"P57ZBSknYzcTmgNtaDiaUnECgYA5sWauhNw/dMEq5QmrnJK2LsQRakdqo+CR3x25\n"
+"LmR4b5Nze51pfvRLrLV/kMpXwQXvQ8bUqFl8og6S2CXxAWzWUcSy/RXhF6h+RbXP\n"
+"Qru1vWvB0fBvqvklF9p6giBSle3YKKzfMNVTBggs+OiR+uA+YHG5gHRfN2nzi5mC\n"
+"9tRtcQKBgBnDSi4lRCjRe9pPnyAYaa4iyBUGhjPysScSLY9orel89+qmTBQ/Py6J\n"
+"0+sefL4ZJaOsuaR2mSSPP/lbSkF9DMFs4tHbBqY+WkVNYLshAkauHwqv26HTVCSd\n"
+"QKzeb7uZw9lNaRIzDvy/3wfCLvXfdDozPFrOUgkyaBN5pJSA/4sv\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEogIBAAKCAQEA9qtiDoJWqU/eSlpj381eG6UcDzfMguFh/q4e4s7QVdRYj5J0\n"
+"Msv0PCkti8JHuvQUyncRpOPccBkhNbVjNbjIgw1pHaIZNdVotUDhP0kseRyJ6z3M\n"
+"qbZ5qKn+0mHjVjPNItVDDe6tebYMT1BZpVyRrCOqY2v5z1ecLC+ReygmHgDpzg+L\n"
+"0rWfIxGT10IPZ8pAlcdEn6xt5aEhi7mPCX/xwqfQChPIJz6zVLEC8UaPtvDBohPR\n"
+"6NQTBTeZZAAtzrQ7+oNxfz1v6Fz6RwMei7Q+qOBnMiwpQmbcDBKABM2RnXSpD0LA\n"
+"1GR7/+CiV1HQoShWVvEwrSIlM6jVAJo6iqF6WQIDAQABAoIBAHqwcdxPnfUm4aTP\n"
+"4r9NcZKEhDlZgqJSoiA/0OL1BRC7xrTanmspoLhPrvTF1FG715+Aq8j9AQbMqQUC\n"
+"zG7LEwiEIhV4K9vn4uXMeHy206UFud/E5EhBl695pmJUB/Q3XcAGnQyP+77++o50\n"
+"o7IpIdeiAbzj1uP3aplbq5u7M4JV7fUZWA/368G4HolqFTxcAfBJ05GXlp97BBwY\n"
+"AnY3/pNrKMz0NiPf3nsJHYWK18up0JCLPL3tomc94wuNZ66spIazHIL9aaKY0q3V\n"
+"LkBrelndfYM1m4xRTnSOy6STu0qKTPOpX0C8XBLYs6uiXjRsChqSYwndCCeASaH3\n"
+"LGNIcbUCgYEA/m4qvt8tdT4wEvnE+QUxEELmBtT4UFa3NnQISrzNlhNeI0Zd2xlp\n"
+"SG0/pcw83mG2uX+V5xSaWL5LYfLBkvy83Y0yIWgYbbIkyyCOUZnTpwaDGU/FjWip\n"
+"3TfXf5qpAgiez94sV+MsFpKfG05yxJh5u+3sIyGTVUAxp0HPx4LVgbMCgYEA+DD1\n"
+"fu6ttpuV1UMrsFdjuk6gBvSbyJ9OilY2jT+yE7hSRc/yP3O9ikuR74tNlVrWTnO2\n"
+"0kcYbyLJXE2cGUC2q5e4r8TDGiozNfQ7/OC2M3XaJ+xJk4zMf/8PuDDpWr+18ZXA\n"
+"Pf+ibXWTFvZ6ZeUmpbrrfCrXdvmIZnwVuOI0FcMCgYAZn26emksxq3mb75tumJ9A\n"
+"S/xuY7Q+Iv2Adl7/Z9QscPbiBowdLIn1yUrHn7Hhk2WbeMXX57NDjKZ6zr+/1cQP\n"
+"a9DInHsZUP9zlWu/vAYcpAM/4VC71PaGWMFTEHhExCl6NZ2xnCcsfseXMGdOdSyN\n"
+"SICnaRI1W6mkdnQ+W2a1EQKBgGEKA3KVr6XuPy8bDEHuaTe29irCCQbwAq1j+ABS\n"
+"HzZGoyRYocbdYgZoda7LMJJs6c3SwHCHC66oU0KbtaTKAKImuDdBH2djiJJX4/yD\n"
+"f7mvIpTpdfsS2gJRn7vMo/CvdFv4ySl0gfV6OwCHbmPYrLuv0dLCjWwfNI2dhoC7\n"
+"MNIxAoGAIPSIG4BrShzbeX4c2L18iwIg+NlOcUbtl0Ccr1t6uLGI+ge/6I6T/5XH\n"
+"DPKqYIf0IRYV8suxpfQNKiz/C0NPffA1d1M2hvuAg2v09o2cSwvdcQwdmakKZ5bl\n"
+"sdCuYKdCIwomEUOz/4XgQrJl4XDUqxftJT6/egAjWvcIYvfNCsY=\n"
+"-----END RSA PRIVATE KEY-----\n",
+
+"-----BEGIN RSA PRIVATE KEY-----\n"
+"MIIEowIBAAKCAQEA1yHZMsgRLckL+v6rgpGq9qmxVBNDxeuul1V/QlFyOlcAk5n/\n"
+"uduTalSqGQhc4NEePMxq6nFui4ucpkZOozmcEnhV0N9jld9IB9rLGt4erdg7RKl9\n"
+"+gQ+zTn69j69U36E2I47H4dM69uxeSOyWP2Odxpw+biisa3o8mMz1zCmuj4GMDtG\n"
+"DlnSpthFzgQR6N1pbvxLXrWg5F16GqFiJOD7kXDfy4/l6kB/mDs1T/3r8kav6DqR\n"
+"c/t3aQZxgWGIpI7hc9Qgvp7coZRMey5dNOZEna3tqS8dn2tZlhkpYV5uyFUjmxjG\n"
+"TERSULQ7hvUqW+eshGGsnxFtL7ANnTSc4xECowIDAQABAoIBAFhJJMhpQFuIySjd\n"
+"AGeZ/g4x/3rgWQzNNp4WUR5XLEhy0eLA7ShJywp06kVRoEQGraEHxsyldldAGS5H\n"
+"ZhgoGTufNKB+PHER646FpJpHE1IGjfQUloVW3qr8I1iQ0MOGBWCVpf+/V7rnMsLi\n"
+"+lr421FXgYuJ0QKXuyRVv72M0q9U6i+ml3aVAhgW/19oFg+dW7YccX+9iVyD05Q5\n"
+"KR64tX8xd4wrAqfAgYA3erbbE6GTyHYD5K54kIgfRr/+pIU4qc1L7XOCblnqc/rI\n"
+"BilFysEC634r2MNe66uQvNui4oQTfBcFFlXg0zAmp7d5QE0ApOL6HpCsmbImm2uJ\n"
+"sdFNYyECgYEA716kfEv7HfnF0P3pAP2AOuEsW6t8q0UtWvnHrwRQXQw8Yv90g7kD\n"
+"pUV3/BjD9VQgsQZosbdSn5wbT4j7dypRdrzYk+8m/hBk4Q8M/tWoRGVOn46NudvK\n"
+"/KX0A4ODLuulj8yAZVc7CM5Cdy4GCGJBVO+oVvBUAnHxfZziOyqBw9MCgYEA5hQg\n"
+"HEORzdxvbbfAx1ggvH1Eg1lqRhmpI43PpRkaoqb8jLwXb2CyBeuv3RBft/X2Tr6F\n"
+"mHpe0U1kN/5YEjii/Q/jUX8azIHaUNNSAjrriEeMQZOqFxmhCdiyeXuqg2fbFbhe\n"
+"K3Q6/fsB1xj9OOSwyPMqm/M5U0LsoGjmg8TFE/ECgYAlImKUIdlwOgp1NJ7MF4eo\n"
+"Gryd8AmkLFQv8+YFgb7R4I8RsJ2rva0SG6fUhScJTSbRL7RYNZ9swXP/L7oLL5Z5\n"
+"vCxBLu22pmZv/7y9X/n9ulWrLRtRhQaFkV08mk9knQwPNeOJVTIEWLM49/vZmxyV\n"
+"h6Ru8FOoGXMkUI1MLnj5HwKBgGJLkNhiacVYeuaWDa9c0EeXARFYvxWJ2wAMkvzG\n"
+"9+ErlFQP+7ciyYvMAItidnJii8NilDLrfNzQwpNFf5zxQ3j4M7bapblfdMT5M10u\n"
+"jPfhEWPm0VEjKvDI+p76HYQcd7YU2W6ZLqbZeRTLYUvQMFL5yGduBzyyJ+P0TR9Y\n"
+"jpYRAoGBAM7vYGTprw4w2tTZPFICXVk1bQ0LO06oNRtwkiQTUT6UqPjWMFyvHnmN\n"
+"11SVVBmRZ0RAk6e5eZLFX8WelJ4J4nSOGRcJheCtoEFlO7D1ewAUSbqWJ0pBqp2T\n"
+"gV4oCS8LYe8zReVoYZJjuLwoHvxZzs/hUjc3SI2HRW2W/HQRPC25\n"
+"-----END RSA PRIVATE KEY-----\n"
+};
+
+#define N_PREGEN_KEYS_1024 ARRAY_LENGTH(PREGEN_KEYS_1024)
+static crypto_pk_t *pregen_keys_1024[N_PREGEN_KEYS_1024];
+static int next_key_idx_1024;
+#define N_PREGEN_KEYS_2048 ARRAY_LENGTH(PREGEN_KEYS_2048)
+static crypto_pk_t *pregen_keys_2048[N_PREGEN_KEYS_2048];
+static int next_key_idx_2048;
+#endif
+
+/** Generate and return a new keypair for use in unit tests.  If we're using
+ * the key cache optimization, we might reuse keys. "idx" is ignored.
+ * Our only guarantee is that we won't reuse a key till this function has been
+ * called several times. The order in which keys are returned is slightly
+ * randomized, so that tests that depend on a particular order will not be
+ * reliable. */
+static crypto_pk_t *
+pk_generate_internal(int bits)
+{
+  tor_assert(bits == 2048 || bits == 1024);
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+  int *idxp;
+  int n_pregen;
+  crypto_pk_t **pregen_array;
+  if (bits == 2048) {
+    idxp = &next_key_idx_2048;
+    n_pregen = N_PREGEN_KEYS_2048;
+    pregen_array = pregen_keys_2048;
+  } else {
+    idxp = &next_key_idx_1024;
+    n_pregen = N_PREGEN_KEYS_1024;
+    pregen_array = pregen_keys_1024;
+  }
+  /* Either skip 1 or 2 keys. */
+  *idxp += crypto_rand_int_range(1,3);
+  *idxp %= n_pregen;
+  return crypto_pk_dup_key(pregen_array[*idxp]);
+#else
+  crypto_pk_t *result;
+  int res;
+  result = crypto_pk_new();
+  res = crypto_pk_generate_key_with_bits__real(result, bits);
+  tor_assert(!res);
+  return result;
+#endif
+}
+
+crypto_pk_t *
+pk_generate(int idx)
+{
+  (void) idx;
+  return pk_generate_internal(1024);
+}
+
+#ifdef USE_PREGENERATED_RSA_KEYS
+static int
+crypto_pk_generate_key_with_bits__get_cached(crypto_pk_t *env, int bits)
+{
+  if (bits == 1024 || bits == 2048)  {
+    crypto_pk_t *newkey = pk_generate_internal(bits);
+    crypto_pk_assign_(env, newkey);
+    crypto_pk_free(newkey);
+  } else {
+    return crypto_pk_generate_key_with_bits__real(env, bits);
+  }
+  return 0;
+}
+#endif
+
+/** Free all storage used for the cached key optimization. */
+void
+free_pregenerated_keys(void)
+{
+#ifdef USE_PREGENERATED_RSA_KEYS
+  unsigned idx;
+  for (idx = 0; idx < N_PREGEN_KEYS_1024; ++idx) {
+    if (pregen_keys_1024[idx]) {
+      crypto_pk_free(pregen_keys_1024[idx]);
+      pregen_keys_1024[idx] = NULL;
+    }
+  }
+  for (idx = 0; idx < N_PREGEN_KEYS_2048; ++idx) {
+    if (pregen_keys_2048[idx]) {
+      crypto_pk_free(pregen_keys_2048[idx]);
+      pregen_keys_2048[idx] = NULL;
+    }
+  }
+#endif
+}
+
+void
+init_pregenerated_keys(void)
+{
+#ifdef USE_PREGENERATED_RSA_KEYS
+  const char *s;
+  crypto_pk_t *pk;
+  unsigned i;
+  for (i = 0; i < N_PREGEN_KEYS_1024; ++i) {
+    pk = pregen_keys_1024[i] = crypto_pk_new();
+    s = PREGEN_KEYS_1024[i];
+    int r = crypto_pk_read_private_key_from_string(pk, s, strlen(s));
+    tor_assert(r == 0);
+  }
+  for (i = 0; i < N_PREGEN_KEYS_2048; ++i) {
+    pk = pregen_keys_2048[i] = crypto_pk_new();
+    s = PREGEN_KEYS_2048[i];
+    int r = crypto_pk_read_private_key_from_string(pk, s, strlen(s));
+    tor_assert(r == 0);
+  }
+
+  MOCK(crypto_pk_generate_key_with_bits,
+       crypto_pk_generate_key_with_bits__get_cached);
+#endif
+}
+
diff --git a/src/tools/include.am b/src/tools/include.am
index d0185b5887..717af9e2ae 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -1,5 +1,4 @@
 bin_PROGRAMS+= src/tools/tor-resolve src/tools/tor-gencert
-noinst_PROGRAMS+=  src/tools/tor-checkkey
 
 if COVERAGE_ENABLED
 noinst_PROGRAMS+= src/tools/tor-cov-resolve src/tools/tor-cov-gencert
@@ -9,7 +8,8 @@ src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_resolve_LDFLAGS =
 src_tools_tor_resolve_LDADD = src/common/libor.a \
 	src/common/libor-ctime.a \
-	@TOR_LIB_MATH@ @TOR_LIB_WS32@
+	@TOR_LIB_MATH@ @TOR_LIB_WS32@ \
+	$(rust_ldadd)
 
 if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
@@ -23,11 +23,12 @@ endif
 src_tools_tor_gencert_SOURCES = src/tools/tor-gencert.c
 src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
 src_tools_tor_gencert_LDADD = src/common/libor.a src/common/libor-crypto.a \
-    src/common/libor-ctime.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+	src/common/libor-ctime.a \
+	$(LIBKECCAK_TINY) \
+	$(LIBDONNA) \
+	@TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
+	@TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \
+	$(rust_ldadd)
 
 if COVERAGE_ENABLED
 src_tools_tor_cov_gencert_SOURCES = src/tools/tor-gencert.c
@@ -43,14 +44,4 @@ src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \
     @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
 endif
 
-src_tools_tor_checkkey_SOURCES = src/tools/tor-checkkey.c
-src_tools_tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@
-src_tools_tor_checkkey_LDADD = src/common/libor.a \
-    src/common/libor-ctime.a \
-    src/common/libor-crypto.a \
-    $(LIBKECCAK_TINY) \
-    $(LIBDONNA) \
-    @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \
-    @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
-
 EXTRA_DIST += src/tools/tor-fw-helper/README
diff --git a/src/tools/tor-checkkey.c b/src/tools/tor-checkkey.c
deleted file mode 100644
index 3e16fd0336..0000000000
--- a/src/tools/tor-checkkey.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* Copyright (c) 2008-2015, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include "crypto.h"
-#include "torlog.h"
-#include "util.h"
-#include "compat.h"
-#include "compat_openssl.h"
-#include <openssl/bn.h>
-#include <openssl/rsa.h>
-
-int
-main(int c, char **v)
-{
-  crypto_pk_t *env;
-  char *str;
-  RSA *rsa;
-  int wantdigest=0;
-  int fname_idx;
-  char *fname=NULL;
-  init_logging(1);
-
-  if (c < 2) {
-    fprintf(stderr, "Hi. I'm tor-checkkey.  Tell me a filename that "
-            "has a PEM-encoded RSA public key (like in a cert) and I'll "
-            "dump the modulus.  Use the --digest option too and I'll "
-            "dump the digest.\n");
-    return 1;
-  }
-
-  if (crypto_global_init(0, NULL, NULL)) {
-    fprintf(stderr, "Couldn't initialize crypto library.\n");
-    return 1;
-  }
-
-  if (!strcmp(v[1], "--digest")) {
-    wantdigest = 1;
-    fname_idx = 2;
-    if (c<3) {
-      fprintf(stderr, "too few arguments");
-      return 1;
-    }
-  } else {
-    wantdigest = 0;
-    fname_idx = 1;
-  }
-
-  fname = expand_filename(v[fname_idx]);
-  str = read_file_to_str(fname, 0, NULL);
-  tor_free(fname);
-  if (!str) {
-    fprintf(stderr, "Couldn't read %s\n", v[fname_idx]);
-    return 1;
-  }
-
-  env = crypto_pk_new();
-  if (crypto_pk_read_public_key_from_string(env, str, strlen(str))<0) {
-    fprintf(stderr, "Couldn't parse key.\n");
-    return 1;
-  }
-  tor_free(str);
-
-  if (wantdigest) {
-    char digest[HEX_DIGEST_LEN+1];
-    if (crypto_pk_get_fingerprint(env, digest, 0)<0)
-      return 1;
-    printf("%s\n",digest);
-  } else {
-    rsa = crypto_pk_get_rsa_(env);
-
-    const BIGNUM *rsa_n;
-#ifdef OPENSSL_1_1_API
-    const BIGNUM *rsa_e, *rsa_d;
-    RSA_get0_key(rsa, &rsa_n, &rsa_e, &rsa_d);
-#else
-    rsa_n = rsa->n;
-#endif
-    str = BN_bn2hex(rsa_n);
-
-    printf("%s\n", str);
-  }
-
-  return 0;
-}
-
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index db308485e6..395535697f 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2015, The Tor Project, Inc. */
+/* Copyright (c) 2007-2017, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6ac866d3c0..1e2409a131 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
- * Copyright (c) 2007-2015, The Tor Project, Inc.
+ * Copyright (c) 2007-2017, The Tor Project, Inc.
  */
 /* See LICENSE for licensing information */
 
diff --git a/src/trace/debug.h b/src/trace/debug.h
new file mode 100644
index 0000000000..3a1652543a
--- /dev/null
+++ b/src/trace/debug.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TRACE_LOG_DEBUG_H
+#define TOR_TRACE_LOG_DEBUG_H
+
+#include "torlog.h"
+
+/* Stringify pre-processor trick. */
+#define XSTR(d) STR(d)
+#define STR(s) #s
+
+/* Send every event to a debug log level. This is useful to debug new trace
+ * events without implementing them for a specific event tracing framework.
+ * Note that the arguments are ignored since at this step we do not know the
+ * types and amount there is. */
+
+/* Example on how to map a tracepoint to log_debug(). */
+#undef tor_trace
+#define tor_trace(subsystem, name, args...) \
+  log_debug(LD_GENERAL, "Trace event \"" XSTR(name) "\" from " \
+                        "\"" XSTR(subsystem) "\" hit. " \
+                        "(line "XSTR(__LINE__) ")")
+
+#endif /* TOR_TRACE_LOG_DEBUG_H */
diff --git a/src/trace/events.h b/src/trace/events.h
new file mode 100644
index 0000000000..1be1fd596e
--- /dev/null
+++ b/src/trace/events.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file events.h
+ * \brief Header file for Tor event tracing.
+ **/
+
+#ifndef TOR_TRACE_EVENTS_H
+#define TOR_TRACE_EVENTS_H
+
+/*
+ * The following defines a generic event tracing function name that has to be
+ * used to trace events in the code base.
+ *
+ * That generic function is then defined by a event tracing framework. For
+ * instance, the "log debug" framework sends all trace events to log_debug()
+ * which is defined in src/trace/debug.h which can only be enabled at compile
+ * time (--enable-event-tracing-debug).
+ *
+ * By default, every trace events in the code base are replaced by a NOP. See
+ * doc/HACKING/Tracing.md for more information on how to use event tracing or
+ * add events.
+ */
+
+#ifdef TOR_EVENT_TRACING_ENABLED
+/* Map every trace event to a per subsystem macro. */
+#define tor_trace(subsystem, name, ...) \
+  tor_trace_##subsystem(name, __VA_ARGS__)
+
+/* Enable event tracing for the debug framework where all trace events are
+ * mapped to a log_debug(). */
+#ifdef USE_EVENT_TRACING_DEBUG
+#include "trace/debug.h"
+#endif
+
+#else /* TOR_EVENT_TRACING_ENABLED */
+
+/* Reaching this point, we NOP every event declaration because event tracing
+ * is not been enabled at compile time. */
+#define tor_trace(subsystem, name, args...)
+
+#endif /* TOR_EVENT_TRACING_ENABLED */
+
+#endif /* TOR_TRACE_EVENTS_H */
diff --git a/src/trace/include.am b/src/trace/include.am
new file mode 100644
index 0000000000..3285b04de6
--- /dev/null
+++ b/src/trace/include.am
@@ -0,0 +1,22 @@
+# Include the src/ so we can use the trace/events.h statement when including
+# any file in that directory.
+AM_CPPFLAGS += -I$(srcdir)/src
+
+noinst_LIBRARIES += \
+	src/trace/libor-trace.a
+LIBOR_TRACE_A_SOURCES = \
+	src/trace/trace.c
+
+TRACEHEADERS = \
+	src/trace/trace.h \
+	src/trace/events.h
+
+if USE_EVENT_TRACING_DEBUG
+TRACEHEADERS += \
+	src/trace/debug.h
+endif
+
+# Library source files.
+src_trace_libor_trace_a_SOURCES = $(LIBOR_TRACE_A_SOURCES)
+
+noinst_HEADERS+= $(TRACEHEADERS)
diff --git a/src/trace/trace.c b/src/trace/trace.c
new file mode 100644
index 0000000000..fcdb80091f
--- /dev/null
+++ b/src/trace/trace.c
@@ -0,0 +1,11 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "trace.h"
+
+/** Initialize the tracing library. */
+void
+tor_trace_init(void)
+{
+}
+
diff --git a/src/trace/trace.h b/src/trace/trace.h
new file mode 100644
index 0000000000..28fcd8eea8
--- /dev/null
+++ b/src/trace/trace.h
@@ -0,0 +1,10 @@
+/* Copyright (c) 2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TRACE_TRACE_H
+#define TOR_TRACE_TRACE_H
+
+void tor_trace_init(void);
+
+#endif // TOR_TRACE_TRACE_H
+
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
new file mode 100644
index 0000000000..172d6f8a03
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -0,0 +1,281 @@
+/* channelpadding_negotiation.c -- generated by Trunnel v1.4.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "channelpadding_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're runnning a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int channelpaddingnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || channelpaddingnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+channelpadding_negotiate_t *
+channelpadding_negotiate_new(void)
+{
+  channelpadding_negotiate_t *val = trunnel_calloc(1, sizeof(channelpadding_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CHANNELPADDING_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+channelpadding_negotiate_clear(channelpadding_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+channelpadding_negotiate_free(channelpadding_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  channelpadding_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(channelpadding_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+channelpadding_negotiate_get_version(channelpadding_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+channelpadding_negotiate_set_version(channelpadding_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+channelpadding_negotiate_get_command(channelpadding_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+channelpadding_negotiate_set_command(channelpadding_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CHANNELPADDING_COMMAND_START || val == CHANNELPADDING_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint16_t
+channelpadding_negotiate_get_ito_low_ms(channelpadding_negotiate_t *inp)
+{
+  return inp->ito_low_ms;
+}
+int
+channelpadding_negotiate_set_ito_low_ms(channelpadding_negotiate_t *inp, uint16_t val)
+{
+  inp->ito_low_ms = val;
+  return 0;
+}
+uint16_t
+channelpadding_negotiate_get_ito_high_ms(channelpadding_negotiate_t *inp)
+{
+  return inp->ito_high_ms;
+}
+int
+channelpadding_negotiate_set_ito_high_ms(channelpadding_negotiate_t *inp, uint16_t val)
+{
+  inp->ito_high_ms = val;
+  return 0;
+}
+const char *
+channelpadding_negotiate_check(const channelpadding_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CHANNELPADDING_COMMAND_START || obj->command == CHANNELPADDING_COMMAND_STOP))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+channelpadding_negotiate_encoded_len(const channelpadding_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != channelpadding_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u16 ito_low_ms */
+  result += 2;
+
+  /* Length of u16 ito_high_ms */
+  result += 2;
+  return result;
+}
+int
+channelpadding_negotiate_clear_errors(channelpadding_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+channelpadding_negotiate_encode(uint8_t *output, const size_t avail, const channelpadding_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = channelpadding_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = channelpadding_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u16 ito_low_ms */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->ito_low_ms));
+  written += 2; ptr += 2;
+
+  /* Encode u16 ito_high_ms */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->ito_high_ms));
+  written += 2; ptr += 2;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As channelpadding_negotiate_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+channelpadding_negotiate_parse_into(channelpadding_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CHANNELPADDING_COMMAND_START || obj->command == CHANNELPADDING_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u16 ito_low_ms */
+  CHECK_REMAINING(2, truncated);
+  obj->ito_low_ms = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u16 ito_high_ms */
+  CHECK_REMAINING(2, truncated);
+  obj->ito_high_ms = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+channelpadding_negotiate_parse(channelpadding_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = channelpadding_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = channelpadding_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    channelpadding_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
new file mode 100644
index 0000000000..e58bda3be1
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -0,0 +1,98 @@
+/* channelpadding_negotiation.h -- generated by by Trunnel v1.4.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CHANNELPADDING_NEGOTIATION_H
+#define TRUNNEL_CHANNELPADDING_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CHANNELPADDING_COMMAND_STOP 1
+#define CHANNELPADDING_COMMAND_START 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CHANNELPADDING_NEGOTIATE)
+struct channelpadding_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  uint16_t ito_low_ms;
+  uint16_t ito_high_ms;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct channelpadding_negotiate_st channelpadding_negotiate_t;
+/** Return a newly allocated channelpadding_negotiate with all
+ * elements set to zero.
+ */
+channelpadding_negotiate_t *channelpadding_negotiate_new(void);
+/** Release all storage held by the channelpadding_negotiate in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void channelpadding_negotiate_free(channelpadding_negotiate_t *victim);
+/** Try to parse a channelpadding_negotiate from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated channelpadding_negotiate_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t channelpadding_negotiate_parse(channelpadding_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * channelpadding_negotiate in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t channelpadding_negotiate_encoded_len(const channelpadding_negotiate_t *obj);
+/** Try to encode the channelpadding_negotiate from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t channelpadding_negotiate_encode(uint8_t *output, size_t avail, const channelpadding_negotiate_t *input);
+/** Check whether the internal state of the channelpadding_negotiate
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *channelpadding_negotiate_check(const channelpadding_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int channelpadding_negotiate_clear_errors(channelpadding_negotiate_t *obj);
+/** Return the value of the version field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint8_t channelpadding_negotiate_get_version(channelpadding_negotiate_t *inp);
+/** Set the value of the version field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_version(channelpadding_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint8_t channelpadding_negotiate_get_command(channelpadding_negotiate_t *inp);
+/** Set the value of the command field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_command(channelpadding_negotiate_t *inp, uint8_t val);
+/** Return the value of the ito_low_ms field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint16_t channelpadding_negotiate_get_ito_low_ms(channelpadding_negotiate_t *inp);
+/** Set the value of the ito_low_ms field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_ito_low_ms(channelpadding_negotiate_t *inp, uint16_t val);
+/** Return the value of the ito_high_ms field of the
+ * channelpadding_negotiate_t in 'inp'
+ */
+uint16_t channelpadding_negotiate_get_ito_high_ms(channelpadding_negotiate_t *inp);
+/** Set the value of the ito_high_ms field of the
+ * channelpadding_negotiate_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int channelpadding_negotiate_set_ito_high_ms(channelpadding_negotiate_t *inp, uint16_t val);
+
+
+#endif
diff --git a/src/trunnel/channelpadding_negotiation.trunnel b/src/trunnel/channelpadding_negotiation.trunnel
new file mode 100644
index 0000000000..7f2d4795b0
--- /dev/null
+++ b/src/trunnel/channelpadding_negotiation.trunnel
@@ -0,0 +1,17 @@
+const CHANNELPADDING_COMMAND_STOP = 1;
+const CHANNELPADDING_COMMAND_START = 2;
+
+/* This command tells the relay to alter its min and max netflow
+   timeout range values, and send padding at that rate (resuming
+   if stopped). */
+struct channelpadding_negotiate {
+  u8 version IN [0];
+  u8 command IN [CHANNELPADDING_COMMAND_START, CHANNELPADDING_COMMAND_STOP];
+
+  /* Min must not be lower than the current consensus parameter
+     nf_ito_low. */
+  u16 ito_low_ms;
+
+  /* Max must not be lower than ito_low_ms */
+  u16 ito_high_ms;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 24988d510b..ee02fda646 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.4.6.
+/* ed25519_cert.c -- generated by Trunnel v1.5.1.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -28,6 +28,281 @@ int edcert_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
+create2_cell_body_t *
+create2_cell_body_new(void)
+{
+  create2_cell_body_t *val = trunnel_calloc(1, sizeof(create2_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+create2_cell_body_clear(create2_cell_body_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->handshake_data);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->handshake_data);
+}
+
+void
+create2_cell_body_free(create2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  create2_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(create2_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint16_t
+create2_cell_body_get_handshake_type(const create2_cell_body_t *inp)
+{
+  return inp->handshake_type;
+}
+int
+create2_cell_body_set_handshake_type(create2_cell_body_t *inp, uint16_t val)
+{
+  inp->handshake_type = val;
+  return 0;
+}
+uint16_t
+create2_cell_body_get_handshake_len(const create2_cell_body_t *inp)
+{
+  return inp->handshake_len;
+}
+int
+create2_cell_body_set_handshake_len(create2_cell_body_t *inp, uint16_t val)
+{
+  inp->handshake_len = val;
+  return 0;
+}
+size_t
+create2_cell_body_getlen_handshake_data(const create2_cell_body_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->handshake_data);
+}
+
+uint8_t
+create2_cell_body_get_handshake_data(create2_cell_body_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->handshake_data, idx);
+}
+
+uint8_t
+create2_cell_body_getconst_handshake_data(const create2_cell_body_t *inp, size_t idx)
+{
+  return create2_cell_body_get_handshake_data((create2_cell_body_t*)inp, idx);
+}
+int
+create2_cell_body_set_handshake_data(create2_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->handshake_data, idx, elt);
+  return 0;
+}
+int
+create2_cell_body_add_handshake_data(create2_cell_body_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->handshake_data.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->handshake_data, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+create2_cell_body_getarray_handshake_data(create2_cell_body_t *inp)
+{
+  return inp->handshake_data.elts_;
+}
+const uint8_t  *
+create2_cell_body_getconstarray_handshake_data(const create2_cell_body_t *inp)
+{
+  return (const uint8_t  *)create2_cell_body_getarray_handshake_data((create2_cell_body_t*)inp);
+}
+int
+create2_cell_body_setlen_handshake_data(create2_cell_body_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->handshake_data.allocated_,
+                 &inp->handshake_data.n_, inp->handshake_data.elts_, newlen,
+                 sizeof(inp->handshake_data.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->handshake_data.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+create2_cell_body_check(const create2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->handshake_data) != obj->handshake_len)
+    return "Length mismatch for handshake_data";
+  return NULL;
+}
+
+ssize_t
+create2_cell_body_encoded_len(const create2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != create2_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u16 handshake_type */
+  result += 2;
+
+  /* Length of u16 handshake_len */
+  result += 2;
+
+  /* Length of u8 handshake_data[handshake_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->handshake_data);
+  return result;
+}
+int
+create2_cell_body_clear_errors(create2_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+create2_cell_body_encode(uint8_t *output, const size_t avail, const create2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = create2_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = create2_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u16 handshake_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->handshake_type));
+  written += 2; ptr += 2;
+
+  /* Encode u16 handshake_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->handshake_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 handshake_data[handshake_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->handshake_data);
+    trunnel_assert(obj->handshake_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->handshake_data.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As create2_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+create2_cell_body_parse_into(create2_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u16 handshake_type */
+  CHECK_REMAINING(2, truncated);
+  obj->handshake_type = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u16 handshake_len */
+  CHECK_REMAINING(2, truncated);
+  obj->handshake_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 handshake_data[handshake_len] */
+  CHECK_REMAINING(obj->handshake_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->handshake_data, obj->handshake_len, {});
+  obj->handshake_data.n_ = obj->handshake_len;
+  if (obj->handshake_len)
+    memcpy(obj->handshake_data.elts_, ptr, obj->handshake_len);
+  ptr += obj->handshake_len; remaining -= obj->handshake_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+create2_cell_body_parse(create2_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = create2_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = create2_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    create2_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 ed25519_cert_extension_t *
 ed25519_cert_extension_new(void)
 {
@@ -58,7 +333,7 @@ ed25519_cert_extension_free(ed25519_cert_extension_t *obj)
 }
 
 uint16_t
-ed25519_cert_extension_get_ext_length(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_length(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_length;
 }
@@ -69,7 +344,7 @@ ed25519_cert_extension_set_ext_length(ed25519_cert_extension_t *inp, uint16_t va
   return 0;
 }
 uint8_t
-ed25519_cert_extension_get_ext_type(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_type(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_type;
 }
@@ -80,7 +355,7 @@ ed25519_cert_extension_set_ext_type(ed25519_cert_extension_t *inp, uint8_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_extension_get_ext_flags(ed25519_cert_extension_t *inp)
+ed25519_cert_extension_get_ext_flags(const ed25519_cert_extension_t *inp)
 {
   return inp->ext_flags;
 }
@@ -97,12 +372,17 @@ ed25519_cert_extension_getlen_un_signing_key(const ed25519_cert_extension_t *inp
 }
 
 uint8_t
-ed25519_cert_extension_get_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx)
+ed25519_cert_extension_get_un_signing_key(ed25519_cert_extension_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->un_signing_key[idx];
 }
 
+uint8_t
+ed25519_cert_extension_getconst_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx)
+{
+  return ed25519_cert_extension_get_un_signing_key((ed25519_cert_extension_t*)inp, idx);
+}
 int
 ed25519_cert_extension_set_un_signing_key(ed25519_cert_extension_t *inp, size_t idx, uint8_t elt)
 {
@@ -116,6 +396,11 @@ ed25519_cert_extension_getarray_un_signing_key(ed25519_cert_extension_t *inp)
 {
   return inp->un_signing_key;
 }
+const uint8_t  *
+ed25519_cert_extension_getconstarray_un_signing_key(const ed25519_cert_extension_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_extension_getarray_un_signing_key((ed25519_cert_extension_t*)inp);
+}
 size_t
 ed25519_cert_extension_getlen_un_unparsed(const ed25519_cert_extension_t *inp)
 {
@@ -128,6 +413,11 @@ ed25519_cert_extension_get_un_unparsed(ed25519_cert_extension_t *inp, size_t idx
   return TRUNNEL_DYNARRAY_GET(&inp->un_unparsed, idx);
 }
 
+uint8_t
+ed25519_cert_extension_getconst_un_unparsed(const ed25519_cert_extension_t *inp, size_t idx)
+{
+  return ed25519_cert_extension_get_un_unparsed((ed25519_cert_extension_t*)inp, idx);
+}
 int
 ed25519_cert_extension_set_un_unparsed(ed25519_cert_extension_t *inp, size_t idx, uint8_t elt)
 {
@@ -149,6 +439,11 @@ ed25519_cert_extension_getarray_un_unparsed(ed25519_cert_extension_t *inp)
 {
   return inp->un_unparsed.elts_;
 }
+const uint8_t  *
+ed25519_cert_extension_getconstarray_un_unparsed(const ed25519_cert_extension_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_extension_getarray_un_unparsed((ed25519_cert_extension_t*)inp);
+}
 int
 ed25519_cert_extension_setlen_un_unparsed(ed25519_cert_extension_t *inp, size_t newlen)
 {
@@ -410,6 +705,878 @@ ed25519_cert_extension_parse(ed25519_cert_extension_t **output, const uint8_t *i
   }
   return result;
 }
+extend1_cell_body_t *
+extend1_cell_body_new(void)
+{
+  extend1_cell_body_t *val = trunnel_calloc(1, sizeof(extend1_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+extend1_cell_body_clear(extend1_cell_body_t *obj)
+{
+  (void) obj;
+}
+
+void
+extend1_cell_body_free(extend1_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  extend1_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(extend1_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+extend1_cell_body_get_ipv4addr(const extend1_cell_body_t *inp)
+{
+  return inp->ipv4addr;
+}
+int
+extend1_cell_body_set_ipv4addr(extend1_cell_body_t *inp, uint32_t val)
+{
+  inp->ipv4addr = val;
+  return 0;
+}
+uint16_t
+extend1_cell_body_get_port(const extend1_cell_body_t *inp)
+{
+  return inp->port;
+}
+int
+extend1_cell_body_set_port(extend1_cell_body_t *inp, uint16_t val)
+{
+  inp->port = val;
+  return 0;
+}
+size_t
+extend1_cell_body_getlen_onionskin(const extend1_cell_body_t *inp)
+{
+  (void)inp;  return 186;
+}
+
+uint8_t
+extend1_cell_body_get_onionskin(extend1_cell_body_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 186);
+  return inp->onionskin[idx];
+}
+
+uint8_t
+extend1_cell_body_getconst_onionskin(const extend1_cell_body_t *inp, size_t idx)
+{
+  return extend1_cell_body_get_onionskin((extend1_cell_body_t*)inp, idx);
+}
+int
+extend1_cell_body_set_onionskin(extend1_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 186);
+  inp->onionskin[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+extend1_cell_body_getarray_onionskin(extend1_cell_body_t *inp)
+{
+  return inp->onionskin;
+}
+const uint8_t  *
+extend1_cell_body_getconstarray_onionskin(const extend1_cell_body_t *inp)
+{
+  return (const uint8_t  *)extend1_cell_body_getarray_onionskin((extend1_cell_body_t*)inp);
+}
+size_t
+extend1_cell_body_getlen_identity(const extend1_cell_body_t *inp)
+{
+  (void)inp;  return 20;
+}
+
+uint8_t
+extend1_cell_body_get_identity(extend1_cell_body_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 20);
+  return inp->identity[idx];
+}
+
+uint8_t
+extend1_cell_body_getconst_identity(const extend1_cell_body_t *inp, size_t idx)
+{
+  return extend1_cell_body_get_identity((extend1_cell_body_t*)inp, idx);
+}
+int
+extend1_cell_body_set_identity(extend1_cell_body_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 20);
+  inp->identity[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+extend1_cell_body_getarray_identity(extend1_cell_body_t *inp)
+{
+  return inp->identity;
+}
+const uint8_t  *
+extend1_cell_body_getconstarray_identity(const extend1_cell_body_t *inp)
+{
+  return (const uint8_t  *)extend1_cell_body_getarray_identity((extend1_cell_body_t*)inp);
+}
+const char *
+extend1_cell_body_check(const extend1_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+extend1_cell_body_encoded_len(const extend1_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != extend1_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u32 ipv4addr */
+  result += 4;
+
+  /* Length of u16 port */
+  result += 2;
+
+  /* Length of u8 onionskin[186] */
+  result += 186;
+
+  /* Length of u8 identity[20] */
+  result += 20;
+  return result;
+}
+int
+extend1_cell_body_clear_errors(extend1_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+extend1_cell_body_encode(uint8_t *output, const size_t avail, const extend1_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = extend1_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = extend1_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 ipv4addr */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->ipv4addr));
+  written += 4; ptr += 4;
+
+  /* Encode u16 port */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->port));
+  written += 2; ptr += 2;
+
+  /* Encode u8 onionskin[186] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 186)
+    goto truncated;
+  memcpy(ptr, obj->onionskin, 186);
+  written += 186; ptr += 186;
+
+  /* Encode u8 identity[20] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 20)
+    goto truncated;
+  memcpy(ptr, obj->identity, 20);
+  written += 20; ptr += 20;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As extend1_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+extend1_cell_body_parse_into(extend1_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 ipv4addr */
+  CHECK_REMAINING(4, truncated);
+  obj->ipv4addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse u16 port */
+  CHECK_REMAINING(2, truncated);
+  obj->port = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 onionskin[186] */
+  CHECK_REMAINING(186, truncated);
+  memcpy(obj->onionskin, ptr, 186);
+  remaining -= 186; ptr += 186;
+
+  /* Parse u8 identity[20] */
+  CHECK_REMAINING(20, truncated);
+  memcpy(obj->identity, ptr, 20);
+  remaining -= 20; ptr += 20;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+extend1_cell_body_parse(extend1_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = extend1_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = extend1_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    extend1_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+link_specifier_t *
+link_specifier_new(void)
+{
+  link_specifier_t *val = trunnel_calloc(1, sizeof(link_specifier_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+link_specifier_clear(link_specifier_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->un_unrecognized);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->un_unrecognized);
+}
+
+void
+link_specifier_free(link_specifier_t *obj)
+{
+  if (obj == NULL)
+    return;
+  link_specifier_clear(obj);
+  trunnel_memwipe(obj, sizeof(link_specifier_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+link_specifier_get_ls_type(const link_specifier_t *inp)
+{
+  return inp->ls_type;
+}
+int
+link_specifier_set_ls_type(link_specifier_t *inp, uint8_t val)
+{
+  inp->ls_type = val;
+  return 0;
+}
+uint8_t
+link_specifier_get_ls_len(const link_specifier_t *inp)
+{
+  return inp->ls_len;
+}
+int
+link_specifier_set_ls_len(link_specifier_t *inp, uint8_t val)
+{
+  inp->ls_len = val;
+  return 0;
+}
+uint32_t
+link_specifier_get_un_ipv4_addr(const link_specifier_t *inp)
+{
+  return inp->un_ipv4_addr;
+}
+int
+link_specifier_set_un_ipv4_addr(link_specifier_t *inp, uint32_t val)
+{
+  inp->un_ipv4_addr = val;
+  return 0;
+}
+uint16_t
+link_specifier_get_un_ipv4_port(const link_specifier_t *inp)
+{
+  return inp->un_ipv4_port;
+}
+int
+link_specifier_set_un_ipv4_port(link_specifier_t *inp, uint16_t val)
+{
+  inp->un_ipv4_port = val;
+  return 0;
+}
+size_t
+link_specifier_getlen_un_ipv6_addr(const link_specifier_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+link_specifier_get_un_ipv6_addr(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->un_ipv6_addr[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_ipv6_addr(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_ipv6_addr((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_ipv6_addr(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->un_ipv6_addr[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_ipv6_addr(link_specifier_t *inp)
+{
+  return inp->un_ipv6_addr;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_ipv6_addr(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_ipv6_addr((link_specifier_t*)inp);
+}
+uint16_t
+link_specifier_get_un_ipv6_port(const link_specifier_t *inp)
+{
+  return inp->un_ipv6_port;
+}
+int
+link_specifier_set_un_ipv6_port(link_specifier_t *inp, uint16_t val)
+{
+  inp->un_ipv6_port = val;
+  return 0;
+}
+size_t
+link_specifier_getlen_un_legacy_id(const link_specifier_t *inp)
+{
+  (void)inp;  return 20;
+}
+
+uint8_t
+link_specifier_get_un_legacy_id(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 20);
+  return inp->un_legacy_id[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_legacy_id(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_legacy_id((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_legacy_id(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 20);
+  inp->un_legacy_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_legacy_id(link_specifier_t *inp)
+{
+  return inp->un_legacy_id;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_legacy_id(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_legacy_id((link_specifier_t*)inp);
+}
+size_t
+link_specifier_getlen_un_ed25519_id(const link_specifier_t *inp)
+{
+  (void)inp;  return 32;
+}
+
+uint8_t
+link_specifier_get_un_ed25519_id(link_specifier_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 32);
+  return inp->un_ed25519_id[idx];
+}
+
+uint8_t
+link_specifier_getconst_un_ed25519_id(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_ed25519_id((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_ed25519_id(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 32);
+  inp->un_ed25519_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+link_specifier_getarray_un_ed25519_id(link_specifier_t *inp)
+{
+  return inp->un_ed25519_id;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_ed25519_id(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_ed25519_id((link_specifier_t*)inp);
+}
+size_t
+link_specifier_getlen_un_unrecognized(const link_specifier_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->un_unrecognized);
+}
+
+uint8_t
+link_specifier_get_un_unrecognized(link_specifier_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->un_unrecognized, idx);
+}
+
+uint8_t
+link_specifier_getconst_un_unrecognized(const link_specifier_t *inp, size_t idx)
+{
+  return link_specifier_get_un_unrecognized((link_specifier_t*)inp, idx);
+}
+int
+link_specifier_set_un_unrecognized(link_specifier_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->un_unrecognized, idx, elt);
+  return 0;
+}
+int
+link_specifier_add_un_unrecognized(link_specifier_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->un_unrecognized, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+link_specifier_getarray_un_unrecognized(link_specifier_t *inp)
+{
+  return inp->un_unrecognized.elts_;
+}
+const uint8_t  *
+link_specifier_getconstarray_un_unrecognized(const link_specifier_t *inp)
+{
+  return (const uint8_t  *)link_specifier_getarray_un_unrecognized((link_specifier_t*)inp);
+}
+int
+link_specifier_setlen_un_unrecognized(link_specifier_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->un_unrecognized.allocated_,
+                 &inp->un_unrecognized.n_, inp->un_unrecognized.elts_, newlen,
+                 sizeof(inp->un_unrecognized.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->un_unrecognized.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+link_specifier_check(const link_specifier_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->ls_type) {
+
+    case LS_IPV4:
+      break;
+
+    case LS_IPV6:
+      break;
+
+    case LS_LEGACY_ID:
+      break;
+
+    case LS_ED25519_ID:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+link_specifier_encoded_len(const link_specifier_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != link_specifier_check(obj))
+     return -1;
+
+
+  /* Length of u8 ls_type */
+  result += 1;
+
+  /* Length of u8 ls_len */
+  result += 1;
+  switch (obj->ls_type) {
+
+    case LS_IPV4:
+
+      /* Length of u32 un_ipv4_addr */
+      result += 4;
+
+      /* Length of u16 un_ipv4_port */
+      result += 2;
+      break;
+
+    case LS_IPV6:
+
+      /* Length of u8 un_ipv6_addr[16] */
+      result += 16;
+
+      /* Length of u16 un_ipv6_port */
+      result += 2;
+      break;
+
+    case LS_LEGACY_ID:
+
+      /* Length of u8 un_legacy_id[20] */
+      result += 20;
+      break;
+
+    case LS_ED25519_ID:
+
+      /* Length of u8 un_ed25519_id[32] */
+      result += 32;
+      break;
+
+    default:
+
+      /* Length of u8 un_unrecognized[] */
+      result += TRUNNEL_DYNARRAY_LEN(&obj->un_unrecognized);
+      break;
+  }
+  return result;
+}
+int
+link_specifier_clear_errors(link_specifier_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+link_specifier_encode(uint8_t *output, const size_t avail, const link_specifier_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = link_specifier_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_ls_len = NULL;
+
+  if (NULL != (msg = link_specifier_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 ls_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->ls_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 ls_len */
+  backptr_ls_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->ls_len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union un[ls_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->ls_type) {
+
+      case LS_IPV4:
+
+        /* Encode u32 un_ipv4_addr */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->un_ipv4_addr));
+        written += 4; ptr += 4;
+
+        /* Encode u16 un_ipv4_port */
+        trunnel_assert(written <= avail);
+        if (avail - written < 2)
+          goto truncated;
+        trunnel_set_uint16(ptr, trunnel_htons(obj->un_ipv4_port));
+        written += 2; ptr += 2;
+        break;
+
+      case LS_IPV6:
+
+        /* Encode u8 un_ipv6_addr[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->un_ipv6_addr, 16);
+        written += 16; ptr += 16;
+
+        /* Encode u16 un_ipv6_port */
+        trunnel_assert(written <= avail);
+        if (avail - written < 2)
+          goto truncated;
+        trunnel_set_uint16(ptr, trunnel_htons(obj->un_ipv6_port));
+        written += 2; ptr += 2;
+        break;
+
+      case LS_LEGACY_ID:
+
+        /* Encode u8 un_legacy_id[20] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 20)
+          goto truncated;
+        memcpy(ptr, obj->un_legacy_id, 20);
+        written += 20; ptr += 20;
+        break;
+
+      case LS_ED25519_ID:
+
+        /* Encode u8 un_ed25519_id[32] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 32)
+          goto truncated;
+        memcpy(ptr, obj->un_ed25519_id, 32);
+        written += 32; ptr += 32;
+        break;
+
+      default:
+
+        /* Encode u8 un_unrecognized[] */
+        {
+          size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->un_unrecognized);
+          trunnel_assert(written <= avail);
+          if (avail - written < elt_len)
+            goto truncated;
+          if (elt_len)
+            memcpy(ptr, obj->un_unrecognized.elts_, elt_len);
+          written += elt_len; ptr += elt_len;
+        }
+        break;
+    }
+    /* Write the length field back to ls_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_ls_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As link_specifier_parse(), but do not allocate the output object.
+ */
+static ssize_t
+link_specifier_parse_into(link_specifier_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 ls_type */
+  CHECK_REMAINING(1, truncated);
+  obj->ls_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 ls_len */
+  CHECK_REMAINING(1, truncated);
+  obj->ls_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->ls_len, truncated);
+    remaining_after = remaining - obj->ls_len;
+    remaining = obj->ls_len;
+
+    /* Parse union un[ls_type] */
+    switch (obj->ls_type) {
+
+      case LS_IPV4:
+
+        /* Parse u32 un_ipv4_addr */
+        CHECK_REMAINING(4, fail);
+        obj->un_ipv4_addr = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+
+        /* Parse u16 un_ipv4_port */
+        CHECK_REMAINING(2, fail);
+        obj->un_ipv4_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+        remaining -= 2; ptr += 2;
+        break;
+
+      case LS_IPV6:
+
+        /* Parse u8 un_ipv6_addr[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->un_ipv6_addr, ptr, 16);
+        remaining -= 16; ptr += 16;
+
+        /* Parse u16 un_ipv6_port */
+        CHECK_REMAINING(2, fail);
+        obj->un_ipv6_port = trunnel_ntohs(trunnel_get_uint16(ptr));
+        remaining -= 2; ptr += 2;
+        break;
+
+      case LS_LEGACY_ID:
+
+        /* Parse u8 un_legacy_id[20] */
+        CHECK_REMAINING(20, fail);
+        memcpy(obj->un_legacy_id, ptr, 20);
+        remaining -= 20; ptr += 20;
+        break;
+
+      case LS_ED25519_ID:
+
+        /* Parse u8 un_ed25519_id[32] */
+        CHECK_REMAINING(32, fail);
+        memcpy(obj->un_ed25519_id, ptr, 32);
+        remaining -= 32; ptr += 32;
+        break;
+
+      default:
+
+        /* Parse u8 un_unrecognized[] */
+        TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->un_unrecognized, remaining, {});
+        obj->un_unrecognized.n_ = remaining;
+        if (remaining)
+          memcpy(obj->un_unrecognized.elts_, ptr, remaining);
+        ptr += remaining; remaining -= remaining;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = link_specifier_new();
+  if (NULL == *output)
+    return -1;
+  result = link_specifier_parse_into(*output, input, len_in);
+  if (result < 0) {
+    link_specifier_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 ed25519_cert_t *
 ed25519_cert_new(void)
 {
@@ -448,7 +1615,7 @@ ed25519_cert_free(ed25519_cert_t *obj)
 }
 
 uint8_t
-ed25519_cert_get_version(ed25519_cert_t *inp)
+ed25519_cert_get_version(const ed25519_cert_t *inp)
 {
   return inp->version;
 }
@@ -463,7 +1630,7 @@ ed25519_cert_set_version(ed25519_cert_t *inp, uint8_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_get_cert_type(ed25519_cert_t *inp)
+ed25519_cert_get_cert_type(const ed25519_cert_t *inp)
 {
   return inp->cert_type;
 }
@@ -474,7 +1641,7 @@ ed25519_cert_set_cert_type(ed25519_cert_t *inp, uint8_t val)
   return 0;
 }
 uint32_t
-ed25519_cert_get_exp_field(ed25519_cert_t *inp)
+ed25519_cert_get_exp_field(const ed25519_cert_t *inp)
 {
   return inp->exp_field;
 }
@@ -485,7 +1652,7 @@ ed25519_cert_set_exp_field(ed25519_cert_t *inp, uint32_t val)
   return 0;
 }
 uint8_t
-ed25519_cert_get_cert_key_type(ed25519_cert_t *inp)
+ed25519_cert_get_cert_key_type(const ed25519_cert_t *inp)
 {
   return inp->cert_key_type;
 }
@@ -502,12 +1669,17 @@ ed25519_cert_getlen_certified_key(const ed25519_cert_t *inp)
 }
 
 uint8_t
-ed25519_cert_get_certified_key(const ed25519_cert_t *inp, size_t idx)
+ed25519_cert_get_certified_key(ed25519_cert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->certified_key[idx];
 }
 
+uint8_t
+ed25519_cert_getconst_certified_key(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_certified_key((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_certified_key(ed25519_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -521,8 +1693,13 @@ ed25519_cert_getarray_certified_key(ed25519_cert_t *inp)
 {
   return inp->certified_key;
 }
+const uint8_t  *
+ed25519_cert_getconstarray_certified_key(const ed25519_cert_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_getarray_certified_key((ed25519_cert_t*)inp);
+}
 uint8_t
-ed25519_cert_get_n_extensions(ed25519_cert_t *inp)
+ed25519_cert_get_n_extensions(const ed25519_cert_t *inp)
 {
   return inp->n_extensions;
 }
@@ -544,6 +1721,11 @@ ed25519_cert_get_ext(ed25519_cert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->ext, idx);
 }
 
+ const struct ed25519_cert_extension_st *
+ed25519_cert_getconst_ext(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_ext((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_ext(ed25519_cert_t *inp, size_t idx, struct ed25519_cert_extension_st * elt)
 {
@@ -577,6 +1759,11 @@ ed25519_cert_getarray_ext(ed25519_cert_t *inp)
 {
   return inp->ext.elts_;
 }
+const struct ed25519_cert_extension_st *  const  *
+ed25519_cert_getconstarray_ext(const ed25519_cert_t *inp)
+{
+  return (const struct ed25519_cert_extension_st *  const  *)ed25519_cert_getarray_ext((ed25519_cert_t*)inp);
+}
 int
 ed25519_cert_setlen_ext(ed25519_cert_t *inp, size_t newlen)
 {
@@ -604,12 +1791,17 @@ ed25519_cert_getlen_signature(const ed25519_cert_t *inp)
 }
 
 uint8_t
-ed25519_cert_get_signature(const ed25519_cert_t *inp, size_t idx)
+ed25519_cert_get_signature(ed25519_cert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 64);
   return inp->signature[idx];
 }
 
+uint8_t
+ed25519_cert_getconst_signature(const ed25519_cert_t *inp, size_t idx)
+{
+  return ed25519_cert_get_signature((ed25519_cert_t*)inp, idx);
+}
 int
 ed25519_cert_set_signature(ed25519_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -623,6 +1815,11 @@ ed25519_cert_getarray_signature(ed25519_cert_t *inp)
 {
   return inp->signature;
 }
+const uint8_t  *
+ed25519_cert_getconstarray_signature(const ed25519_cert_t *inp)
+{
+  return (const uint8_t  *)ed25519_cert_getarray_signature((ed25519_cert_t*)inp);
+}
 const char *
 ed25519_cert_check(const ed25519_cert_t *obj)
 {
@@ -887,3 +2084,630 @@ ed25519_cert_parse(ed25519_cert_t **output, const uint8_t *input, const size_t l
   }
   return result;
 }
+extend2_cell_body_t *
+extend2_cell_body_new(void)
+{
+  extend2_cell_body_t *val = trunnel_calloc(1, sizeof(extend2_cell_body_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+extend2_cell_body_clear(extend2_cell_body_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->ls);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->ls);
+  create2_cell_body_free(obj->create2);
+  obj->create2 = NULL;
+}
+
+void
+extend2_cell_body_free(extend2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return;
+  extend2_cell_body_clear(obj);
+  trunnel_memwipe(obj, sizeof(extend2_cell_body_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+extend2_cell_body_get_n_spec(const extend2_cell_body_t *inp)
+{
+  return inp->n_spec;
+}
+int
+extend2_cell_body_set_n_spec(extend2_cell_body_t *inp, uint8_t val)
+{
+  inp->n_spec = val;
+  return 0;
+}
+size_t
+extend2_cell_body_getlen_ls(const extend2_cell_body_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->ls);
+}
+
+struct link_specifier_st *
+extend2_cell_body_get_ls(extend2_cell_body_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->ls, idx);
+}
+
+ const struct link_specifier_st *
+extend2_cell_body_getconst_ls(const extend2_cell_body_t *inp, size_t idx)
+{
+  return extend2_cell_body_get_ls((extend2_cell_body_t*)inp, idx);
+}
+int
+extend2_cell_body_set_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->ls, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return extend2_cell_body_set0_ls(inp, idx, elt);
+}
+int
+extend2_cell_body_set0_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->ls, idx, elt);
+  return 0;
+}
+int
+extend2_cell_body_add_ls(extend2_cell_body_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->ls.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->ls, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+extend2_cell_body_getarray_ls(extend2_cell_body_t *inp)
+{
+  return inp->ls.elts_;
+}
+const struct link_specifier_st *  const  *
+extend2_cell_body_getconstarray_ls(const extend2_cell_body_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)extend2_cell_body_getarray_ls((extend2_cell_body_t*)inp);
+}
+int
+extend2_cell_body_setlen_ls(extend2_cell_body_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->ls.allocated_,
+                 &inp->ls.n_, inp->ls.elts_, newlen,
+                 sizeof(inp->ls.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->ls.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct create2_cell_body_st *
+extend2_cell_body_get_create2(extend2_cell_body_t *inp)
+{
+  return inp->create2;
+}
+const struct create2_cell_body_st *
+extend2_cell_body_getconst_create2(const extend2_cell_body_t *inp)
+{
+  return extend2_cell_body_get_create2((extend2_cell_body_t*) inp);
+}
+int
+extend2_cell_body_set_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val)
+{
+  if (inp->create2 && inp->create2 != val)
+    create2_cell_body_free(inp->create2);
+  return extend2_cell_body_set0_create2(inp, val);
+}
+int
+extend2_cell_body_set0_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val)
+{
+  inp->create2 = val;
+  return 0;
+}
+const char *
+extend2_cell_body_check(const extend2_cell_body_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->ls, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->ls) != obj->n_spec)
+    return "Length mismatch for ls";
+  {
+    const char *msg;
+    if (NULL != (msg = create2_cell_body_check(obj->create2)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+extend2_cell_body_encoded_len(const extend2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != extend2_cell_body_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_spec */
+  result += 1;
+
+  /* Length of struct link_specifier ls[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+    }
+  }
+
+  /* Length of struct create2_cell_body create2 */
+  result += create2_cell_body_encoded_len(obj->create2);
+  return result;
+}
+int
+extend2_cell_body_clear_errors(extend2_cell_body_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+extend2_cell_body_encode(uint8_t *output, const size_t avail, const extend2_cell_body_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = extend2_cell_body_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = extend2_cell_body_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_spec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_spec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier ls[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->ls); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->ls, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+  /* Encode struct create2_cell_body create2 */
+  trunnel_assert(written <= avail);
+  result = create2_cell_body_encode(ptr, avail - written, obj->create2);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As extend2_cell_body_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+extend2_cell_body_parse_into(extend2_cell_body_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_spec */
+  CHECK_REMAINING(1, truncated);
+  obj->n_spec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier ls[n_spec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->ls, obj->n_spec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_spec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->ls, elt, {link_specifier_free(elt);});
+    }
+  }
+
+  /* Parse struct create2_cell_body create2 */
+  result = create2_cell_body_parse(&obj->create2, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+extend2_cell_body_parse(extend2_cell_body_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = extend2_cell_body_new();
+  if (NULL == *output)
+    return -1;
+  result = extend2_cell_body_parse_into(*output, input, len_in);
+  if (result < 0) {
+    extend2_cell_body_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+link_specifier_list_t *
+link_specifier_list_new(void)
+{
+  link_specifier_list_t *val = trunnel_calloc(1, sizeof(link_specifier_list_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+link_specifier_list_clear(link_specifier_list_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->spec);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->spec);
+}
+
+void
+link_specifier_list_free(link_specifier_list_t *obj)
+{
+  if (obj == NULL)
+    return;
+  link_specifier_list_clear(obj);
+  trunnel_memwipe(obj, sizeof(link_specifier_list_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+link_specifier_list_get_n_spec(const link_specifier_list_t *inp)
+{
+  return inp->n_spec;
+}
+int
+link_specifier_list_set_n_spec(link_specifier_list_t *inp, uint8_t val)
+{
+  inp->n_spec = val;
+  return 0;
+}
+size_t
+link_specifier_list_getlen_spec(const link_specifier_list_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->spec);
+}
+
+struct link_specifier_st *
+link_specifier_list_get_spec(link_specifier_list_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->spec, idx);
+}
+
+ const struct link_specifier_st *
+link_specifier_list_getconst_spec(const link_specifier_list_t *inp, size_t idx)
+{
+  return link_specifier_list_get_spec((link_specifier_list_t*)inp, idx);
+}
+int
+link_specifier_list_set_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->spec, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return link_specifier_list_set0_spec(inp, idx, elt);
+}
+int
+link_specifier_list_set0_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->spec, idx, elt);
+  return 0;
+}
+int
+link_specifier_list_add_spec(link_specifier_list_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->spec.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->spec, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+link_specifier_list_getarray_spec(link_specifier_list_t *inp)
+{
+  return inp->spec.elts_;
+}
+const struct link_specifier_st *  const  *
+link_specifier_list_getconstarray_spec(const link_specifier_list_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)link_specifier_list_getarray_spec((link_specifier_list_t*)inp);
+}
+int
+link_specifier_list_setlen_spec(link_specifier_list_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->spec.allocated_,
+                 &inp->spec.n_, inp->spec.elts_, newlen,
+                 sizeof(inp->spec.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->spec.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+link_specifier_list_check(const link_specifier_list_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->spec, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->spec) != obj->n_spec)
+    return "Length mismatch for spec";
+  return NULL;
+}
+
+ssize_t
+link_specifier_list_encoded_len(const link_specifier_list_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != link_specifier_list_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_spec */
+  result += 1;
+
+  /* Length of struct link_specifier spec[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+    }
+  }
+  return result;
+}
+int
+link_specifier_list_clear_errors(link_specifier_list_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+link_specifier_list_encode(uint8_t *output, const size_t avail, const link_specifier_list_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = link_specifier_list_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = link_specifier_list_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_spec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_spec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier spec[n_spec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->spec); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->spec, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As link_specifier_list_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+link_specifier_list_parse_into(link_specifier_list_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_spec */
+  CHECK_REMAINING(1, truncated);
+  obj->n_spec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier spec[n_spec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->spec, obj->n_spec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_spec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->spec, elt, {link_specifier_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+link_specifier_list_parse(link_specifier_list_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = link_specifier_list_new();
+  if (NULL == *output)
+    return -1;
+  result = link_specifier_list_parse_into(*output, input, len_in);
+  if (result < 0) {
+    link_specifier_list_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index 28f6feef31..782bd59585 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by by Trunnel v1.4.6.
+/* ed25519_cert.h -- generated by by Trunnel v1.5.1.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,19 @@
 
 #define CERTEXT_SIGNED_WITH_KEY 4
 #define CERTEXT_FLAG_AFFECTS_VALIDATION 1
+#define LS_IPV4 0
+#define LS_IPV6 1
+#define LS_LEGACY_ID 2
+#define LS_ED25519_ID 3
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CREATE2_CELL_BODY)
+struct create2_cell_body_st {
+  uint16_t handshake_type;
+  uint16_t handshake_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) handshake_data;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct create2_cell_body_st create2_cell_body_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_ED25519_CERT_EXTENSION)
 struct ed25519_cert_extension_st {
   uint16_t ext_length;
@@ -21,6 +34,31 @@ struct ed25519_cert_extension_st {
 };
 #endif
 typedef struct ed25519_cert_extension_st ed25519_cert_extension_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_EXTEND1_CELL_BODY)
+struct extend1_cell_body_st {
+  uint32_t ipv4addr;
+  uint16_t port;
+  uint8_t onionskin[186];
+  uint8_t identity[20];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct extend1_cell_body_st extend1_cell_body_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_LINK_SPECIFIER)
+struct link_specifier_st {
+  uint8_t ls_type;
+  uint8_t ls_len;
+  uint32_t un_ipv4_addr;
+  uint16_t un_ipv4_port;
+  uint8_t un_ipv6_addr[16];
+  uint16_t un_ipv6_port;
+  uint8_t un_legacy_id[20];
+  uint8_t un_ed25519_id[32];
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) un_unrecognized;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct link_specifier_st link_specifier_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_ED25519_CERT)
 struct ed25519_cert_st {
   uint8_t version;
@@ -35,6 +73,112 @@ struct ed25519_cert_st {
 };
 #endif
 typedef struct ed25519_cert_st ed25519_cert_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_EXTEND2_CELL_BODY)
+struct extend2_cell_body_st {
+  uint8_t n_spec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) ls;
+  struct create2_cell_body_st *create2;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct extend2_cell_body_st extend2_cell_body_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_LINK_SPECIFIER_LIST)
+struct link_specifier_list_st {
+  uint8_t n_spec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) spec;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct link_specifier_list_st link_specifier_list_t;
+/** Return a newly allocated create2_cell_body with all elements set
+ * to zero.
+ */
+create2_cell_body_t *create2_cell_body_new(void);
+/** Release all storage held by the create2_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void create2_cell_body_free(create2_cell_body_t *victim);
+/** Try to parse a create2_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * create2_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t create2_cell_body_parse(create2_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * create2_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t create2_cell_body_encoded_len(const create2_cell_body_t *obj);
+/** Try to encode the create2_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t create2_cell_body_encode(uint8_t *output, size_t avail, const create2_cell_body_t *input);
+/** Check whether the internal state of the create2_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *create2_cell_body_check(const create2_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int create2_cell_body_clear_errors(create2_cell_body_t *obj);
+/** Return the value of the handshake_type field of the
+ * create2_cell_body_t in 'inp'
+ */
+uint16_t create2_cell_body_get_handshake_type(const create2_cell_body_t *inp);
+/** Set the value of the handshake_type field of the
+ * create2_cell_body_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int create2_cell_body_set_handshake_type(create2_cell_body_t *inp, uint16_t val);
+/** Return the value of the handshake_len field of the
+ * create2_cell_body_t in 'inp'
+ */
+uint16_t create2_cell_body_get_handshake_len(const create2_cell_body_t *inp);
+/** Set the value of the handshake_len field of the
+ * create2_cell_body_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int create2_cell_body_set_handshake_len(create2_cell_body_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the handshake_data
+ * field of the create2_cell_body_t in 'inp'.
+ */
+size_t create2_cell_body_getlen_handshake_data(const create2_cell_body_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp'.
+ */
+uint8_t create2_cell_body_get_handshake_data(create2_cell_body_t *inp, size_t idx);
+/** As create2_cell_body_get_handshake_data, but take and return a
+ * const pointer
+ */
+uint8_t create2_cell_body_getconst_handshake_data(const create2_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int create2_cell_body_set_handshake_data(create2_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * handshake_data of the create2_cell_body_t in 'inp'.
+ */
+int create2_cell_body_add_handshake_data(create2_cell_body_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field handshake_data
+ * of 'inp'.
+ */
+uint8_t * create2_cell_body_getarray_handshake_data(create2_cell_body_t *inp);
+/** As create2_cell_body_get_handshake_data, but take and return a
+ * const pointer
+ */
+const uint8_t  * create2_cell_body_getconstarray_handshake_data(const create2_cell_body_t *inp);
+/** Change the length of the variable-length array field
+ * handshake_data of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int create2_cell_body_setlen_handshake_data(create2_cell_body_t *inp, size_t newlen);
 /** Return a newly allocated ed25519_cert_extension with all elements
  * set to zero.
  */
@@ -74,7 +218,7 @@ int ed25519_cert_extension_clear_errors(ed25519_cert_extension_t *obj);
 /** Return the value of the ext_length field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint16_t ed25519_cert_extension_get_ext_length(ed25519_cert_extension_t *inp);
+uint16_t ed25519_cert_extension_get_ext_length(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_length field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -83,7 +227,7 @@ int ed25519_cert_extension_set_ext_length(ed25519_cert_extension_t *inp, uint16_
 /** Return the value of the ext_type field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint8_t ed25519_cert_extension_get_ext_type(ed25519_cert_extension_t *inp);
+uint8_t ed25519_cert_extension_get_ext_type(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_type field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -92,7 +236,7 @@ int ed25519_cert_extension_set_ext_type(ed25519_cert_extension_t *inp, uint8_t v
 /** Return the value of the ext_flags field of the
  * ed25519_cert_extension_t in 'inp'
  */
-uint8_t ed25519_cert_extension_get_ext_flags(ed25519_cert_extension_t *inp);
+uint8_t ed25519_cert_extension_get_ext_flags(const ed25519_cert_extension_t *inp);
 /** Set the value of the ext_flags field of the
  * ed25519_cert_extension_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -105,7 +249,11 @@ size_t ed25519_cert_extension_getlen_un_signing_key(const ed25519_cert_extension
 /** Return the element at position 'idx' of the fixed array field
  * un_signing_key of the ed25519_cert_extension_t in 'inp'.
  */
-uint8_t ed25519_cert_extension_get_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx);
+uint8_t ed25519_cert_extension_get_un_signing_key(ed25519_cert_extension_t *inp, size_t idx);
+/** As ed25519_cert_extension_get_un_signing_key, but take and return
+ * a const pointer
+ */
+uint8_t ed25519_cert_extension_getconst_un_signing_key(const ed25519_cert_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * un_signing_key of the ed25519_cert_extension_t in 'inp', so that it
  * will hold the value 'elt'.
@@ -115,6 +263,10 @@ int ed25519_cert_extension_set_un_signing_key(ed25519_cert_extension_t *inp, siz
  * 'inp'.
  */
 uint8_t * ed25519_cert_extension_getarray_un_signing_key(ed25519_cert_extension_t *inp);
+/** As ed25519_cert_extension_get_un_signing_key, but take and return
+ * a const pointer
+ */
+const uint8_t  * ed25519_cert_extension_getconstarray_un_signing_key(const ed25519_cert_extension_t *inp);
 /** Return the length of the dynamic array holding the un_unparsed
  * field of the ed25519_cert_extension_t in 'inp'.
  */
@@ -123,6 +275,10 @@ size_t ed25519_cert_extension_getlen_un_unparsed(const ed25519_cert_extension_t
  * un_unparsed of the ed25519_cert_extension_t in 'inp'.
  */
 uint8_t ed25519_cert_extension_get_un_unparsed(ed25519_cert_extension_t *inp, size_t idx);
+/** As ed25519_cert_extension_get_un_unparsed, but take and return a
+ * const pointer
+ */
+uint8_t ed25519_cert_extension_getconst_un_unparsed(const ed25519_cert_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * un_unparsed of the ed25519_cert_extension_t in 'inp', so that it
  * will hold the value 'elt'.
@@ -136,11 +292,308 @@ int ed25519_cert_extension_add_un_unparsed(ed25519_cert_extension_t *inp, uint8_
  * 'inp'.
  */
 uint8_t * ed25519_cert_extension_getarray_un_unparsed(ed25519_cert_extension_t *inp);
+/** As ed25519_cert_extension_get_un_unparsed, but take and return a
+ * const pointer
+ */
+const uint8_t  * ed25519_cert_extension_getconstarray_un_unparsed(const ed25519_cert_extension_t *inp);
 /** Change the length of the variable-length array field un_unparsed
  * of 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on
  * success; return -1 and set the error code on 'inp' on failure.
  */
 int ed25519_cert_extension_setlen_un_unparsed(ed25519_cert_extension_t *inp, size_t newlen);
+/** Return a newly allocated extend1_cell_body with all elements set
+ * to zero.
+ */
+extend1_cell_body_t *extend1_cell_body_new(void);
+/** Release all storage held by the extend1_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void extend1_cell_body_free(extend1_cell_body_t *victim);
+/** Try to parse a extend1_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * extend1_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t extend1_cell_body_parse(extend1_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * extend1_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t extend1_cell_body_encoded_len(const extend1_cell_body_t *obj);
+/** Try to encode the extend1_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t extend1_cell_body_encode(uint8_t *output, size_t avail, const extend1_cell_body_t *input);
+/** Check whether the internal state of the extend1_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *extend1_cell_body_check(const extend1_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int extend1_cell_body_clear_errors(extend1_cell_body_t *obj);
+/** Return the value of the ipv4addr field of the extend1_cell_body_t
+ * in 'inp'
+ */
+uint32_t extend1_cell_body_get_ipv4addr(const extend1_cell_body_t *inp);
+/** Set the value of the ipv4addr field of the extend1_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend1_cell_body_set_ipv4addr(extend1_cell_body_t *inp, uint32_t val);
+/** Return the value of the port field of the extend1_cell_body_t in
+ * 'inp'
+ */
+uint16_t extend1_cell_body_get_port(const extend1_cell_body_t *inp);
+/** Set the value of the port field of the extend1_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend1_cell_body_set_port(extend1_cell_body_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the onionskin
+ * field of the extend1_cell_body_t in 'inp'.
+ */
+size_t extend1_cell_body_getlen_onionskin(const extend1_cell_body_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * onionskin of the extend1_cell_body_t in 'inp'.
+ */
+uint8_t extend1_cell_body_get_onionskin(extend1_cell_body_t *inp, size_t idx);
+/** As extend1_cell_body_get_onionskin, but take and return a const
+ * pointer
+ */
+uint8_t extend1_cell_body_getconst_onionskin(const extend1_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * onionskin of the extend1_cell_body_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int extend1_cell_body_set_onionskin(extend1_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 186-element array field onionskin of
+ * 'inp'.
+ */
+uint8_t * extend1_cell_body_getarray_onionskin(extend1_cell_body_t *inp);
+/** As extend1_cell_body_get_onionskin, but take and return a const
+ * pointer
+ */
+const uint8_t  * extend1_cell_body_getconstarray_onionskin(const extend1_cell_body_t *inp);
+/** Return the (constant) length of the array holding the identity
+ * field of the extend1_cell_body_t in 'inp'.
+ */
+size_t extend1_cell_body_getlen_identity(const extend1_cell_body_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * identity of the extend1_cell_body_t in 'inp'.
+ */
+uint8_t extend1_cell_body_get_identity(extend1_cell_body_t *inp, size_t idx);
+/** As extend1_cell_body_get_identity, but take and return a const
+ * pointer
+ */
+uint8_t extend1_cell_body_getconst_identity(const extend1_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * identity of the extend1_cell_body_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int extend1_cell_body_set_identity(extend1_cell_body_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 20-element array field identity of 'inp'.
+ */
+uint8_t * extend1_cell_body_getarray_identity(extend1_cell_body_t *inp);
+/** As extend1_cell_body_get_identity, but take and return a const
+ * pointer
+ */
+const uint8_t  * extend1_cell_body_getconstarray_identity(const extend1_cell_body_t *inp);
+/** Return a newly allocated link_specifier with all elements set to
+ * zero.
+ */
+link_specifier_t *link_specifier_new(void);
+/** Release all storage held by the link_specifier in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void link_specifier_free(link_specifier_t *victim);
+/** Try to parse a link_specifier from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * link_specifier_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * link_specifier in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t link_specifier_encoded_len(const link_specifier_t *obj);
+/** Try to encode the link_specifier from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t link_specifier_encode(uint8_t *output, size_t avail, const link_specifier_t *input);
+/** Check whether the internal state of the link_specifier in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *link_specifier_check(const link_specifier_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int link_specifier_clear_errors(link_specifier_t *obj);
+/** Return the value of the ls_type field of the link_specifier_t in
+ * 'inp'
+ */
+uint8_t link_specifier_get_ls_type(const link_specifier_t *inp);
+/** Set the value of the ls_type field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_ls_type(link_specifier_t *inp, uint8_t val);
+/** Return the value of the ls_len field of the link_specifier_t in
+ * 'inp'
+ */
+uint8_t link_specifier_get_ls_len(const link_specifier_t *inp);
+/** Set the value of the ls_len field of the link_specifier_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int link_specifier_set_ls_len(link_specifier_t *inp, uint8_t val);
+/** Return the value of the un_ipv4_addr field of the link_specifier_t
+ * in 'inp'
+ */
+uint32_t link_specifier_get_un_ipv4_addr(const link_specifier_t *inp);
+/** Set the value of the un_ipv4_addr field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv4_addr(link_specifier_t *inp, uint32_t val);
+/** Return the value of the un_ipv4_port field of the link_specifier_t
+ * in 'inp'
+ */
+uint16_t link_specifier_get_un_ipv4_port(const link_specifier_t *inp);
+/** Set the value of the un_ipv4_port field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv4_port(link_specifier_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the un_ipv6_addr
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_ipv6_addr(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_ipv6_addr of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_ipv6_addr(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_ipv6_addr, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_ipv6_addr(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_ipv6_addr of the link_specifier_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int link_specifier_set_un_ipv6_addr(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field un_ipv6_addr of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_ipv6_addr(link_specifier_t *inp);
+/** As link_specifier_get_un_ipv6_addr, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_ipv6_addr(const link_specifier_t *inp);
+/** Return the value of the un_ipv6_port field of the link_specifier_t
+ * in 'inp'
+ */
+uint16_t link_specifier_get_un_ipv6_port(const link_specifier_t *inp);
+/** Set the value of the un_ipv6_port field of the link_specifier_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_set_un_ipv6_port(link_specifier_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the un_legacy_id
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_legacy_id(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_legacy_id of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_legacy_id(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_legacy_id, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_legacy_id(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_legacy_id of the link_specifier_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int link_specifier_set_un_legacy_id(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 20-element array field un_legacy_id of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_legacy_id(link_specifier_t *inp);
+/** As link_specifier_get_un_legacy_id, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_legacy_id(const link_specifier_t *inp);
+/** Return the (constant) length of the array holding the
+ * un_ed25519_id field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_ed25519_id(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * un_ed25519_id of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_ed25519_id(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_ed25519_id, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_ed25519_id(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * un_ed25519_id of the link_specifier_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int link_specifier_set_un_ed25519_id(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 32-element array field un_ed25519_id of
+ * 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_ed25519_id(link_specifier_t *inp);
+/** As link_specifier_get_un_ed25519_id, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_ed25519_id(const link_specifier_t *inp);
+/** Return the length of the dynamic array holding the un_unrecognized
+ * field of the link_specifier_t in 'inp'.
+ */
+size_t link_specifier_getlen_un_unrecognized(const link_specifier_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp'.
+ */
+uint8_t link_specifier_get_un_unrecognized(link_specifier_t *inp, size_t idx);
+/** As link_specifier_get_un_unrecognized, but take and return a const
+ * pointer
+ */
+uint8_t link_specifier_getconst_un_unrecognized(const link_specifier_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int link_specifier_set_un_unrecognized(link_specifier_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field
+ * un_unrecognized of the link_specifier_t in 'inp'.
+ */
+int link_specifier_add_un_unrecognized(link_specifier_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field
+ * un_unrecognized of 'inp'.
+ */
+uint8_t * link_specifier_getarray_un_unrecognized(link_specifier_t *inp);
+/** As link_specifier_get_un_unrecognized, but take and return a const
+ * pointer
+ */
+const uint8_t  * link_specifier_getconstarray_un_unrecognized(const link_specifier_t *inp);
+/** Change the length of the variable-length array field
+ * un_unrecognized of 'inp' to 'newlen'.Fill extra elements with 0.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int link_specifier_setlen_un_unrecognized(link_specifier_t *inp, size_t newlen);
 /** Return a newly allocated ed25519_cert with all elements set to
  * zero.
  */
@@ -179,7 +632,7 @@ int ed25519_cert_clear_errors(ed25519_cert_t *obj);
 /** Return the value of the version field of the ed25519_cert_t in
  * 'inp'
  */
-uint8_t ed25519_cert_get_version(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_version(const ed25519_cert_t *inp);
 /** Set the value of the version field of the ed25519_cert_t in 'inp'
  * to 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -188,7 +641,7 @@ int ed25519_cert_set_version(ed25519_cert_t *inp, uint8_t val);
 /** Return the value of the cert_type field of the ed25519_cert_t in
  * 'inp'
  */
-uint8_t ed25519_cert_get_cert_type(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_cert_type(const ed25519_cert_t *inp);
 /** Set the value of the cert_type field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -197,7 +650,7 @@ int ed25519_cert_set_cert_type(ed25519_cert_t *inp, uint8_t val);
 /** Return the value of the exp_field field of the ed25519_cert_t in
  * 'inp'
  */
-uint32_t ed25519_cert_get_exp_field(ed25519_cert_t *inp);
+uint32_t ed25519_cert_get_exp_field(const ed25519_cert_t *inp);
 /** Set the value of the exp_field field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -206,7 +659,7 @@ int ed25519_cert_set_exp_field(ed25519_cert_t *inp, uint32_t val);
 /** Return the value of the cert_key_type field of the ed25519_cert_t
  * in 'inp'
  */
-uint8_t ed25519_cert_get_cert_key_type(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_cert_key_type(const ed25519_cert_t *inp);
 /** Set the value of the cert_key_type field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -219,7 +672,11 @@ size_t ed25519_cert_getlen_certified_key(const ed25519_cert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * certified_key of the ed25519_cert_t in 'inp'.
  */
-uint8_t ed25519_cert_get_certified_key(const ed25519_cert_t *inp, size_t idx);
+uint8_t ed25519_cert_get_certified_key(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_certified_key, but take and return a const
+ * pointer
+ */
+uint8_t ed25519_cert_getconst_certified_key(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * certified_key of the ed25519_cert_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -229,10 +686,14 @@ int ed25519_cert_set_certified_key(ed25519_cert_t *inp, size_t idx, uint8_t elt)
  * 'inp'.
  */
 uint8_t * ed25519_cert_getarray_certified_key(ed25519_cert_t *inp);
+/** As ed25519_cert_get_certified_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * ed25519_cert_getconstarray_certified_key(const ed25519_cert_t *inp);
 /** Return the value of the n_extensions field of the ed25519_cert_t
  * in 'inp'
  */
-uint8_t ed25519_cert_get_n_extensions(ed25519_cert_t *inp);
+uint8_t ed25519_cert_get_n_extensions(const ed25519_cert_t *inp);
 /** Set the value of the n_extensions field of the ed25519_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -246,6 +707,9 @@ size_t ed25519_cert_getlen_ext(const ed25519_cert_t *inp);
  * ext of the ed25519_cert_t in 'inp'.
  */
 struct ed25519_cert_extension_st * ed25519_cert_get_ext(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_ext, but take and return a const pointer
+ */
+ const struct ed25519_cert_extension_st * ed25519_cert_getconst_ext(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * ext of the ed25519_cert_t in 'inp', so that it will hold the value
  * 'elt'. Free the previous value, if any.
@@ -261,6 +725,9 @@ int ed25519_cert_add_ext(ed25519_cert_t *inp, struct ed25519_cert_extension_st *
 /** Return a pointer to the variable-length array field ext of 'inp'.
  */
 struct ed25519_cert_extension_st * * ed25519_cert_getarray_ext(ed25519_cert_t *inp);
+/** As ed25519_cert_get_ext, but take and return a const pointer
+ */
+const struct ed25519_cert_extension_st *  const  * ed25519_cert_getconstarray_ext(const ed25519_cert_t *inp);
 /** Change the length of the variable-length array field ext of 'inp'
  * to 'newlen'.Fill extra elements with NULL; free removed elements.
  * Return 0 on success; return -1 and set the error code on 'inp' on
@@ -274,7 +741,10 @@ size_t ed25519_cert_getlen_signature(const ed25519_cert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * signature of the ed25519_cert_t in 'inp'.
  */
-uint8_t ed25519_cert_get_signature(const ed25519_cert_t *inp, size_t idx);
+uint8_t ed25519_cert_get_signature(ed25519_cert_t *inp, size_t idx);
+/** As ed25519_cert_get_signature, but take and return a const pointer
+ */
+uint8_t ed25519_cert_getconst_signature(const ed25519_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * signature of the ed25519_cert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -283,6 +753,190 @@ int ed25519_cert_set_signature(ed25519_cert_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 64-element array field signature of 'inp'.
  */
 uint8_t * ed25519_cert_getarray_signature(ed25519_cert_t *inp);
+/** As ed25519_cert_get_signature, but take and return a const pointer
+ */
+const uint8_t  * ed25519_cert_getconstarray_signature(const ed25519_cert_t *inp);
+/** Return a newly allocated extend2_cell_body with all elements set
+ * to zero.
+ */
+extend2_cell_body_t *extend2_cell_body_new(void);
+/** Release all storage held by the extend2_cell_body in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void extend2_cell_body_free(extend2_cell_body_t *victim);
+/** Try to parse a extend2_cell_body from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * extend2_cell_body_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t extend2_cell_body_parse(extend2_cell_body_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * extend2_cell_body in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t extend2_cell_body_encoded_len(const extend2_cell_body_t *obj);
+/** Try to encode the extend2_cell_body from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t extend2_cell_body_encode(uint8_t *output, size_t avail, const extend2_cell_body_t *input);
+/** Check whether the internal state of the extend2_cell_body in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *extend2_cell_body_check(const extend2_cell_body_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int extend2_cell_body_clear_errors(extend2_cell_body_t *obj);
+/** Return the value of the n_spec field of the extend2_cell_body_t in
+ * 'inp'
+ */
+uint8_t extend2_cell_body_get_n_spec(const extend2_cell_body_t *inp);
+/** Set the value of the n_spec field of the extend2_cell_body_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int extend2_cell_body_set_n_spec(extend2_cell_body_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the ls field of the
+ * extend2_cell_body_t in 'inp'.
+ */
+size_t extend2_cell_body_getlen_ls(const extend2_cell_body_t *inp);
+/** Return the element at position 'idx' of the dynamic array field ls
+ * of the extend2_cell_body_t in 'inp'.
+ */
+struct link_specifier_st * extend2_cell_body_get_ls(extend2_cell_body_t *inp, size_t idx);
+/** As extend2_cell_body_get_ls, but take and return a const pointer
+ */
+ const struct link_specifier_st * extend2_cell_body_getconst_ls(const extend2_cell_body_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field ls
+ * of the extend2_cell_body_t in 'inp', so that it will hold the value
+ * 'elt'. Free the previous value, if any.
+ */
+int extend2_cell_body_set_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As extend2_cell_body_set_ls, but does not free the previous value.
+ */
+int extend2_cell_body_set0_ls(extend2_cell_body_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field ls of the
+ * extend2_cell_body_t in 'inp'.
+ */
+int extend2_cell_body_add_ls(extend2_cell_body_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field ls of 'inp'.
+ */
+struct link_specifier_st * * extend2_cell_body_getarray_ls(extend2_cell_body_t *inp);
+/** As extend2_cell_body_get_ls, but take and return a const pointer
+ */
+const struct link_specifier_st *  const  * extend2_cell_body_getconstarray_ls(const extend2_cell_body_t *inp);
+/** Change the length of the variable-length array field ls of 'inp'
+ * to 'newlen'.Fill extra elements with NULL; free removed elements.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int extend2_cell_body_setlen_ls(extend2_cell_body_t *inp, size_t newlen);
+/** Return the value of the create2 field of the extend2_cell_body_t
+ * in 'inp'
+ */
+struct create2_cell_body_st * extend2_cell_body_get_create2(extend2_cell_body_t *inp);
+/** As extend2_cell_body_get_create2, but take and return a const
+ * pointer
+ */
+const struct create2_cell_body_st * extend2_cell_body_getconst_create2(const extend2_cell_body_t *inp);
+/** Set the value of the create2 field of the extend2_cell_body_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int extend2_cell_body_set_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val);
+/** As extend2_cell_body_set_create2, but does not free the previous
+ * value.
+ */
+int extend2_cell_body_set0_create2(extend2_cell_body_t *inp, struct create2_cell_body_st *val);
+/** Return a newly allocated link_specifier_list with all elements set
+ * to zero.
+ */
+link_specifier_list_t *link_specifier_list_new(void);
+/** Release all storage held by the link_specifier_list in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void link_specifier_list_free(link_specifier_list_t *victim);
+/** Try to parse a link_specifier_list from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated link_specifier_list_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t link_specifier_list_parse(link_specifier_list_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * link_specifier_list in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t link_specifier_list_encoded_len(const link_specifier_list_t *obj);
+/** Try to encode the link_specifier_list from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t link_specifier_list_encode(uint8_t *output, size_t avail, const link_specifier_list_t *input);
+/** Check whether the internal state of the link_specifier_list in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *link_specifier_list_check(const link_specifier_list_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int link_specifier_list_clear_errors(link_specifier_list_t *obj);
+/** Return the value of the n_spec field of the link_specifier_list_t
+ * in 'inp'
+ */
+uint8_t link_specifier_list_get_n_spec(const link_specifier_list_t *inp);
+/** Set the value of the n_spec field of the link_specifier_list_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int link_specifier_list_set_n_spec(link_specifier_list_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the spec field of
+ * the link_specifier_list_t in 'inp'.
+ */
+size_t link_specifier_list_getlen_spec(const link_specifier_list_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * spec of the link_specifier_list_t in 'inp'.
+ */
+struct link_specifier_st * link_specifier_list_get_spec(link_specifier_list_t *inp, size_t idx);
+/** As link_specifier_list_get_spec, but take and return a const
+ * pointer
+ */
+ const struct link_specifier_st * link_specifier_list_getconst_spec(const link_specifier_list_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * spec of the link_specifier_list_t in 'inp', so that it will hold
+ * the value 'elt'. Free the previous value, if any.
+ */
+int link_specifier_list_set_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As link_specifier_list_set_spec, but does not free the previous
+ * value.
+ */
+int link_specifier_list_set0_spec(link_specifier_list_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field spec of the
+ * link_specifier_list_t in 'inp'.
+ */
+int link_specifier_list_add_spec(link_specifier_list_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field spec of 'inp'.
+ */
+struct link_specifier_st * * link_specifier_list_getarray_spec(link_specifier_list_t *inp);
+/** As link_specifier_list_get_spec, but take and return a const
+ * pointer
+ */
+const struct link_specifier_st *  const  * link_specifier_list_getconstarray_spec(const link_specifier_list_t *inp);
+/** Change the length of the variable-length array field spec of 'inp'
+ * to 'newlen'.Fill extra elements with NULL; free removed elements.
+ * Return 0 on success; return -1 and set the error code on 'inp' on
+ * failure.
+ */
+int link_specifier_list_setlen_spec(link_specifier_list_t *inp, size_t newlen);
 
 
 #endif
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index c46f1b6c6b..e424ce5464 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -23,39 +23,6 @@ struct ed25519_cert_extension {
   };
 }
 
-/*
-struct cert_revocation {
-  u8 prefix[8];
-  u8 version IN [1];
-  u8 keytype;
-  u8 identity_key[32];
-  u8 revoked_key[32];
-  u64 published;
-  u8 n_extensions;
-  struct cert_extension ext[n_extensions];
-  u8 signature[64];
-}
-
-struct crosscert_ed_rsa {
-  u8 ed_key[32];
-  u32 expiration_date;
-  u8 signature[128];
-}
-
-struct auth02_cell {
-  u8 type[8];
-  u8 cid[32];
-  u8 sid[32];
-  u8 cid_ed[32];
-  u8 sid_ed[32];
-  u8 slog[32];
-  u8 clog[32];
-  u8 scert[32];
-  u8 tlssecrets[32];
-  u8 rand[24];
-  u8 sig[64];
-}
-
 const LS_IPV4 = 0x00;
 const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
@@ -73,4 +40,27 @@ struct link_specifier {
     default: u8 unrecognized[];
   };
 }
-*/
\ No newline at end of file
+
+struct link_specifier_list {
+  u8 n_spec;
+  struct link_specifier spec[n_spec];
+}
+
+struct extend1_cell_body {
+  u32 ipv4addr;
+  u16 port;
+  u8 onionskin[186];
+  u8 identity[20];
+}
+
+struct create2_cell_body {
+  u16 handshake_type;
+  u16 handshake_len;
+  u8 handshake_data[handshake_len];
+}
+
+struct extend2_cell_body {
+  u8 n_spec;
+  struct link_specifier ls[n_spec];
+  struct create2_cell_body create2;
+}
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
new file mode 100644
index 0000000000..b7f19ffc60
--- /dev/null
+++ b/src/trunnel/hs/cell_common.c
@@ -0,0 +1,595 @@
+/* cell_common.c -- generated by Trunnel v1.5.1.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_common.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're runnning a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellcommon_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellcommon_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+trn_cell_extension_fields_t *
+trn_cell_extension_fields_new(void)
+{
+  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->field);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->field);
+}
+
+void
+trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_fields_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+{
+  return inp->field_type;
+}
+int
+trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+{
+  inp->field_type = val;
+  return 0;
+}
+uint8_t
+trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+{
+  return inp->field_len;
+}
+int
+trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+{
+  inp->field_len = val;
+  return 0;
+}
+size_t
+trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->field);
+}
+
+uint8_t
+trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
+}
+
+uint8_t
+trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+{
+  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+}
+int
+trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->field.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->field, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+{
+  return inp->field.elts_;
+}
+const uint8_t  *
+trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+{
+  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+}
+int
+trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->field.allocated_,
+                 &inp->field.n_, inp->field.elts_, newlen,
+                 sizeof(inp->field.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->field.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->field) != obj->field_len)
+    return "Length mismatch for field";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_fields_check(obj))
+     return -1;
+
+
+  /* Length of u8 field_type */
+  result += 1;
+
+  /* Length of u8 field_len */
+  result += 1;
+
+  /* Length of u8 field[field_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->field);
+  return result;
+}
+int
+trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 field_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->field_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 field_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->field_len));
+  written += 1; ptr += 1;
+
+  /* Encode u8 field[field_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->field);
+    trunnel_assert(obj->field_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->field.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_fields_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 field_type */
+  CHECK_REMAINING(1, truncated);
+  obj->field_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 field_len */
+  CHECK_REMAINING(1, truncated);
+  obj->field_len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 field[field_len] */
+  CHECK_REMAINING(obj->field_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->field, obj->field_len, {});
+  obj->field.n_ = obj->field_len;
+  if (obj->field_len)
+    memcpy(obj->field.elts_, ptr, obj->field_len);
+  ptr += obj->field_len; remaining -= obj->field_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_fields_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_fields_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_extension_t *
+trn_cell_extension_new(void)
+{
+  trn_cell_extension_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_clear(trn_cell_extension_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->fields);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->fields);
+}
+
+void
+trn_cell_extension_free(trn_cell_extension_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_get_num(const trn_cell_extension_t *inp)
+{
+  return inp->num;
+}
+int
+trn_cell_extension_set_num(trn_cell_extension_t *inp, uint8_t val)
+{
+  inp->num = val;
+  return 0;
+}
+size_t
+trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->fields);
+}
+
+struct trn_cell_extension_fields_st *
+trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+}
+
+ const struct trn_cell_extension_fields_st *
+trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
+{
+  return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
+}
+int
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+{
+  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_fields_free(oldval);
+  return trn_cell_extension_set0_fields(inp, idx, elt);
+}
+int
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->fields.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_fields_st * *
+trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
+{
+  return inp->fields.elts_;
+}
+const struct trn_cell_extension_fields_st *  const  *
+trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
+{
+  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+}
+int
+trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_fields_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
+                 &inp->fields.n_, inp->fields.elts_, newlen,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->fields.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_check(const trn_cell_extension_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->fields) != obj->num)
+    return "Length mismatch for fields";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_check(obj))
+     return -1;
+
+
+  /* Length of u8 num */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_fields fields[num] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_clear_errors(trn_cell_extension_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_extension_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 num */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->num));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_fields fields[num] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 num */
+  CHECK_REMAINING(1, truncated);
+  obj->num = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_fields fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  {
+    trn_cell_extension_fields_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->num; ++idx) {
+      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
new file mode 100644
index 0000000000..4d98a54cf4
--- /dev/null
+++ b/src/trunnel/hs/cell_common.h
@@ -0,0 +1,203 @@
+/* cell_common.h -- generated by by Trunnel v1.5.1.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_COMMON_H
+#define TRUNNEL_CELL_COMMON_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
+struct trn_cell_extension_fields_st {
+  uint8_t field_type;
+  uint8_t field_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
+struct trn_cell_extension_st {
+  uint8_t num;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+/** Return a newly allocated trn_cell_extension_fields with all
+ * elements set to zero.
+ */
+trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
+/** Release all storage held by the trn_cell_extension_fields in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
+/** Try to parse a trn_cell_extension_fields from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
+/** Try to encode the trn_cell_extension_fields from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
+/** Check whether the internal state of the trn_cell_extension_fields
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+/** Return the value of the field_type field of the
+ * trn_cell_extension_fields_t in 'inp'
+ */
+uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+/** Set the value of the field_type field of the
+ * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+/** Return the value of the field_len field of the
+ * trn_cell_extension_fields_t in 'inp'
+ */
+uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+/** Set the value of the field_len field of the
+ * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the field field of
+ * the trn_cell_extension_fields_t in 'inp'.
+ */
+size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * field of the trn_cell_extension_fields_t in 'inp'.
+ */
+uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
+/** As trn_cell_extension_fields_get_field, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field field of the
+ * trn_cell_extension_fields_t in 'inp'.
+ */
+int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field field of
+ * 'inp'.
+ */
+uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
+/** As trn_cell_extension_fields_get_field, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+/** Change the length of the variable-length array field field of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension with all elements set
+ * to zero.
+ */
+trn_cell_extension_t *trn_cell_extension_new(void);
+/** Release all storage held by the trn_cell_extension in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+/** Try to parse a trn_cell_extension from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+/** Try to encode the trn_cell_extension from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+/** Check whether the internal state of the trn_cell_extension in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+/** Return the value of the num field of the trn_cell_extension_t in
+ * 'inp'
+ */
+uint8_t trn_cell_extension_get_num(const trn_cell_extension_t *inp);
+/** Set the value of the num field of the trn_cell_extension_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_extension_set_num(trn_cell_extension_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the fields field of
+ * the trn_cell_extension_t in 'inp'.
+ */
+size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * fields of the trn_cell_extension_t in 'inp'.
+ */
+struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+/** As trn_cell_extension_get_fields, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * fields of the trn_cell_extension_t in 'inp', so that it will hold
+ * the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+/** As trn_cell_extension_set_fields, but does not free the previous
+ * value.
+ */
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+/** Append a new element 'elt' to the dynamic array field fields of
+ * the trn_cell_extension_t in 'inp'.
+ */
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+/** Return a pointer to the variable-length array field fields of
+ * 'inp'.
+ */
+struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+/** As trn_cell_extension_get_fields, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+/** Change the length of the variable-length array field fields of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
new file mode 100644
index 0000000000..1aa6999de7
--- /dev/null
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -0,0 +1,12 @@
+/* This file contains common data structure that cells use. */
+
+struct trn_cell_extension_fields {
+  u8 field_type;
+  u8 field_len;
+  u8 field[field_len];
+};
+
+struct trn_cell_extension {
+  u8 num;
+  struct trn_cell_extension_fields fields[num];
+};
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
new file mode 100644
index 0000000000..22e198c369
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -0,0 +1,735 @@
+/* cell_establish_intro.c -- generated by Trunnel v1.5.1.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_establish_intro.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're runnning a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellestablishintro_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellestablishintro_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+trn_cell_extension_t *trn_cell_extension_new(void);
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_establish_intro_t *
+trn_cell_establish_intro_new(void)
+{
+  trn_cell_establish_intro_t *val = trunnel_calloc(1, sizeof(trn_cell_establish_intro_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_establish_intro_clear(trn_cell_establish_intro_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->auth_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->auth_key);
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->sig);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->sig);
+}
+
+void
+trn_cell_establish_intro_free(trn_cell_establish_intro_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_establish_intro_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_establish_intro_t));
+  trunnel_free_(obj);
+}
+
+const uint8_t *
+trn_cell_establish_intro_get_start_cell(const trn_cell_establish_intro_t *inp)
+{
+  return inp->start_cell;
+}
+uint8_t
+trn_cell_establish_intro_get_auth_key_type(const trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key_type;
+}
+int
+trn_cell_establish_intro_set_auth_key_type(trn_cell_establish_intro_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->auth_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_establish_intro_get_auth_key_len(const trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key_len;
+}
+int
+trn_cell_establish_intro_set_auth_key_len(trn_cell_establish_intro_t *inp, uint16_t val)
+{
+  inp->auth_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_establish_intro_getlen_auth_key(const trn_cell_establish_intro_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->auth_key);
+}
+
+uint8_t
+trn_cell_establish_intro_get_auth_key(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->auth_key, idx);
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_auth_key(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_auth_key((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_auth_key(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->auth_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_establish_intro_add_auth_key(trn_cell_establish_intro_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->auth_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->auth_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_auth_key(trn_cell_establish_intro_t *inp)
+{
+  return inp->auth_key.elts_;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_auth_key(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_auth_key((trn_cell_establish_intro_t*)inp);
+}
+int
+trn_cell_establish_intro_setlen_auth_key(trn_cell_establish_intro_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->auth_key.allocated_,
+                 &inp->auth_key.n_, inp->auth_key.elts_, newlen,
+                 sizeof(inp->auth_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->auth_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct trn_cell_extension_st *
+trn_cell_establish_intro_get_extensions(trn_cell_establish_intro_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_establish_intro_getconst_extensions(const trn_cell_establish_intro_t *inp)
+{
+  return trn_cell_establish_intro_get_extensions((trn_cell_establish_intro_t*) inp);
+}
+int
+trn_cell_establish_intro_set_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_establish_intro_set0_extensions(inp, val);
+}
+int
+trn_cell_establish_intro_set0_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const uint8_t *
+trn_cell_establish_intro_get_end_mac_fields(const trn_cell_establish_intro_t *inp)
+{
+  return inp->end_mac_fields;
+}
+size_t
+trn_cell_establish_intro_getlen_handshake_mac(const trn_cell_establish_intro_t *inp)
+{
+  (void)inp;  return TRUNNEL_SHA3_256_LEN;
+}
+
+uint8_t
+trn_cell_establish_intro_get_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SHA3_256_LEN);
+  return inp->handshake_mac[idx];
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_handshake_mac(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_handshake_mac((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SHA3_256_LEN);
+  inp->handshake_mac[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_handshake_mac(trn_cell_establish_intro_t *inp)
+{
+  return inp->handshake_mac;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_handshake_mac(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_handshake_mac((trn_cell_establish_intro_t*)inp);
+}
+const uint8_t *
+trn_cell_establish_intro_get_end_sig_fields(const trn_cell_establish_intro_t *inp)
+{
+  return inp->end_sig_fields;
+}
+uint16_t
+trn_cell_establish_intro_get_sig_len(const trn_cell_establish_intro_t *inp)
+{
+  return inp->sig_len;
+}
+int
+trn_cell_establish_intro_set_sig_len(trn_cell_establish_intro_t *inp, uint16_t val)
+{
+  inp->sig_len = val;
+  return 0;
+}
+size_t
+trn_cell_establish_intro_getlen_sig(const trn_cell_establish_intro_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->sig);
+}
+
+uint8_t
+trn_cell_establish_intro_get_sig(trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
+}
+
+uint8_t
+trn_cell_establish_intro_getconst_sig(const trn_cell_establish_intro_t *inp, size_t idx)
+{
+  return trn_cell_establish_intro_get_sig((trn_cell_establish_intro_t*)inp, idx);
+}
+int
+trn_cell_establish_intro_set_sig(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->sig, idx, elt);
+  return 0;
+}
+int
+trn_cell_establish_intro_add_sig(trn_cell_establish_intro_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->sig.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->sig, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_establish_intro_getarray_sig(trn_cell_establish_intro_t *inp)
+{
+  return inp->sig.elts_;
+}
+const uint8_t  *
+trn_cell_establish_intro_getconstarray_sig(const trn_cell_establish_intro_t *inp)
+{
+  return (const uint8_t  *)trn_cell_establish_intro_getarray_sig((trn_cell_establish_intro_t*)inp);
+}
+int
+trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->sig.allocated_,
+                 &inp->sig.n_, inp->sig.elts_, newlen,
+                 sizeof(inp->sig.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->sig.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_establish_intro_check(const trn_cell_establish_intro_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->auth_key) != obj->auth_key_len)
+    return "Length mismatch for auth_key";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->sig) != obj->sig_len)
+    return "Length mismatch for sig";
+  return NULL;
+}
+
+ssize_t
+trn_cell_establish_intro_encoded_len(const trn_cell_establish_intro_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_establish_intro_check(obj))
+     return -1;
+
+
+  /* Length of u8 auth_key_type IN [0, 1, 2] */
+  result += 1;
+
+  /* Length of u16 auth_key_len */
+  result += 2;
+
+  /* Length of u8 auth_key[auth_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  result += TRUNNEL_SHA3_256_LEN;
+
+  /* Length of u16 sig_len */
+  result += 2;
+
+  /* Length of u8 sig[sig_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->sig);
+  return result;
+}
+int
+trn_cell_establish_intro_clear_errors(trn_cell_establish_intro_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_establish_intro_encode(uint8_t *output, const size_t avail, const trn_cell_establish_intro_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_establish_intro_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_establish_intro_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 auth_key_type IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->auth_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 auth_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->auth_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 auth_key[auth_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+    trunnel_assert(obj->auth_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->auth_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_SHA3_256_LEN)
+    goto truncated;
+  memcpy(ptr, obj->handshake_mac, TRUNNEL_SHA3_256_LEN);
+  written += TRUNNEL_SHA3_256_LEN; ptr += TRUNNEL_SHA3_256_LEN;
+
+  /* Encode u16 sig_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->sig_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 sig[sig_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->sig);
+    trunnel_assert(obj->sig_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->sig.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_establish_intro_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_establish_intro_parse_into(trn_cell_establish_intro_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+  obj->start_cell = ptr;
+
+  /* Parse u8 auth_key_type IN [0, 1, 2] */
+  CHECK_REMAINING(1, truncated);
+  obj->auth_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    goto fail;
+
+  /* Parse u16 auth_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->auth_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 auth_key[auth_key_len] */
+  CHECK_REMAINING(obj->auth_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->auth_key, obj->auth_key_len, {});
+  obj->auth_key.n_ = obj->auth_key_len;
+  if (obj->auth_key_len)
+    memcpy(obj->auth_key.elts_, ptr, obj->auth_key_len);
+  ptr += obj->auth_key_len; remaining -= obj->auth_key_len;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  obj->end_mac_fields = ptr;
+
+  /* Parse u8 handshake_mac[TRUNNEL_SHA3_256_LEN] */
+  CHECK_REMAINING(TRUNNEL_SHA3_256_LEN, truncated);
+  memcpy(obj->handshake_mac, ptr, TRUNNEL_SHA3_256_LEN);
+  remaining -= TRUNNEL_SHA3_256_LEN; ptr += TRUNNEL_SHA3_256_LEN;
+  obj->end_sig_fields = ptr;
+
+  /* Parse u16 sig_len */
+  CHECK_REMAINING(2, truncated);
+  obj->sig_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 sig[sig_len] */
+  CHECK_REMAINING(obj->sig_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->sig, obj->sig_len, {});
+  obj->sig.n_ = obj->sig_len;
+  if (obj->sig_len)
+    memcpy(obj->sig.elts_, ptr, obj->sig_len);
+  ptr += obj->sig_len; remaining -= obj->sig_len;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_establish_intro_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_establish_intro_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_establish_intro_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_intro_established_t *
+trn_cell_intro_established_new(void)
+{
+  trn_cell_intro_established_t *val = trunnel_calloc(1, sizeof(trn_cell_intro_established_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_intro_established_clear(trn_cell_intro_established_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+}
+
+void
+trn_cell_intro_established_free(trn_cell_intro_established_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_intro_established_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_intro_established_t));
+  trunnel_free_(obj);
+}
+
+struct trn_cell_extension_st *
+trn_cell_intro_established_get_extensions(trn_cell_intro_established_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_intro_established_getconst_extensions(const trn_cell_intro_established_t *inp)
+{
+  return trn_cell_intro_established_get_extensions((trn_cell_intro_established_t*) inp);
+}
+int
+trn_cell_intro_established_set_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_intro_established_set0_extensions(inp, val);
+}
+int
+trn_cell_intro_established_set0_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const char *
+trn_cell_intro_established_check(const trn_cell_intro_established_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_intro_established_encoded_len(const trn_cell_intro_established_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_intro_established_check(obj))
+     return -1;
+
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+  return result;
+}
+int
+trn_cell_intro_established_clear_errors(trn_cell_intro_established_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_intro_established_encode(uint8_t *output, const size_t avail, const trn_cell_intro_established_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_intro_established_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_intro_established_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_intro_established_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_intro_established_parse_into(trn_cell_intro_established_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+ssize_t
+trn_cell_intro_established_parse(trn_cell_intro_established_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_intro_established_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_intro_established_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_intro_established_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
new file mode 100644
index 0000000000..2f0d893659
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -0,0 +1,276 @@
+/* cell_establish_intro.h -- generated by by Trunnel v1.5.1.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_ESTABLISH_INTRO_H
+#define TRUNNEL_CELL_ESTABLISH_INTRO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+struct trn_cell_extension_st;
+#define TRUNNEL_SHA3_256_LEN 32
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
+struct trn_cell_establish_intro_st {
+  const uint8_t *start_cell;
+  uint8_t auth_key_type;
+  uint16_t auth_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) auth_key;
+  struct trn_cell_extension_st *extensions;
+  const uint8_t *end_mac_fields;
+  uint8_t handshake_mac[TRUNNEL_SHA3_256_LEN];
+  const uint8_t *end_sig_fields;
+  uint16_t sig_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) sig;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
+struct trn_cell_intro_established_st {
+  struct trn_cell_extension_st *extensions;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_establish_intro with all
+ * elements set to zero.
+ */
+trn_cell_establish_intro_t *trn_cell_establish_intro_new(void);
+/** Release all storage held by the trn_cell_establish_intro in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_establish_intro_free(trn_cell_establish_intro_t *victim);
+/** Try to parse a trn_cell_establish_intro from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_establish_intro_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_establish_intro in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_establish_intro_encoded_len(const trn_cell_establish_intro_t *obj);
+/** Try to encode the trn_cell_establish_intro from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_establish_intro_encode(uint8_t *output, size_t avail, const trn_cell_establish_intro_t *input);
+/** Check whether the internal state of the trn_cell_establish_intro
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_establish_intro_check(const trn_cell_establish_intro_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_establish_intro_clear_errors(trn_cell_establish_intro_t *obj);
+/** Return the position for start_cell when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_start_cell(const trn_cell_establish_intro_t *inp);
+/** Return the value of the auth_key_type field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint8_t trn_cell_establish_intro_get_auth_key_type(const trn_cell_establish_intro_t *inp);
+/** Set the value of the auth_key_type field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_auth_key_type(trn_cell_establish_intro_t *inp, uint8_t val);
+/** Return the value of the auth_key_len field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint16_t trn_cell_establish_intro_get_auth_key_len(const trn_cell_establish_intro_t *inp);
+/** Set the value of the auth_key_len field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_auth_key_len(trn_cell_establish_intro_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the auth_key field
+ * of the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_auth_key(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_auth_key(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_auth_key, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_auth_key(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_establish_intro_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_auth_key(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field auth_key of
+ * the trn_cell_establish_intro_t in 'inp'.
+ */
+int trn_cell_establish_intro_add_auth_key(trn_cell_establish_intro_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field auth_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_auth_key(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_auth_key, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_auth_key(const trn_cell_establish_intro_t *inp);
+/** Change the length of the variable-length array field auth_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_setlen_auth_key(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return the value of the extensions field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_establish_intro_get_extensions(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_extensions, but take and return a
+ * const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_establish_intro_getconst_extensions(const trn_cell_establish_intro_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_establish_intro_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_establish_intro_set0_extensions(trn_cell_establish_intro_t *inp, struct trn_cell_extension_st *val);
+/** Return the position for end_mac_fields when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_end_mac_fields(const trn_cell_establish_intro_t *inp);
+/** Return the (constant) length of the array holding the
+ * handshake_mac field of the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_handshake_mac(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * handshake_mac of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_handshake_mac, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_handshake_mac(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * handshake_mac of the trn_cell_establish_intro_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_handshake_mac(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SHA3_256_LEN-element array field
+ * handshake_mac of 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_handshake_mac(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_handshake_mac, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_handshake_mac(const trn_cell_establish_intro_t *inp);
+/** Return the position for end_sig_fields when we parsed this object
+ */
+const uint8_t * trn_cell_establish_intro_get_end_sig_fields(const trn_cell_establish_intro_t *inp);
+/** Return the value of the sig_len field of the
+ * trn_cell_establish_intro_t in 'inp'
+ */
+uint16_t trn_cell_establish_intro_get_sig_len(const trn_cell_establish_intro_t *inp);
+/** Set the value of the sig_len field of the
+ * trn_cell_establish_intro_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_set_sig_len(trn_cell_establish_intro_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the sig field of
+ * the trn_cell_establish_intro_t in 'inp'.
+ */
+size_t trn_cell_establish_intro_getlen_sig(const trn_cell_establish_intro_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * sig of the trn_cell_establish_intro_t in 'inp'.
+ */
+uint8_t trn_cell_establish_intro_get_sig(trn_cell_establish_intro_t *inp, size_t idx);
+/** As trn_cell_establish_intro_get_sig, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_establish_intro_getconst_sig(const trn_cell_establish_intro_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * sig of the trn_cell_establish_intro_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_establish_intro_set_sig(trn_cell_establish_intro_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field sig of the
+ * trn_cell_establish_intro_t in 'inp'.
+ */
+int trn_cell_establish_intro_add_sig(trn_cell_establish_intro_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field sig of 'inp'.
+ */
+uint8_t * trn_cell_establish_intro_getarray_sig(trn_cell_establish_intro_t *inp);
+/** As trn_cell_establish_intro_get_sig, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_establish_intro_t *inp);
+/** Change the length of the variable-length array field sig of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_intro_established with all
+ * elements set to zero.
+ */
+trn_cell_intro_established_t *trn_cell_intro_established_new(void);
+/** Release all storage held by the trn_cell_intro_established in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_intro_established_free(trn_cell_intro_established_t *victim);
+/** Try to parse a trn_cell_intro_established from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_intro_established_t. On failure, return -2
+ * if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_intro_established_parse(trn_cell_intro_established_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_intro_established in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_intro_established_encoded_len(const trn_cell_intro_established_t *obj);
+/** Try to encode the trn_cell_intro_established from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_intro_established_encode(uint8_t *output, size_t avail, const trn_cell_intro_established_t *input);
+/** Check whether the internal state of the trn_cell_intro_established
+ * in 'obj' is consistent. Return NULL if it is, and a short message
+ * if it is not.
+ */
+const char *trn_cell_intro_established_check(const trn_cell_intro_established_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_intro_established_clear_errors(trn_cell_intro_established_t *obj);
+/** Return the value of the extensions field of the
+ * trn_cell_intro_established_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_intro_established_get_extensions(trn_cell_intro_established_t *inp);
+/** As trn_cell_intro_established_get_extensions, but take and return
+ * a const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_intro_established_getconst_extensions(const trn_cell_intro_established_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_intro_established_t in 'inp' to 'val'. Free the old value
+ * if any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_intro_established_set_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_intro_established_set_extensions, but does not free
+ * the previous value.
+ */
+int trn_cell_intro_established_set0_extensions(trn_cell_intro_established_t *inp, struct trn_cell_extension_st *val);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
new file mode 100644
index 0000000000..011ee62a15
--- /dev/null
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -0,0 +1,41 @@
+/*
+ * This contains the definition of the ESTABLISH_INTRO and INTRO_ESTABLISHED
+ * cell for onion service version 3 and onward. The following format is
+ * specified in proposal 224 section 3.1.
+ */
+
+extern struct trn_cell_extension;
+
+const TRUNNEL_SHA3_256_LEN = 32;
+
+/* ESTABLISH_INTRO payload. See details in section 3.1.1 */
+struct trn_cell_establish_intro {
+  /* Indicate the start of the handshake authentication data. */
+  @ptr start_cell;
+
+  /* Authentication key material. */
+  u8 auth_key_type IN [0x00, 0x01, 0x02];
+  u16 auth_key_len;
+  u8 auth_key[auth_key_len];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+  @ptr end_mac_fields;
+
+  /* Handshake MAC. */
+  u8 handshake_mac[TRUNNEL_SHA3_256_LEN];
+
+  /* Signature */
+  /* Indicate the end of the handshake authentication data. */
+  @ptr end_sig_fields;
+  u16 sig_len;
+  u8 sig[sig_len];
+};
+
+/* INTRO_ESTABLISHED payload which is an acknowledge of the ESTABLISH_INTRO
+ * cell. For legacy node, this payload is empty so the following only applies
+ * to version >= 3. */
+struct trn_cell_intro_established {
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
new file mode 100644
index 0000000000..7501f6f196
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -0,0 +1,1357 @@
+/* cell_introduce1.c -- generated by Trunnel v1.5.1.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "cell_introduce1.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're runnning a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int cellintroduce_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || cellintroduce_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+typedef struct trn_cell_extension_st trn_cell_extension_t;
+trn_cell_extension_t *trn_cell_extension_new(void);
+void trn_cell_extension_free(trn_cell_extension_t *victim);
+ssize_t trn_cell_extension_parse(trn_cell_extension_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
+ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
+const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
+int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+typedef struct link_specifier_st link_specifier_t;
+link_specifier_t *link_specifier_new(void);
+void link_specifier_free(link_specifier_t *victim);
+ssize_t link_specifier_parse(link_specifier_t **output, const uint8_t *input, const size_t len_in);
+ssize_t link_specifier_encoded_len(const link_specifier_t *obj);
+ssize_t link_specifier_encode(uint8_t *output, size_t avail, const link_specifier_t *input);
+const char *link_specifier_check(const link_specifier_t *obj);
+int link_specifier_clear_errors(link_specifier_t *obj);
+trn_cell_introduce1_t *
+trn_cell_introduce1_new(void)
+{
+  trn_cell_introduce1_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce1_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce1_clear(trn_cell_introduce1_t *obj)
+{
+  (void) obj;
+  TRUNNEL_DYNARRAY_WIPE(&obj->auth_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->auth_key);
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->encrypted);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->encrypted);
+}
+
+void
+trn_cell_introduce1_free(trn_cell_introduce1_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce1_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce1_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_introduce1_getlen_legacy_key_id(const trn_cell_introduce1_t *inp)
+{
+  (void)inp;  return TRUNNEL_SHA1_LEN;
+}
+
+uint8_t
+trn_cell_introduce1_get_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SHA1_LEN);
+  return inp->legacy_key_id[idx];
+}
+
+uint8_t
+trn_cell_introduce1_getconst_legacy_key_id(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_legacy_key_id((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SHA1_LEN);
+  inp->legacy_key_id[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_legacy_key_id(trn_cell_introduce1_t *inp)
+{
+  return inp->legacy_key_id;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_legacy_key_id(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_legacy_key_id((trn_cell_introduce1_t*)inp);
+}
+uint8_t
+trn_cell_introduce1_get_auth_key_type(const trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key_type;
+}
+int
+trn_cell_introduce1_set_auth_key_type(trn_cell_introduce1_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->auth_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_introduce1_get_auth_key_len(const trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key_len;
+}
+int
+trn_cell_introduce1_set_auth_key_len(trn_cell_introduce1_t *inp, uint16_t val)
+{
+  inp->auth_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_introduce1_getlen_auth_key(const trn_cell_introduce1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->auth_key);
+}
+
+uint8_t
+trn_cell_introduce1_get_auth_key(trn_cell_introduce1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->auth_key, idx);
+}
+
+uint8_t
+trn_cell_introduce1_getconst_auth_key(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_auth_key((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_auth_key(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->auth_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce1_add_auth_key(trn_cell_introduce1_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->auth_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->auth_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_auth_key(trn_cell_introduce1_t *inp)
+{
+  return inp->auth_key.elts_;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_auth_key(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_auth_key((trn_cell_introduce1_t*)inp);
+}
+int
+trn_cell_introduce1_setlen_auth_key(trn_cell_introduce1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->auth_key.allocated_,
+                 &inp->auth_key.n_, inp->auth_key.elts_, newlen,
+                 sizeof(inp->auth_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->auth_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+struct trn_cell_extension_st *
+trn_cell_introduce1_get_extensions(trn_cell_introduce1_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce1_getconst_extensions(const trn_cell_introduce1_t *inp)
+{
+  return trn_cell_introduce1_get_extensions((trn_cell_introduce1_t*) inp);
+}
+int
+trn_cell_introduce1_set_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce1_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce1_set0_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+size_t
+trn_cell_introduce1_getlen_encrypted(const trn_cell_introduce1_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->encrypted);
+}
+
+uint8_t
+trn_cell_introduce1_get_encrypted(trn_cell_introduce1_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->encrypted, idx);
+}
+
+uint8_t
+trn_cell_introduce1_getconst_encrypted(const trn_cell_introduce1_t *inp, size_t idx)
+{
+  return trn_cell_introduce1_get_encrypted((trn_cell_introduce1_t*)inp, idx);
+}
+int
+trn_cell_introduce1_set_encrypted(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->encrypted, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce1_add_encrypted(trn_cell_introduce1_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->encrypted, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce1_getarray_encrypted(trn_cell_introduce1_t *inp)
+{
+  return inp->encrypted.elts_;
+}
+const uint8_t  *
+trn_cell_introduce1_getconstarray_encrypted(const trn_cell_introduce1_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce1_getarray_encrypted((trn_cell_introduce1_t*)inp);
+}
+int
+trn_cell_introduce1_setlen_encrypted(trn_cell_introduce1_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->encrypted.allocated_,
+                 &inp->encrypted.n_, inp->encrypted.elts_, newlen,
+                 sizeof(inp->encrypted.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->encrypted.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_introduce1_check(const trn_cell_introduce1_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->auth_key) != obj->auth_key_len)
+    return "Length mismatch for auth_key";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce1_encoded_len(const trn_cell_introduce1_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce1_check(obj))
+     return -1;
+
+
+  /* Length of u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  result += TRUNNEL_SHA1_LEN;
+
+  /* Length of u8 auth_key_type IN [0, 1, 2] */
+  result += 1;
+
+  /* Length of u16 auth_key_len */
+  result += 2;
+
+  /* Length of u8 auth_key[auth_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 encrypted[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->encrypted);
+  return result;
+}
+int
+trn_cell_introduce1_clear_errors(trn_cell_introduce1_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce1_encode(uint8_t *output, const size_t avail, const trn_cell_introduce1_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce1_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce1_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_SHA1_LEN)
+    goto truncated;
+  memcpy(ptr, obj->legacy_key_id, TRUNNEL_SHA1_LEN);
+  written += TRUNNEL_SHA1_LEN; ptr += TRUNNEL_SHA1_LEN;
+
+  /* Encode u8 auth_key_type IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->auth_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 auth_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->auth_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 auth_key[auth_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->auth_key);
+    trunnel_assert(obj->auth_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->auth_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 encrypted[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->encrypted);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->encrypted.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce1_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_introduce1_parse_into(trn_cell_introduce1_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 legacy_key_id[TRUNNEL_SHA1_LEN] */
+  CHECK_REMAINING(TRUNNEL_SHA1_LEN, truncated);
+  memcpy(obj->legacy_key_id, ptr, TRUNNEL_SHA1_LEN);
+  remaining -= TRUNNEL_SHA1_LEN; ptr += TRUNNEL_SHA1_LEN;
+
+  /* Parse u8 auth_key_type IN [0, 1, 2] */
+  CHECK_REMAINING(1, truncated);
+  obj->auth_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->auth_key_type == 0 || obj->auth_key_type == 1 || obj->auth_key_type == 2))
+    goto fail;
+
+  /* Parse u16 auth_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->auth_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 auth_key[auth_key_len] */
+  CHECK_REMAINING(obj->auth_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->auth_key, obj->auth_key_len, {});
+  obj->auth_key.n_ = obj->auth_key_len;
+  if (obj->auth_key_len)
+    memcpy(obj->auth_key.elts_, ptr, obj->auth_key_len);
+  ptr += obj->auth_key_len; remaining -= obj->auth_key_len;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 encrypted[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->encrypted, remaining, {});
+  obj->encrypted.n_ = remaining;
+  if (remaining)
+    memcpy(obj->encrypted.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce1_parse(trn_cell_introduce1_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce1_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce1_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce1_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_introduce_ack_t *
+trn_cell_introduce_ack_new(void)
+{
+  trn_cell_introduce_ack_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce_ack_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce_ack_clear(trn_cell_introduce_ack_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+}
+
+void
+trn_cell_introduce_ack_free(trn_cell_introduce_ack_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce_ack_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce_ack_t));
+  trunnel_free_(obj);
+}
+
+uint16_t
+trn_cell_introduce_ack_get_status(const trn_cell_introduce_ack_t *inp)
+{
+  return inp->status;
+}
+int
+trn_cell_introduce_ack_set_status(trn_cell_introduce_ack_t *inp, uint16_t val)
+{
+  if (! ((val == 0 || val == 1 || val == 2))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->status = val;
+  return 0;
+}
+struct trn_cell_extension_st *
+trn_cell_introduce_ack_get_extensions(trn_cell_introduce_ack_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce_ack_getconst_extensions(const trn_cell_introduce_ack_t *inp)
+{
+  return trn_cell_introduce_ack_get_extensions((trn_cell_introduce_ack_t*) inp);
+}
+int
+trn_cell_introduce_ack_set_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce_ack_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce_ack_set0_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+const char *
+trn_cell_introduce_ack_check(const trn_cell_introduce_ack_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->status == 0 || obj->status == 1 || obj->status == 2))
+    return "Integer out of bounds";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce_ack_encoded_len(const trn_cell_introduce_ack_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce_ack_check(obj))
+     return -1;
+
+
+  /* Length of u16 status IN [0, 1, 2] */
+  result += 2;
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+  return result;
+}
+int
+trn_cell_introduce_ack_clear_errors(trn_cell_introduce_ack_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce_ack_encode(uint8_t *output, const size_t avail, const trn_cell_introduce_ack_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce_ack_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce_ack_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u16 status IN [0, 1, 2] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->status));
+  written += 2; ptr += 2;
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce_ack_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_introduce_ack_parse_into(trn_cell_introduce_ack_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u16 status IN [0, 1, 2] */
+  CHECK_REMAINING(2, truncated);
+  obj->status = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  if (! (obj->status == 0 || obj->status == 1 || obj->status == 2))
+    goto fail;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce_ack_parse(trn_cell_introduce_ack_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce_ack_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce_ack_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce_ack_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+trn_cell_introduce_encrypted_t *
+trn_cell_introduce_encrypted_new(void)
+{
+  trn_cell_introduce_encrypted_t *val = trunnel_calloc(1, sizeof(trn_cell_introduce_encrypted_t));
+  if (NULL == val)
+    return NULL;
+  val->onion_key_type = 1;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_introduce_encrypted_clear(trn_cell_introduce_encrypted_t *obj)
+{
+  (void) obj;
+  trn_cell_extension_free(obj->extensions);
+  obj->extensions = NULL;
+  TRUNNEL_DYNARRAY_WIPE(&obj->onion_key);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->onion_key);
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      link_specifier_free(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->nspecs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->nspecs);
+  TRUNNEL_DYNARRAY_WIPE(&obj->pad);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->pad);
+}
+
+void
+trn_cell_introduce_encrypted_free(trn_cell_introduce_encrypted_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_introduce_encrypted_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_introduce_encrypted_t));
+  trunnel_free_(obj);
+}
+
+size_t
+trn_cell_introduce_encrypted_getlen_rend_cookie(const trn_cell_introduce_encrypted_t *inp)
+{
+  (void)inp;  return TRUNNEL_REND_COOKIE_LEN;
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  return inp->rend_cookie[idx];
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_rend_cookie(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_rend_cookie((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_REND_COOKIE_LEN);
+  inp->rend_cookie[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_rend_cookie(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->rend_cookie;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_rend_cookie(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_rend_cookie((trn_cell_introduce_encrypted_t*)inp);
+}
+struct trn_cell_extension_st *
+trn_cell_introduce_encrypted_get_extensions(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->extensions;
+}
+const struct trn_cell_extension_st *
+trn_cell_introduce_encrypted_getconst_extensions(const trn_cell_introduce_encrypted_t *inp)
+{
+  return trn_cell_introduce_encrypted_get_extensions((trn_cell_introduce_encrypted_t*) inp);
+}
+int
+trn_cell_introduce_encrypted_set_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val)
+{
+  if (inp->extensions && inp->extensions != val)
+    trn_cell_extension_free(inp->extensions);
+  return trn_cell_introduce_encrypted_set0_extensions(inp, val);
+}
+int
+trn_cell_introduce_encrypted_set0_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val)
+{
+  inp->extensions = val;
+  return 0;
+}
+uint8_t
+trn_cell_introduce_encrypted_get_onion_key_type(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key_type;
+}
+int
+trn_cell_introduce_encrypted_set_onion_key_type(trn_cell_introduce_encrypted_t *inp, uint8_t val)
+{
+  if (! ((val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->onion_key_type = val;
+  return 0;
+}
+uint16_t
+trn_cell_introduce_encrypted_get_onion_key_len(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key_len;
+}
+int
+trn_cell_introduce_encrypted_set_onion_key_len(trn_cell_introduce_encrypted_t *inp, uint16_t val)
+{
+  inp->onion_key_len = val;
+  return 0;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_onion_key(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->onion_key);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->onion_key, idx);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_onion_key(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_onion_key((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->onion_key, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_onion_key(trn_cell_introduce_encrypted_t *inp, uint8_t elt)
+{
+#if SIZE_MAX >= UINT16_MAX
+  if (inp->onion_key.n_ == UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->onion_key, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_onion_key(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->onion_key.elts_;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_onion_key(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_onion_key((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_onion_key(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+#if UINT16_MAX < SIZE_MAX
+  if (newlen > UINT16_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->onion_key.allocated_,
+                 &inp->onion_key.n_, inp->onion_key.elts_, newlen,
+                 sizeof(inp->onion_key.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->onion_key.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+uint8_t
+trn_cell_introduce_encrypted_get_nspec(const trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->nspec;
+}
+int
+trn_cell_introduce_encrypted_set_nspec(trn_cell_introduce_encrypted_t *inp, uint8_t val)
+{
+  inp->nspec = val;
+  return 0;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_nspecs(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->nspecs);
+}
+
+struct link_specifier_st *
+trn_cell_introduce_encrypted_get_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->nspecs, idx);
+}
+
+ const struct link_specifier_st *
+trn_cell_introduce_encrypted_getconst_nspecs(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_nspecs((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  link_specifier_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->nspecs, idx);
+  if (oldval && oldval != elt)
+    link_specifier_free(oldval);
+  return trn_cell_introduce_encrypted_set0_nspecs(inp, idx, elt);
+}
+int
+trn_cell_introduce_encrypted_set0_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->nspecs, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_nspecs(trn_cell_introduce_encrypted_t *inp, struct link_specifier_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->nspecs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct link_specifier_st *, &inp->nspecs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct link_specifier_st * *
+trn_cell_introduce_encrypted_getarray_nspecs(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->nspecs.elts_;
+}
+const struct link_specifier_st *  const  *
+trn_cell_introduce_encrypted_getconstarray_nspecs(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const struct link_specifier_st *  const  *)trn_cell_introduce_encrypted_getarray_nspecs((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_nspecs(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  struct link_specifier_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->nspecs.allocated_,
+                 &inp->nspecs.n_, inp->nspecs.elts_, newlen,
+                 sizeof(inp->nspecs.elts_[0]), (trunnel_free_fn_t) link_specifier_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->nspecs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+size_t
+trn_cell_introduce_encrypted_getlen_pad(const trn_cell_introduce_encrypted_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->pad);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_get_pad(trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->pad, idx);
+}
+
+uint8_t
+trn_cell_introduce_encrypted_getconst_pad(const trn_cell_introduce_encrypted_t *inp, size_t idx)
+{
+  return trn_cell_introduce_encrypted_get_pad((trn_cell_introduce_encrypted_t*)inp, idx);
+}
+int
+trn_cell_introduce_encrypted_set_pad(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->pad, idx, elt);
+  return 0;
+}
+int
+trn_cell_introduce_encrypted_add_pad(trn_cell_introduce_encrypted_t *inp, uint8_t elt)
+{
+  TRUNNEL_DYNARRAY_ADD(uint8_t, &inp->pad, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+uint8_t *
+trn_cell_introduce_encrypted_getarray_pad(trn_cell_introduce_encrypted_t *inp)
+{
+  return inp->pad.elts_;
+}
+const uint8_t  *
+trn_cell_introduce_encrypted_getconstarray_pad(const trn_cell_introduce_encrypted_t *inp)
+{
+  return (const uint8_t  *)trn_cell_introduce_encrypted_getarray_pad((trn_cell_introduce_encrypted_t*)inp);
+}
+int
+trn_cell_introduce_encrypted_setlen_pad(trn_cell_introduce_encrypted_t *inp, size_t newlen)
+{
+  uint8_t *newptr;
+  newptr = trunnel_dynarray_setlen(&inp->pad.allocated_,
+                 &inp->pad.n_, inp->pad.elts_, newlen,
+                 sizeof(inp->pad.elts_[0]), (trunnel_free_fn_t) NULL,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->pad.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_introduce_encrypted_check(const trn_cell_introduce_encrypted_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = trn_cell_extension_check(obj->extensions)))
+      return msg;
+  }
+  if (! (obj->onion_key_type == 1))
+    return "Integer out of bounds";
+  if (TRUNNEL_DYNARRAY_LEN(&obj->onion_key) != obj->onion_key_len)
+    return "Length mismatch for onion_key";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      if (NULL != (msg = link_specifier_check(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->nspecs) != obj->nspec)
+    return "Length mismatch for nspecs";
+  return NULL;
+}
+
+ssize_t
+trn_cell_introduce_encrypted_encoded_len(const trn_cell_introduce_encrypted_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_introduce_encrypted_check(obj))
+     return -1;
+
+
+  /* Length of u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  result += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Length of struct trn_cell_extension extensions */
+  result += trn_cell_extension_encoded_len(obj->extensions);
+
+  /* Length of u8 onion_key_type IN [1] */
+  result += 1;
+
+  /* Length of u16 onion_key_len */
+  result += 2;
+
+  /* Length of u8 onion_key[onion_key_len] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->onion_key);
+
+  /* Length of u8 nspec */
+  result += 1;
+
+  /* Length of struct link_specifier nspecs[nspec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      result += link_specifier_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+    }
+  }
+
+  /* Length of u8 pad[] */
+  result += TRUNNEL_DYNARRAY_LEN(&obj->pad);
+  return result;
+}
+int
+trn_cell_introduce_encrypted_clear_errors(trn_cell_introduce_encrypted_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_introduce_encrypted_encode(uint8_t *output, const size_t avail, const trn_cell_introduce_encrypted_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_introduce_encrypted_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_introduce_encrypted_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  trunnel_assert(written <= avail);
+  if (avail - written < TRUNNEL_REND_COOKIE_LEN)
+    goto truncated;
+  memcpy(ptr, obj->rend_cookie, TRUNNEL_REND_COOKIE_LEN);
+  written += TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Encode struct trn_cell_extension extensions */
+  trunnel_assert(written <= avail);
+  result = trn_cell_extension_encode(ptr, avail - written, obj->extensions);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 onion_key_type IN [1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->onion_key_type));
+  written += 1; ptr += 1;
+
+  /* Encode u16 onion_key_len */
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->onion_key_len));
+  written += 2; ptr += 2;
+
+  /* Encode u8 onion_key[onion_key_len] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->onion_key);
+    trunnel_assert(obj->onion_key_len == elt_len);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->onion_key.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+  /* Encode u8 nspec */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->nspec));
+  written += 1; ptr += 1;
+
+  /* Encode struct link_specifier nspecs[nspec] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->nspecs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = link_specifier_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->nspecs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+  /* Encode u8 pad[] */
+  {
+    size_t elt_len = TRUNNEL_DYNARRAY_LEN(&obj->pad);
+    trunnel_assert(written <= avail);
+    if (avail - written < elt_len)
+      goto truncated;
+    if (elt_len)
+      memcpy(ptr, obj->pad.elts_, elt_len);
+    written += elt_len; ptr += elt_len;
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_introduce_encrypted_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_introduce_encrypted_parse_into(trn_cell_introduce_encrypted_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN] */
+  CHECK_REMAINING(TRUNNEL_REND_COOKIE_LEN, truncated);
+  memcpy(obj->rend_cookie, ptr, TRUNNEL_REND_COOKIE_LEN);
+  remaining -= TRUNNEL_REND_COOKIE_LEN; ptr += TRUNNEL_REND_COOKIE_LEN;
+
+  /* Parse struct trn_cell_extension extensions */
+  result = trn_cell_extension_parse(&obj->extensions, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 onion_key_type IN [1] */
+  CHECK_REMAINING(1, truncated);
+  obj->onion_key_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->onion_key_type == 1))
+    goto fail;
+
+  /* Parse u16 onion_key_len */
+  CHECK_REMAINING(2, truncated);
+  obj->onion_key_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+
+  /* Parse u8 onion_key[onion_key_len] */
+  CHECK_REMAINING(obj->onion_key_len, truncated);
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->onion_key, obj->onion_key_len, {});
+  obj->onion_key.n_ = obj->onion_key_len;
+  if (obj->onion_key_len)
+    memcpy(obj->onion_key.elts_, ptr, obj->onion_key_len);
+  ptr += obj->onion_key_len; remaining -= obj->onion_key_len;
+
+  /* Parse u8 nspec */
+  CHECK_REMAINING(1, truncated);
+  obj->nspec = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct link_specifier nspecs[nspec] */
+  TRUNNEL_DYNARRAY_EXPAND(link_specifier_t *, &obj->nspecs, obj->nspec, {});
+  {
+    link_specifier_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->nspec; ++idx) {
+      result = link_specifier_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(link_specifier_t *, &obj->nspecs, elt, {link_specifier_free(elt);});
+    }
+  }
+
+  /* Parse u8 pad[] */
+  TRUNNEL_DYNARRAY_EXPAND(uint8_t, &obj->pad, remaining, {});
+  obj->pad.n_ = remaining;
+  if (remaining)
+    memcpy(obj->pad.elts_, ptr, remaining);
+  ptr += remaining; remaining -= remaining;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+trn_cell_introduce_encrypted_parse(trn_cell_introduce_encrypted_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_introduce_encrypted_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_introduce_encrypted_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_introduce_encrypted_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
new file mode 100644
index 0000000000..cca825a431
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -0,0 +1,493 @@
+/* cell_introduce1.h -- generated by by Trunnel v1.5.1.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CELL_INTRODUCE1_H
+#define TRUNNEL_CELL_INTRODUCE1_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+struct trn_cell_extension_st;
+struct link_specifier_st;
+#define TRUNNEL_SHA1_LEN 20
+#define TRUNNEL_REND_COOKIE_LEN 20
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE1)
+struct trn_cell_introduce1_st {
+  uint8_t legacy_key_id[TRUNNEL_SHA1_LEN];
+  uint8_t auth_key_type;
+  uint16_t auth_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) auth_key;
+  struct trn_cell_extension_st *extensions;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) encrypted;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce1_st trn_cell_introduce1_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE_ACK)
+struct trn_cell_introduce_ack_st {
+  uint16_t status;
+  struct trn_cell_extension_st *extensions;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce_ack_st trn_cell_introduce_ack_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRODUCE_ENCRYPTED)
+struct trn_cell_introduce_encrypted_st {
+  uint8_t rend_cookie[TRUNNEL_REND_COOKIE_LEN];
+  struct trn_cell_extension_st *extensions;
+  uint8_t onion_key_type;
+  uint16_t onion_key_len;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) onion_key;
+  uint8_t nspec;
+  TRUNNEL_DYNARRAY_HEAD(, struct link_specifier_st *) nspecs;
+  TRUNNEL_DYNARRAY_HEAD(, uint8_t) pad;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_introduce_encrypted_st trn_cell_introduce_encrypted_t;
+/** Return a newly allocated trn_cell_introduce1 with all elements set
+ * to zero.
+ */
+trn_cell_introduce1_t *trn_cell_introduce1_new(void);
+/** Release all storage held by the trn_cell_introduce1 in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce1_free(trn_cell_introduce1_t *victim);
+/** Try to parse a trn_cell_introduce1 from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_introduce1_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_introduce1_parse(trn_cell_introduce1_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce1 in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce1_encoded_len(const trn_cell_introduce1_t *obj);
+/** Try to encode the trn_cell_introduce1 from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_introduce1_encode(uint8_t *output, size_t avail, const trn_cell_introduce1_t *input);
+/** Check whether the internal state of the trn_cell_introduce1 in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_introduce1_check(const trn_cell_introduce1_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce1_clear_errors(trn_cell_introduce1_t *obj);
+/** Return the (constant) length of the array holding the
+ * legacy_key_id field of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_legacy_key_id(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * legacy_key_id of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_legacy_key_id, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_introduce1_getconst_legacy_key_id(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * legacy_key_id of the trn_cell_introduce1_t in 'inp', so that it
+ * will hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_legacy_key_id(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SHA1_LEN-element array field
+ * legacy_key_id of 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_legacy_key_id(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_legacy_key_id, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_legacy_key_id(const trn_cell_introduce1_t *inp);
+/** Return the value of the auth_key_type field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+uint8_t trn_cell_introduce1_get_auth_key_type(const trn_cell_introduce1_t *inp);
+/** Set the value of the auth_key_type field of the
+ * trn_cell_introduce1_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_auth_key_type(trn_cell_introduce1_t *inp, uint8_t val);
+/** Return the value of the auth_key_len field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+uint16_t trn_cell_introduce1_get_auth_key_len(const trn_cell_introduce1_t *inp);
+/** Set the value of the auth_key_len field of the
+ * trn_cell_introduce1_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_auth_key_len(trn_cell_introduce1_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the auth_key field
+ * of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_auth_key(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_auth_key(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_auth_key, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_introduce1_getconst_auth_key(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * auth_key of the trn_cell_introduce1_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_auth_key(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field auth_key of
+ * the trn_cell_introduce1_t in 'inp'.
+ */
+int trn_cell_introduce1_add_auth_key(trn_cell_introduce1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field auth_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_auth_key(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_auth_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_auth_key(const trn_cell_introduce1_t *inp);
+/** Change the length of the variable-length array field auth_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_setlen_auth_key(trn_cell_introduce1_t *inp, size_t newlen);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce1_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce1_get_extensions(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_extensions, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce1_getconst_extensions(const trn_cell_introduce1_t *inp);
+/** Set the value of the extensions field of the trn_cell_introduce1_t
+ * in 'inp' to 'val'. Free the old value if any. Steals the
+ * referenceto 'val'.Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_introduce1_set_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce1_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce1_set0_extensions(trn_cell_introduce1_t *inp, struct trn_cell_extension_st *val);
+/** Return the length of the dynamic array holding the encrypted field
+ * of the trn_cell_introduce1_t in 'inp'.
+ */
+size_t trn_cell_introduce1_getlen_encrypted(const trn_cell_introduce1_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * encrypted of the trn_cell_introduce1_t in 'inp'.
+ */
+uint8_t trn_cell_introduce1_get_encrypted(trn_cell_introduce1_t *inp, size_t idx);
+/** As trn_cell_introduce1_get_encrypted, but take and return a const
+ * pointer
+ */
+uint8_t trn_cell_introduce1_getconst_encrypted(const trn_cell_introduce1_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * encrypted of the trn_cell_introduce1_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce1_set_encrypted(trn_cell_introduce1_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field encrypted of
+ * the trn_cell_introduce1_t in 'inp'.
+ */
+int trn_cell_introduce1_add_encrypted(trn_cell_introduce1_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field encrypted of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce1_getarray_encrypted(trn_cell_introduce1_t *inp);
+/** As trn_cell_introduce1_get_encrypted, but take and return a const
+ * pointer
+ */
+const uint8_t  * trn_cell_introduce1_getconstarray_encrypted(const trn_cell_introduce1_t *inp);
+/** Change the length of the variable-length array field encrypted of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce1_setlen_encrypted(trn_cell_introduce1_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_introduce_ack with all elements
+ * set to zero.
+ */
+trn_cell_introduce_ack_t *trn_cell_introduce_ack_new(void);
+/** Release all storage held by the trn_cell_introduce_ack in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce_ack_free(trn_cell_introduce_ack_t *victim);
+/** Try to parse a trn_cell_introduce_ack from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_introduce_ack_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_introduce_ack_parse(trn_cell_introduce_ack_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce_ack in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce_ack_encoded_len(const trn_cell_introduce_ack_t *obj);
+/** Try to encode the trn_cell_introduce_ack from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_introduce_ack_encode(uint8_t *output, size_t avail, const trn_cell_introduce_ack_t *input);
+/** Check whether the internal state of the trn_cell_introduce_ack in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_introduce_ack_check(const trn_cell_introduce_ack_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce_ack_clear_errors(trn_cell_introduce_ack_t *obj);
+/** Return the value of the status field of the
+ * trn_cell_introduce_ack_t in 'inp'
+ */
+uint16_t trn_cell_introduce_ack_get_status(const trn_cell_introduce_ack_t *inp);
+/** Set the value of the status field of the trn_cell_introduce_ack_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int trn_cell_introduce_ack_set_status(trn_cell_introduce_ack_t *inp, uint16_t val);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce_ack_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce_ack_get_extensions(trn_cell_introduce_ack_t *inp);
+/** As trn_cell_introduce_ack_get_extensions, but take and return a
+ * const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce_ack_getconst_extensions(const trn_cell_introduce_ack_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_introduce_ack_t in 'inp' to 'val'. Free the old value if
+ * any. Steals the referenceto 'val'.Return 0 on success; return -1
+ * and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_ack_set_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce_ack_set_extensions, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce_ack_set0_extensions(trn_cell_introduce_ack_t *inp, struct trn_cell_extension_st *val);
+/** Return a newly allocated trn_cell_introduce_encrypted with all
+ * elements set to zero.
+ */
+trn_cell_introduce_encrypted_t *trn_cell_introduce_encrypted_new(void);
+/** Release all storage held by the trn_cell_introduce_encrypted in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_introduce_encrypted_free(trn_cell_introduce_encrypted_t *victim);
+/** Try to parse a trn_cell_introduce_encrypted from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_introduce_encrypted_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_introduce_encrypted_parse(trn_cell_introduce_encrypted_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_introduce_encrypted in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_introduce_encrypted_encoded_len(const trn_cell_introduce_encrypted_t *obj);
+/** Try to encode the trn_cell_introduce_encrypted from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_introduce_encrypted_encode(uint8_t *output, size_t avail, const trn_cell_introduce_encrypted_t *input);
+/** Check whether the internal state of the
+ * trn_cell_introduce_encrypted in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_introduce_encrypted_check(const trn_cell_introduce_encrypted_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_introduce_encrypted_clear_errors(trn_cell_introduce_encrypted_t *obj);
+/** Return the (constant) length of the array holding the rend_cookie
+ * field of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_rend_cookie(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * rend_cookie of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_rend_cookie, but take and
+ * return a const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_rend_cookie(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * rend_cookie of the trn_cell_introduce_encrypted_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_rend_cookie(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_REND_COOKIE_LEN-element array
+ * field rend_cookie of 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_rend_cookie(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_rend_cookie, but take and
+ * return a const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_rend_cookie(const trn_cell_introduce_encrypted_t *inp);
+/** Return the value of the extensions field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+struct trn_cell_extension_st * trn_cell_introduce_encrypted_get_extensions(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_extensions, but take and
+ * return a const pointer
+ */
+const struct trn_cell_extension_st * trn_cell_introduce_encrypted_getconst_extensions(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the extensions field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Free the old
+ * value if any. Steals the referenceto 'val'.Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val);
+/** As trn_cell_introduce_encrypted_set_extensions, but does not free
+ * the previous value.
+ */
+int trn_cell_introduce_encrypted_set0_extensions(trn_cell_introduce_encrypted_t *inp, struct trn_cell_extension_st *val);
+/** Return the value of the onion_key_type field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint8_t trn_cell_introduce_encrypted_get_onion_key_type(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the onion_key_type field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_onion_key_type(trn_cell_introduce_encrypted_t *inp, uint8_t val);
+/** Return the value of the onion_key_len field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint16_t trn_cell_introduce_encrypted_get_onion_key_len(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the onion_key_len field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_onion_key_len(trn_cell_introduce_encrypted_t *inp, uint16_t val);
+/** Return the length of the dynamic array holding the onion_key field
+ * of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_onion_key(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * onion_key of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_onion_key, but take and return
+ * a const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_onion_key(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * onion_key of the trn_cell_introduce_encrypted_t in 'inp', so that
+ * it will hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_onion_key(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field onion_key of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_onion_key(trn_cell_introduce_encrypted_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field onion_key of
+ * 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_onion_key(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_onion_key, but take and return
+ * a const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_onion_key(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field onion_key of
+ * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_onion_key(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+/** Return the value of the nspec field of the
+ * trn_cell_introduce_encrypted_t in 'inp'
+ */
+uint8_t trn_cell_introduce_encrypted_get_nspec(const trn_cell_introduce_encrypted_t *inp);
+/** Set the value of the nspec field of the
+ * trn_cell_introduce_encrypted_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_set_nspec(trn_cell_introduce_encrypted_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the nspecs field of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_nspecs(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * nspecs of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+struct link_specifier_st * trn_cell_introduce_encrypted_get_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_nspecs, but take and return a
+ * const pointer
+ */
+ const struct link_specifier_st * trn_cell_introduce_encrypted_getconst_nspecs(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * nspecs of the trn_cell_introduce_encrypted_t in 'inp', so that it
+ * will hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_introduce_encrypted_set_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt);
+/** As trn_cell_introduce_encrypted_set_nspecs, but does not free the
+ * previous value.
+ */
+int trn_cell_introduce_encrypted_set0_nspecs(trn_cell_introduce_encrypted_t *inp, size_t idx, struct link_specifier_st * elt);
+/** Append a new element 'elt' to the dynamic array field nspecs of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_nspecs(trn_cell_introduce_encrypted_t *inp, struct link_specifier_st * elt);
+/** Return a pointer to the variable-length array field nspecs of
+ * 'inp'.
+ */
+struct link_specifier_st * * trn_cell_introduce_encrypted_getarray_nspecs(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_nspecs, but take and return a
+ * const pointer
+ */
+const struct link_specifier_st *  const  * trn_cell_introduce_encrypted_getconstarray_nspecs(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field nspecs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_nspecs(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+/** Return the length of the dynamic array holding the pad field of
+ * the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+size_t trn_cell_introduce_encrypted_getlen_pad(const trn_cell_introduce_encrypted_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * pad of the trn_cell_introduce_encrypted_t in 'inp'.
+ */
+uint8_t trn_cell_introduce_encrypted_get_pad(trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** As trn_cell_introduce_encrypted_get_pad, but take and return a
+ * const pointer
+ */
+uint8_t trn_cell_introduce_encrypted_getconst_pad(const trn_cell_introduce_encrypted_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * pad of the trn_cell_introduce_encrypted_t in 'inp', so that it will
+ * hold the value 'elt'.
+ */
+int trn_cell_introduce_encrypted_set_pad(trn_cell_introduce_encrypted_t *inp, size_t idx, uint8_t elt);
+/** Append a new element 'elt' to the dynamic array field pad of the
+ * trn_cell_introduce_encrypted_t in 'inp'.
+ */
+int trn_cell_introduce_encrypted_add_pad(trn_cell_introduce_encrypted_t *inp, uint8_t elt);
+/** Return a pointer to the variable-length array field pad of 'inp'.
+ */
+uint8_t * trn_cell_introduce_encrypted_getarray_pad(trn_cell_introduce_encrypted_t *inp);
+/** As trn_cell_introduce_encrypted_get_pad, but take and return a
+ * const pointer
+ */
+const uint8_t  * trn_cell_introduce_encrypted_getconstarray_pad(const trn_cell_introduce_encrypted_t *inp);
+/** Change the length of the variable-length array field pad of 'inp'
+ * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_introduce_encrypted_setlen_pad(trn_cell_introduce_encrypted_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/hs/cell_introduce1.trunnel b/src/trunnel/hs/cell_introduce1.trunnel
new file mode 100644
index 0000000000..7577c1526f
--- /dev/null
+++ b/src/trunnel/hs/cell_introduce1.trunnel
@@ -0,0 +1,60 @@
+/*
+ * This contains the definition of the INTRODUCE1 and INTRODUCE_ACK cell for
+ * onion service version 3 and onward. The following format is specified in
+ * proposal 224 section 3.2.
+ */
+
+/* From cell_common.trunnel. */
+extern struct trn_cell_extension;
+/* From ed25519_cert.trunnel. */
+extern struct link_specifier;
+
+const TRUNNEL_SHA1_LEN = 20;
+const TRUNNEL_REND_COOKIE_LEN = 20;
+
+/* INTRODUCE1 payload. See details in section 3.2.1. */
+struct trn_cell_introduce1 {
+  /* Always zeroed. MUST be checked explicitely by the caller. */
+  u8 legacy_key_id[TRUNNEL_SHA1_LEN];
+
+  /* Authentication key material. */
+  u8 auth_key_type IN [0x00, 0x01, 0x02];
+  u16 auth_key_len;
+  u8 auth_key[auth_key_len];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+
+  /* Variable length, up to the end of cell. */
+  u8 encrypted[];
+};
+
+/* INTRODUCE_ACK payload. See details in section 3.2.2. */
+struct trn_cell_introduce_ack {
+  /* Status of introduction. */
+  u16 status IN [0x0000, 0x0001, 0x0002];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+};
+
+/* Encrypted section of the INTRODUCE1/INTRODUCE2 cell. */
+struct trn_cell_introduce_encrypted {
+  /* Rendezvous cookie. */
+  u8 rend_cookie[TRUNNEL_REND_COOKIE_LEN];
+
+  /* Extension(s). Reserved fields. */
+  struct trn_cell_extension extensions;
+
+  /* Onion key material. */
+  u8 onion_key_type IN [0x01];
+  u16 onion_key_len;
+  u8 onion_key[onion_key_len];
+
+  /* Link specifiers(s) */
+  u8 nspec;
+  struct link_specifier nspecs[nspec];
+
+  /* Optional padding. This might be empty or not. */
+  u8 pad[];
+};
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index b1448b7cb2..de6cf4781f 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -1,4 +1,3 @@
-
 noinst_LIBRARIES += \
 	src/trunnel/libor-trunnel.a
 
@@ -12,21 +11,30 @@ AM_CPPFLAGS += -I$(srcdir)/src/ext/trunnel -I$(srcdir)/src/trunnel
 TRUNNELINPUTS = \
 	src/trunnel/ed25519_cert.trunnel \
 	src/trunnel/link_handshake.trunnel \
-	src/trunnel/pwbox.trunnel
+	src/trunnel/pwbox.trunnel \
+	src/trunnel/channelpadding_negotiation.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
 	src/trunnel/ed25519_cert.c \
 	src/trunnel/link_handshake.c \
-	src/trunnel/pwbox.c
+	src/trunnel/pwbox.c			\
+	src/trunnel/hs/cell_common.c            \
+	src/trunnel/hs/cell_establish_intro.c	\
+	src/trunnel/hs/cell_introduce1.c \
+	src/trunnel/channelpadding_negotiation.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
 	src/ext/trunnel/trunnel-impl.h		\
-	src/trunnel/trunnel-local.h 		\
+	src/trunnel/trunnel-local.h		\
 	src/trunnel/ed25519_cert.h		\
-	src/trunnel/link_handshake.h	 	\
-	src/trunnel/pwbox.h
+	src/trunnel/link_handshake.h		\
+	src/trunnel/pwbox.h			\
+	src/trunnel/hs/cell_common.h            \
+	src/trunnel/hs/cell_establish_intro.h	\
+	src/trunnel/hs/cell_introduce1.h \
+	src/trunnel/channelpadding_negotiation.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
 src_trunnel_libor_trunnel_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS)
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index c2717f36bf..887f710d9c 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.4.6.
+/* link_handshake.c -- generated by Trunnel v1.5.1.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -64,12 +64,17 @@ auth_challenge_cell_getlen_challenge(const auth_challenge_cell_t *inp)
 }
 
 uint8_t
-auth_challenge_cell_get_challenge(const auth_challenge_cell_t *inp, size_t idx)
+auth_challenge_cell_get_challenge(auth_challenge_cell_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->challenge[idx];
 }
 
+uint8_t
+auth_challenge_cell_getconst_challenge(const auth_challenge_cell_t *inp, size_t idx)
+{
+  return auth_challenge_cell_get_challenge((auth_challenge_cell_t*)inp, idx);
+}
 int
 auth_challenge_cell_set_challenge(auth_challenge_cell_t *inp, size_t idx, uint8_t elt)
 {
@@ -83,8 +88,13 @@ auth_challenge_cell_getarray_challenge(auth_challenge_cell_t *inp)
 {
   return inp->challenge;
 }
+const uint8_t  *
+auth_challenge_cell_getconstarray_challenge(const auth_challenge_cell_t *inp)
+{
+  return (const uint8_t  *)auth_challenge_cell_getarray_challenge((auth_challenge_cell_t*)inp);
+}
 uint16_t
-auth_challenge_cell_get_n_methods(auth_challenge_cell_t *inp)
+auth_challenge_cell_get_n_methods(const auth_challenge_cell_t *inp)
 {
   return inp->n_methods;
 }
@@ -106,6 +116,11 @@ auth_challenge_cell_get_methods(auth_challenge_cell_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->methods, idx);
 }
 
+uint16_t
+auth_challenge_cell_getconst_methods(const auth_challenge_cell_t *inp, size_t idx)
+{
+  return auth_challenge_cell_get_methods((auth_challenge_cell_t*)inp, idx);
+}
 int
 auth_challenge_cell_set_methods(auth_challenge_cell_t *inp, size_t idx, uint16_t elt)
 {
@@ -131,6 +146,11 @@ auth_challenge_cell_getarray_methods(auth_challenge_cell_t *inp)
 {
   return inp->methods.elts_;
 }
+const uint16_t  *
+auth_challenge_cell_getconstarray_methods(const auth_challenge_cell_t *inp)
+{
+  return (const uint16_t  *)auth_challenge_cell_getarray_methods((auth_challenge_cell_t*)inp);
+}
 int
 auth_challenge_cell_setlen_methods(auth_challenge_cell_t *inp, size_t newlen)
 {
@@ -342,7 +362,7 @@ auth_ctx_free(auth_ctx_t *obj)
 }
 
 uint8_t
-auth_ctx_get_is_ed(auth_ctx_t *inp)
+auth_ctx_get_is_ed(const auth_ctx_t *inp)
 {
   return inp->is_ed;
 }
@@ -382,7 +402,7 @@ certs_cell_cert_free(certs_cell_cert_t *obj)
 }
 
 uint8_t
-certs_cell_cert_get_cert_type(certs_cell_cert_t *inp)
+certs_cell_cert_get_cert_type(const certs_cell_cert_t *inp)
 {
   return inp->cert_type;
 }
@@ -393,7 +413,7 @@ certs_cell_cert_set_cert_type(certs_cell_cert_t *inp, uint8_t val)
   return 0;
 }
 uint16_t
-certs_cell_cert_get_cert_len(certs_cell_cert_t *inp)
+certs_cell_cert_get_cert_len(const certs_cell_cert_t *inp)
 {
   return inp->cert_len;
 }
@@ -415,6 +435,11 @@ certs_cell_cert_get_body(certs_cell_cert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->body, idx);
 }
 
+uint8_t
+certs_cell_cert_getconst_body(const certs_cell_cert_t *inp, size_t idx)
+{
+  return certs_cell_cert_get_body((certs_cell_cert_t*)inp, idx);
+}
 int
 certs_cell_cert_set_body(certs_cell_cert_t *inp, size_t idx, uint8_t elt)
 {
@@ -440,6 +465,11 @@ certs_cell_cert_getarray_body(certs_cell_cert_t *inp)
 {
   return inp->body.elts_;
 }
+const uint8_t  *
+certs_cell_cert_getconstarray_body(const certs_cell_cert_t *inp)
+{
+  return (const uint8_t  *)certs_cell_cert_getarray_body((certs_cell_cert_t*)inp);
+}
 int
 certs_cell_cert_setlen_body(certs_cell_cert_t *inp, size_t newlen)
 {
@@ -652,12 +682,17 @@ rsa_ed_crosscert_getlen_ed_key(const rsa_ed_crosscert_t *inp)
 }
 
 uint8_t
-rsa_ed_crosscert_get_ed_key(const rsa_ed_crosscert_t *inp, size_t idx)
+rsa_ed_crosscert_get_ed_key(rsa_ed_crosscert_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->ed_key[idx];
 }
 
+uint8_t
+rsa_ed_crosscert_getconst_ed_key(const rsa_ed_crosscert_t *inp, size_t idx)
+{
+  return rsa_ed_crosscert_get_ed_key((rsa_ed_crosscert_t*)inp, idx);
+}
 int
 rsa_ed_crosscert_set_ed_key(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt)
 {
@@ -671,8 +706,13 @@ rsa_ed_crosscert_getarray_ed_key(rsa_ed_crosscert_t *inp)
 {
   return inp->ed_key;
 }
+const uint8_t  *
+rsa_ed_crosscert_getconstarray_ed_key(const rsa_ed_crosscert_t *inp)
+{
+  return (const uint8_t  *)rsa_ed_crosscert_getarray_ed_key((rsa_ed_crosscert_t*)inp);
+}
 uint32_t
-rsa_ed_crosscert_get_expiration(rsa_ed_crosscert_t *inp)
+rsa_ed_crosscert_get_expiration(const rsa_ed_crosscert_t *inp)
 {
   return inp->expiration;
 }
@@ -688,7 +728,7 @@ rsa_ed_crosscert_get_end_of_signed(const rsa_ed_crosscert_t *inp)
   return inp->end_of_signed;
 }
 uint8_t
-rsa_ed_crosscert_get_sig_len(rsa_ed_crosscert_t *inp)
+rsa_ed_crosscert_get_sig_len(const rsa_ed_crosscert_t *inp)
 {
   return inp->sig_len;
 }
@@ -710,6 +750,11 @@ rsa_ed_crosscert_get_sig(rsa_ed_crosscert_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
 }
 
+uint8_t
+rsa_ed_crosscert_getconst_sig(const rsa_ed_crosscert_t *inp, size_t idx)
+{
+  return rsa_ed_crosscert_get_sig((rsa_ed_crosscert_t*)inp, idx);
+}
 int
 rsa_ed_crosscert_set_sig(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt)
 {
@@ -735,6 +780,11 @@ rsa_ed_crosscert_getarray_sig(rsa_ed_crosscert_t *inp)
 {
   return inp->sig.elts_;
 }
+const uint8_t  *
+rsa_ed_crosscert_getconstarray_sig(const rsa_ed_crosscert_t *inp)
+{
+  return (const uint8_t  *)rsa_ed_crosscert_getarray_sig((rsa_ed_crosscert_t*)inp);
+}
 int
 rsa_ed_crosscert_setlen_sig(rsa_ed_crosscert_t *inp, size_t newlen)
 {
@@ -964,12 +1014,17 @@ auth1_getlen_type(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_type(const auth1_t *inp, size_t idx)
+auth1_get_type(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 8);
   return inp->type[idx];
 }
 
+uint8_t
+auth1_getconst_type(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_type((auth1_t*)inp, idx);
+}
 int
 auth1_set_type(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -983,6 +1038,11 @@ auth1_getarray_type(auth1_t *inp)
 {
   return inp->type;
 }
+const uint8_t  *
+auth1_getconstarray_type(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_type((auth1_t*)inp);
+}
 size_t
 auth1_getlen_cid(const auth1_t *inp)
 {
@@ -990,12 +1050,17 @@ auth1_getlen_cid(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_cid(const auth1_t *inp, size_t idx)
+auth1_get_cid(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->cid[idx];
 }
 
+uint8_t
+auth1_getconst_cid(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_cid((auth1_t*)inp, idx);
+}
 int
 auth1_set_cid(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1009,6 +1074,11 @@ auth1_getarray_cid(auth1_t *inp)
 {
   return inp->cid;
 }
+const uint8_t  *
+auth1_getconstarray_cid(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_cid((auth1_t*)inp);
+}
 size_t
 auth1_getlen_sid(const auth1_t *inp)
 {
@@ -1016,12 +1086,17 @@ auth1_getlen_sid(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_sid(const auth1_t *inp, size_t idx)
+auth1_get_sid(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->sid[idx];
 }
 
+uint8_t
+auth1_getconst_sid(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_sid((auth1_t*)inp, idx);
+}
 int
 auth1_set_sid(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1035,6 +1110,11 @@ auth1_getarray_sid(auth1_t *inp)
 {
   return inp->sid;
 }
+const uint8_t  *
+auth1_getconstarray_sid(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_sid((auth1_t*)inp);
+}
 size_t
 auth1_getlen_u1_cid_ed(const auth1_t *inp)
 {
@@ -1042,12 +1122,17 @@ auth1_getlen_u1_cid_ed(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_u1_cid_ed(const auth1_t *inp, size_t idx)
+auth1_get_u1_cid_ed(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->u1_cid_ed[idx];
 }
 
+uint8_t
+auth1_getconst_u1_cid_ed(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_u1_cid_ed((auth1_t*)inp, idx);
+}
 int
 auth1_set_u1_cid_ed(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1061,6 +1146,11 @@ auth1_getarray_u1_cid_ed(auth1_t *inp)
 {
   return inp->u1_cid_ed;
 }
+const uint8_t  *
+auth1_getconstarray_u1_cid_ed(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_u1_cid_ed((auth1_t*)inp);
+}
 size_t
 auth1_getlen_u1_sid_ed(const auth1_t *inp)
 {
@@ -1068,12 +1158,17 @@ auth1_getlen_u1_sid_ed(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_u1_sid_ed(const auth1_t *inp, size_t idx)
+auth1_get_u1_sid_ed(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->u1_sid_ed[idx];
 }
 
+uint8_t
+auth1_getconst_u1_sid_ed(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_u1_sid_ed((auth1_t*)inp, idx);
+}
 int
 auth1_set_u1_sid_ed(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1087,6 +1182,11 @@ auth1_getarray_u1_sid_ed(auth1_t *inp)
 {
   return inp->u1_sid_ed;
 }
+const uint8_t  *
+auth1_getconstarray_u1_sid_ed(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_u1_sid_ed((auth1_t*)inp);
+}
 size_t
 auth1_getlen_slog(const auth1_t *inp)
 {
@@ -1094,12 +1194,17 @@ auth1_getlen_slog(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_slog(const auth1_t *inp, size_t idx)
+auth1_get_slog(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->slog[idx];
 }
 
+uint8_t
+auth1_getconst_slog(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_slog((auth1_t*)inp, idx);
+}
 int
 auth1_set_slog(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1113,6 +1218,11 @@ auth1_getarray_slog(auth1_t *inp)
 {
   return inp->slog;
 }
+const uint8_t  *
+auth1_getconstarray_slog(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_slog((auth1_t*)inp);
+}
 size_t
 auth1_getlen_clog(const auth1_t *inp)
 {
@@ -1120,12 +1230,17 @@ auth1_getlen_clog(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_clog(const auth1_t *inp, size_t idx)
+auth1_get_clog(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->clog[idx];
 }
 
+uint8_t
+auth1_getconst_clog(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_clog((auth1_t*)inp, idx);
+}
 int
 auth1_set_clog(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1139,6 +1254,11 @@ auth1_getarray_clog(auth1_t *inp)
 {
   return inp->clog;
 }
+const uint8_t  *
+auth1_getconstarray_clog(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_clog((auth1_t*)inp);
+}
 size_t
 auth1_getlen_scert(const auth1_t *inp)
 {
@@ -1146,12 +1266,17 @@ auth1_getlen_scert(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_scert(const auth1_t *inp, size_t idx)
+auth1_get_scert(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->scert[idx];
 }
 
+uint8_t
+auth1_getconst_scert(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_scert((auth1_t*)inp, idx);
+}
 int
 auth1_set_scert(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1165,6 +1290,11 @@ auth1_getarray_scert(auth1_t *inp)
 {
   return inp->scert;
 }
+const uint8_t  *
+auth1_getconstarray_scert(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_scert((auth1_t*)inp);
+}
 size_t
 auth1_getlen_tlssecrets(const auth1_t *inp)
 {
@@ -1172,12 +1302,17 @@ auth1_getlen_tlssecrets(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_tlssecrets(const auth1_t *inp, size_t idx)
+auth1_get_tlssecrets(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->tlssecrets[idx];
 }
 
+uint8_t
+auth1_getconst_tlssecrets(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_tlssecrets((auth1_t*)inp, idx);
+}
 int
 auth1_set_tlssecrets(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1191,6 +1326,11 @@ auth1_getarray_tlssecrets(auth1_t *inp)
 {
   return inp->tlssecrets;
 }
+const uint8_t  *
+auth1_getconstarray_tlssecrets(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_tlssecrets((auth1_t*)inp);
+}
 const uint8_t *
 auth1_get_end_of_fixed_part(const auth1_t *inp)
 {
@@ -1203,12 +1343,17 @@ auth1_getlen_rand(const auth1_t *inp)
 }
 
 uint8_t
-auth1_get_rand(const auth1_t *inp, size_t idx)
+auth1_get_rand(auth1_t *inp, size_t idx)
 {
   trunnel_assert(idx < 24);
   return inp->rand[idx];
 }
 
+uint8_t
+auth1_getconst_rand(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_rand((auth1_t*)inp, idx);
+}
 int
 auth1_set_rand(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1222,6 +1367,11 @@ auth1_getarray_rand(auth1_t *inp)
 {
   return inp->rand;
 }
+const uint8_t  *
+auth1_getconstarray_rand(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_rand((auth1_t*)inp);
+}
 const uint8_t *
 auth1_get_end_of_signed(const auth1_t *inp)
 {
@@ -1239,6 +1389,11 @@ auth1_get_sig(auth1_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->sig, idx);
 }
 
+uint8_t
+auth1_getconst_sig(const auth1_t *inp, size_t idx)
+{
+  return auth1_get_sig((auth1_t*)inp, idx);
+}
 int
 auth1_set_sig(auth1_t *inp, size_t idx, uint8_t elt)
 {
@@ -1260,6 +1415,11 @@ auth1_getarray_sig(auth1_t *inp)
 {
   return inp->sig.elts_;
 }
+const uint8_t  *
+auth1_getconstarray_sig(const auth1_t *inp)
+{
+  return (const uint8_t  *)auth1_getarray_sig((auth1_t*)inp);
+}
 int
 auth1_setlen_sig(auth1_t *inp, size_t newlen)
 {
@@ -1647,7 +1807,7 @@ certs_cell_free(certs_cell_t *obj)
 }
 
 uint8_t
-certs_cell_get_n_certs(certs_cell_t *inp)
+certs_cell_get_n_certs(const certs_cell_t *inp)
 {
   return inp->n_certs;
 }
@@ -1669,6 +1829,11 @@ certs_cell_get_certs(certs_cell_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->certs, idx);
 }
 
+ const struct certs_cell_cert_st *
+certs_cell_getconst_certs(const certs_cell_t *inp, size_t idx)
+{
+  return certs_cell_get_certs((certs_cell_t*)inp, idx);
+}
 int
 certs_cell_set_certs(certs_cell_t *inp, size_t idx, struct certs_cell_cert_st * elt)
 {
@@ -1702,6 +1867,11 @@ certs_cell_getarray_certs(certs_cell_t *inp)
 {
   return inp->certs.elts_;
 }
+const struct certs_cell_cert_st *  const  *
+certs_cell_getconstarray_certs(const certs_cell_t *inp)
+{
+  return (const struct certs_cell_cert_st *  const  *)certs_cell_getarray_certs((certs_cell_t*)inp);
+}
 int
 certs_cell_setlen_certs(certs_cell_t *inp, size_t newlen)
 {
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 54611b96e8..418662631a 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by by Trunnel v1.4.6.
+/* link_handshake.h -- generated by by Trunnel v1.5.1.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -121,7 +121,11 @@ size_t auth_challenge_cell_getlen_challenge(const auth_challenge_cell_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * challenge of the auth_challenge_cell_t in 'inp'.
  */
-uint8_t auth_challenge_cell_get_challenge(const auth_challenge_cell_t *inp, size_t idx);
+uint8_t auth_challenge_cell_get_challenge(auth_challenge_cell_t *inp, size_t idx);
+/** As auth_challenge_cell_get_challenge, but take and return a const
+ * pointer
+ */
+uint8_t auth_challenge_cell_getconst_challenge(const auth_challenge_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * challenge of the auth_challenge_cell_t in 'inp', so that it will
  * hold the value 'elt'.
@@ -130,10 +134,14 @@ int auth_challenge_cell_set_challenge(auth_challenge_cell_t *inp, size_t idx, ui
 /** Return a pointer to the 32-element array field challenge of 'inp'.
  */
 uint8_t * auth_challenge_cell_getarray_challenge(auth_challenge_cell_t *inp);
+/** As auth_challenge_cell_get_challenge, but take and return a const
+ * pointer
+ */
+const uint8_t  * auth_challenge_cell_getconstarray_challenge(const auth_challenge_cell_t *inp);
 /** Return the value of the n_methods field of the
  * auth_challenge_cell_t in 'inp'
  */
-uint16_t auth_challenge_cell_get_n_methods(auth_challenge_cell_t *inp);
+uint16_t auth_challenge_cell_get_n_methods(const auth_challenge_cell_t *inp);
 /** Set the value of the n_methods field of the auth_challenge_cell_t
  * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -147,6 +155,10 @@ size_t auth_challenge_cell_getlen_methods(const auth_challenge_cell_t *inp);
  * methods of the auth_challenge_cell_t in 'inp'.
  */
 uint16_t auth_challenge_cell_get_methods(auth_challenge_cell_t *inp, size_t idx);
+/** As auth_challenge_cell_get_methods, but take and return a const
+ * pointer
+ */
+uint16_t auth_challenge_cell_getconst_methods(const auth_challenge_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * methods of the auth_challenge_cell_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -160,6 +172,10 @@ int auth_challenge_cell_add_methods(auth_challenge_cell_t *inp, uint16_t elt);
  * 'inp'.
  */
 uint16_t * auth_challenge_cell_getarray_methods(auth_challenge_cell_t *inp);
+/** As auth_challenge_cell_get_methods, but take and return a const
+ * pointer
+ */
+const uint16_t  * auth_challenge_cell_getconstarray_methods(const auth_challenge_cell_t *inp);
 /** Change the length of the variable-length array field methods of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
@@ -174,7 +190,7 @@ auth_ctx_t *auth_ctx_new(void);
 void auth_ctx_free(auth_ctx_t *victim);
 /** Return the value of the is_ed field of the auth_ctx_t in 'inp'
  */
-uint8_t auth_ctx_get_is_ed(auth_ctx_t *inp);
+uint8_t auth_ctx_get_is_ed(const auth_ctx_t *inp);
 /** Set the value of the is_ed field of the auth_ctx_t in 'inp' to
  * 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -219,7 +235,7 @@ int certs_cell_cert_clear_errors(certs_cell_cert_t *obj);
 /** Return the value of the cert_type field of the certs_cell_cert_t
  * in 'inp'
  */
-uint8_t certs_cell_cert_get_cert_type(certs_cell_cert_t *inp);
+uint8_t certs_cell_cert_get_cert_type(const certs_cell_cert_t *inp);
 /** Set the value of the cert_type field of the certs_cell_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -228,7 +244,7 @@ int certs_cell_cert_set_cert_type(certs_cell_cert_t *inp, uint8_t val);
 /** Return the value of the cert_len field of the certs_cell_cert_t in
  * 'inp'
  */
-uint16_t certs_cell_cert_get_cert_len(certs_cell_cert_t *inp);
+uint16_t certs_cell_cert_get_cert_len(const certs_cell_cert_t *inp);
 /** Set the value of the cert_len field of the certs_cell_cert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -242,6 +258,9 @@ size_t certs_cell_cert_getlen_body(const certs_cell_cert_t *inp);
  * body of the certs_cell_cert_t in 'inp'.
  */
 uint8_t certs_cell_cert_get_body(certs_cell_cert_t *inp, size_t idx);
+/** As certs_cell_cert_get_body, but take and return a const pointer
+ */
+uint8_t certs_cell_cert_getconst_body(const certs_cell_cert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * body of the certs_cell_cert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -254,6 +273,9 @@ int certs_cell_cert_add_body(certs_cell_cert_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field body of 'inp'.
  */
 uint8_t * certs_cell_cert_getarray_body(certs_cell_cert_t *inp);
+/** As certs_cell_cert_get_body, but take and return a const pointer
+ */
+const uint8_t  * certs_cell_cert_getconstarray_body(const certs_cell_cert_t *inp);
 /** Change the length of the variable-length array field body of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -302,7 +324,11 @@ size_t rsa_ed_crosscert_getlen_ed_key(const rsa_ed_crosscert_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * ed_key of the rsa_ed_crosscert_t in 'inp'.
  */
-uint8_t rsa_ed_crosscert_get_ed_key(const rsa_ed_crosscert_t *inp, size_t idx);
+uint8_t rsa_ed_crosscert_get_ed_key(rsa_ed_crosscert_t *inp, size_t idx);
+/** As rsa_ed_crosscert_get_ed_key, but take and return a const
+ * pointer
+ */
+uint8_t rsa_ed_crosscert_getconst_ed_key(const rsa_ed_crosscert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * ed_key of the rsa_ed_crosscert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -311,10 +337,14 @@ int rsa_ed_crosscert_set_ed_key(rsa_ed_crosscert_t *inp, size_t idx, uint8_t elt
 /** Return a pointer to the 32-element array field ed_key of 'inp'.
  */
 uint8_t * rsa_ed_crosscert_getarray_ed_key(rsa_ed_crosscert_t *inp);
+/** As rsa_ed_crosscert_get_ed_key, but take and return a const
+ * pointer
+ */
+const uint8_t  * rsa_ed_crosscert_getconstarray_ed_key(const rsa_ed_crosscert_t *inp);
 /** Return the value of the expiration field of the rsa_ed_crosscert_t
  * in 'inp'
  */
-uint32_t rsa_ed_crosscert_get_expiration(rsa_ed_crosscert_t *inp);
+uint32_t rsa_ed_crosscert_get_expiration(const rsa_ed_crosscert_t *inp);
 /** Set the value of the expiration field of the rsa_ed_crosscert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -326,7 +356,7 @@ const uint8_t * rsa_ed_crosscert_get_end_of_signed(const rsa_ed_crosscert_t *inp
 /** Return the value of the sig_len field of the rsa_ed_crosscert_t in
  * 'inp'
  */
-uint8_t rsa_ed_crosscert_get_sig_len(rsa_ed_crosscert_t *inp);
+uint8_t rsa_ed_crosscert_get_sig_len(const rsa_ed_crosscert_t *inp);
 /** Set the value of the sig_len field of the rsa_ed_crosscert_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -340,6 +370,9 @@ size_t rsa_ed_crosscert_getlen_sig(const rsa_ed_crosscert_t *inp);
  * sig of the rsa_ed_crosscert_t in 'inp'.
  */
 uint8_t rsa_ed_crosscert_get_sig(rsa_ed_crosscert_t *inp, size_t idx);
+/** As rsa_ed_crosscert_get_sig, but take and return a const pointer
+ */
+uint8_t rsa_ed_crosscert_getconst_sig(const rsa_ed_crosscert_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * sig of the rsa_ed_crosscert_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -352,6 +385,9 @@ int rsa_ed_crosscert_add_sig(rsa_ed_crosscert_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field sig of 'inp'.
  */
 uint8_t * rsa_ed_crosscert_getarray_sig(rsa_ed_crosscert_t *inp);
+/** As rsa_ed_crosscert_get_sig, but take and return a const pointer
+ */
+const uint8_t  * rsa_ed_crosscert_getconstarray_sig(const rsa_ed_crosscert_t *inp);
 /** Change the length of the variable-length array field sig of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -398,7 +434,10 @@ size_t auth1_getlen_type(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field type
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_type(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_type(auth1_t *inp, size_t idx);
+/** As auth1_get_type, but take and return a const pointer
+ */
+uint8_t auth1_getconst_type(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field type
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -406,6 +445,9 @@ int auth1_set_type(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 8-element array field type of 'inp'.
  */
 uint8_t * auth1_getarray_type(auth1_t *inp);
+/** As auth1_get_type, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_type(const auth1_t *inp);
 /** Return the (constant) length of the array holding the cid field of
  * the auth1_t in 'inp'.
  */
@@ -413,7 +455,10 @@ size_t auth1_getlen_cid(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field cid
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_cid(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_cid(auth1_t *inp, size_t idx);
+/** As auth1_get_cid, but take and return a const pointer
+ */
+uint8_t auth1_getconst_cid(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field cid
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -421,6 +466,9 @@ int auth1_set_cid(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field cid of 'inp'.
  */
 uint8_t * auth1_getarray_cid(auth1_t *inp);
+/** As auth1_get_cid, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_cid(const auth1_t *inp);
 /** Return the (constant) length of the array holding the sid field of
  * the auth1_t in 'inp'.
  */
@@ -428,7 +476,10 @@ size_t auth1_getlen_sid(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field sid
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_sid(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_sid(auth1_t *inp, size_t idx);
+/** As auth1_get_sid, but take and return a const pointer
+ */
+uint8_t auth1_getconst_sid(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field sid
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -436,6 +487,9 @@ int auth1_set_sid(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field sid of 'inp'.
  */
 uint8_t * auth1_getarray_sid(auth1_t *inp);
+/** As auth1_get_sid, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_sid(const auth1_t *inp);
 /** Return the (constant) length of the array holding the u1_cid_ed
  * field of the auth1_t in 'inp'.
  */
@@ -443,7 +497,10 @@ size_t auth1_getlen_u1_cid_ed(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * u1_cid_ed of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_u1_cid_ed(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_u1_cid_ed(auth1_t *inp, size_t idx);
+/** As auth1_get_u1_cid_ed, but take and return a const pointer
+ */
+uint8_t auth1_getconst_u1_cid_ed(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * u1_cid_ed of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -452,6 +509,9 @@ int auth1_set_u1_cid_ed(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field u1_cid_ed of 'inp'.
  */
 uint8_t * auth1_getarray_u1_cid_ed(auth1_t *inp);
+/** As auth1_get_u1_cid_ed, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_u1_cid_ed(const auth1_t *inp);
 /** Return the (constant) length of the array holding the u1_sid_ed
  * field of the auth1_t in 'inp'.
  */
@@ -459,7 +519,10 @@ size_t auth1_getlen_u1_sid_ed(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * u1_sid_ed of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_u1_sid_ed(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_u1_sid_ed(auth1_t *inp, size_t idx);
+/** As auth1_get_u1_sid_ed, but take and return a const pointer
+ */
+uint8_t auth1_getconst_u1_sid_ed(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * u1_sid_ed of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -468,6 +531,9 @@ int auth1_set_u1_sid_ed(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field u1_sid_ed of 'inp'.
  */
 uint8_t * auth1_getarray_u1_sid_ed(auth1_t *inp);
+/** As auth1_get_u1_sid_ed, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_u1_sid_ed(const auth1_t *inp);
 /** Return the (constant) length of the array holding the slog field
  * of the auth1_t in 'inp'.
  */
@@ -475,7 +541,10 @@ size_t auth1_getlen_slog(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field slog
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_slog(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_slog(auth1_t *inp, size_t idx);
+/** As auth1_get_slog, but take and return a const pointer
+ */
+uint8_t auth1_getconst_slog(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field slog
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -483,6 +552,9 @@ int auth1_set_slog(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field slog of 'inp'.
  */
 uint8_t * auth1_getarray_slog(auth1_t *inp);
+/** As auth1_get_slog, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_slog(const auth1_t *inp);
 /** Return the (constant) length of the array holding the clog field
  * of the auth1_t in 'inp'.
  */
@@ -490,7 +562,10 @@ size_t auth1_getlen_clog(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field clog
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_clog(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_clog(auth1_t *inp, size_t idx);
+/** As auth1_get_clog, but take and return a const pointer
+ */
+uint8_t auth1_getconst_clog(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field clog
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -498,6 +573,9 @@ int auth1_set_clog(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field clog of 'inp'.
  */
 uint8_t * auth1_getarray_clog(auth1_t *inp);
+/** As auth1_get_clog, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_clog(const auth1_t *inp);
 /** Return the (constant) length of the array holding the scert field
  * of the auth1_t in 'inp'.
  */
@@ -505,7 +583,10 @@ size_t auth1_getlen_scert(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * scert of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_scert(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_scert(auth1_t *inp, size_t idx);
+/** As auth1_get_scert, but take and return a const pointer
+ */
+uint8_t auth1_getconst_scert(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * scert of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -514,6 +595,9 @@ int auth1_set_scert(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field scert of 'inp'.
  */
 uint8_t * auth1_getarray_scert(auth1_t *inp);
+/** As auth1_get_scert, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_scert(const auth1_t *inp);
 /** Return the (constant) length of the array holding the tlssecrets
  * field of the auth1_t in 'inp'.
  */
@@ -521,7 +605,10 @@ size_t auth1_getlen_tlssecrets(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field
  * tlssecrets of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_tlssecrets(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_tlssecrets(auth1_t *inp, size_t idx);
+/** As auth1_get_tlssecrets, but take and return a const pointer
+ */
+uint8_t auth1_getconst_tlssecrets(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field
  * tlssecrets of the auth1_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -531,6 +618,9 @@ int auth1_set_tlssecrets(auth1_t *inp, size_t idx, uint8_t elt);
  * 'inp'.
  */
 uint8_t * auth1_getarray_tlssecrets(auth1_t *inp);
+/** As auth1_get_tlssecrets, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_tlssecrets(const auth1_t *inp);
 /** Return the position for end_of_fixed_part when we parsed this
  * object
  */
@@ -542,7 +632,10 @@ size_t auth1_getlen_rand(const auth1_t *inp);
 /** Return the element at position 'idx' of the fixed array field rand
  * of the auth1_t in 'inp'.
  */
-uint8_t auth1_get_rand(const auth1_t *inp, size_t idx);
+uint8_t auth1_get_rand(auth1_t *inp, size_t idx);
+/** As auth1_get_rand, but take and return a const pointer
+ */
+uint8_t auth1_getconst_rand(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field rand
  * of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -550,6 +643,9 @@ int auth1_set_rand(auth1_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 24-element array field rand of 'inp'.
  */
 uint8_t * auth1_getarray_rand(auth1_t *inp);
+/** As auth1_get_rand, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_rand(const auth1_t *inp);
 /** Return the position for end_of_signed when we parsed this object
  */
 const uint8_t * auth1_get_end_of_signed(const auth1_t *inp);
@@ -561,6 +657,9 @@ size_t auth1_getlen_sig(const auth1_t *inp);
  * sig of the auth1_t in 'inp'.
  */
 uint8_t auth1_get_sig(auth1_t *inp, size_t idx);
+/** As auth1_get_sig, but take and return a const pointer
+ */
+uint8_t auth1_getconst_sig(const auth1_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * sig of the auth1_t in 'inp', so that it will hold the value 'elt'.
  */
@@ -572,6 +671,9 @@ int auth1_add_sig(auth1_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field sig of 'inp'.
  */
 uint8_t * auth1_getarray_sig(auth1_t *inp);
+/** As auth1_get_sig, but take and return a const pointer
+ */
+const uint8_t  * auth1_getconstarray_sig(const auth1_t *inp);
 /** Change the length of the variable-length array field sig of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -613,7 +715,7 @@ const char *certs_cell_check(const certs_cell_t *obj);
 int certs_cell_clear_errors(certs_cell_t *obj);
 /** Return the value of the n_certs field of the certs_cell_t in 'inp'
  */
-uint8_t certs_cell_get_n_certs(certs_cell_t *inp);
+uint8_t certs_cell_get_n_certs(const certs_cell_t *inp);
 /** Set the value of the n_certs field of the certs_cell_t in 'inp' to
  * 'val'. Return 0 on success; return -1 and set the error code on
  * 'inp' on failure.
@@ -627,6 +729,9 @@ size_t certs_cell_getlen_certs(const certs_cell_t *inp);
  * certs of the certs_cell_t in 'inp'.
  */
 struct certs_cell_cert_st * certs_cell_get_certs(certs_cell_t *inp, size_t idx);
+/** As certs_cell_get_certs, but take and return a const pointer
+ */
+ const struct certs_cell_cert_st * certs_cell_getconst_certs(const certs_cell_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * certs of the certs_cell_t in 'inp', so that it will hold the value
  * 'elt'. Free the previous value, if any.
@@ -643,6 +748,9 @@ int certs_cell_add_certs(certs_cell_t *inp, struct certs_cell_cert_st * elt);
  * 'inp'.
  */
 struct certs_cell_cert_st * * certs_cell_getarray_certs(certs_cell_t *inp);
+/** As certs_cell_get_certs, but take and return a const pointer
+ */
+const struct certs_cell_cert_st *  const  * certs_cell_getconstarray_certs(const certs_cell_t *inp);
 /** Change the length of the variable-length array field certs of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index 62662a9369..f4b910bdab 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.4.6.
+/* pwbox.c -- generated by Trunnel v1.5.1.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -62,7 +62,7 @@ pwbox_encoded_free(pwbox_encoded_t *obj)
 }
 
 uint32_t
-pwbox_encoded_get_fixedbytes0(pwbox_encoded_t *inp)
+pwbox_encoded_get_fixedbytes0(const pwbox_encoded_t *inp)
 {
   return inp->fixedbytes0;
 }
@@ -77,7 +77,7 @@ pwbox_encoded_set_fixedbytes0(pwbox_encoded_t *inp, uint32_t val)
   return 0;
 }
 uint32_t
-pwbox_encoded_get_fixedbytes1(pwbox_encoded_t *inp)
+pwbox_encoded_get_fixedbytes1(const pwbox_encoded_t *inp)
 {
   return inp->fixedbytes1;
 }
@@ -92,7 +92,7 @@ pwbox_encoded_set_fixedbytes1(pwbox_encoded_t *inp, uint32_t val)
   return 0;
 }
 uint8_t
-pwbox_encoded_get_header_len(pwbox_encoded_t *inp)
+pwbox_encoded_get_header_len(const pwbox_encoded_t *inp)
 {
   return inp->header_len;
 }
@@ -114,6 +114,11 @@ pwbox_encoded_get_skey_header(pwbox_encoded_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->skey_header, idx);
 }
 
+uint8_t
+pwbox_encoded_getconst_skey_header(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_skey_header((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_skey_header(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -139,6 +144,11 @@ pwbox_encoded_getarray_skey_header(pwbox_encoded_t *inp)
 {
   return inp->skey_header.elts_;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_skey_header(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_skey_header((pwbox_encoded_t*)inp);
+}
 int
 pwbox_encoded_setlen_skey_header(pwbox_encoded_t *inp, size_t newlen)
 {
@@ -166,12 +176,17 @@ pwbox_encoded_getlen_iv(const pwbox_encoded_t *inp)
 }
 
 uint8_t
-pwbox_encoded_get_iv(const pwbox_encoded_t *inp, size_t idx)
+pwbox_encoded_get_iv(pwbox_encoded_t *inp, size_t idx)
 {
   trunnel_assert(idx < 16);
   return inp->iv[idx];
 }
 
+uint8_t
+pwbox_encoded_getconst_iv(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_iv((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_iv(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -185,6 +200,11 @@ pwbox_encoded_getarray_iv(pwbox_encoded_t *inp)
 {
   return inp->iv;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_iv(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_iv((pwbox_encoded_t*)inp);
+}
 size_t
 pwbox_encoded_getlen_data(const pwbox_encoded_t *inp)
 {
@@ -197,6 +217,11 @@ pwbox_encoded_get_data(pwbox_encoded_t *inp, size_t idx)
   return TRUNNEL_DYNARRAY_GET(&inp->data, idx);
 }
 
+uint8_t
+pwbox_encoded_getconst_data(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_data((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_data(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -218,6 +243,11 @@ pwbox_encoded_getarray_data(pwbox_encoded_t *inp)
 {
   return inp->data.elts_;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_data(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_data((pwbox_encoded_t*)inp);
+}
 int
 pwbox_encoded_setlen_data(pwbox_encoded_t *inp, size_t newlen)
 {
@@ -241,12 +271,17 @@ pwbox_encoded_getlen_hmac(const pwbox_encoded_t *inp)
 }
 
 uint8_t
-pwbox_encoded_get_hmac(const pwbox_encoded_t *inp, size_t idx)
+pwbox_encoded_get_hmac(pwbox_encoded_t *inp, size_t idx)
 {
   trunnel_assert(idx < 32);
   return inp->hmac[idx];
 }
 
+uint8_t
+pwbox_encoded_getconst_hmac(const pwbox_encoded_t *inp, size_t idx)
+{
+  return pwbox_encoded_get_hmac((pwbox_encoded_t*)inp, idx);
+}
 int
 pwbox_encoded_set_hmac(pwbox_encoded_t *inp, size_t idx, uint8_t elt)
 {
@@ -260,6 +295,11 @@ pwbox_encoded_getarray_hmac(pwbox_encoded_t *inp)
 {
   return inp->hmac;
 }
+const uint8_t  *
+pwbox_encoded_getconstarray_hmac(const pwbox_encoded_t *inp)
+{
+  return (const uint8_t  *)pwbox_encoded_getarray_hmac((pwbox_encoded_t*)inp);
+}
 const char *
 pwbox_encoded_check(const pwbox_encoded_t *obj)
 {
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index 77a813d123..939b3c41a7 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by by Trunnel v1.4.6.
+/* pwbox.h -- generated by by Trunnel v1.5.1.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -61,7 +61,7 @@ int pwbox_encoded_clear_errors(pwbox_encoded_t *obj);
 /** Return the value of the fixedbytes0 field of the pwbox_encoded_t
  * in 'inp'
  */
-uint32_t pwbox_encoded_get_fixedbytes0(pwbox_encoded_t *inp);
+uint32_t pwbox_encoded_get_fixedbytes0(const pwbox_encoded_t *inp);
 /** Set the value of the fixedbytes0 field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -70,7 +70,7 @@ int pwbox_encoded_set_fixedbytes0(pwbox_encoded_t *inp, uint32_t val);
 /** Return the value of the fixedbytes1 field of the pwbox_encoded_t
  * in 'inp'
  */
-uint32_t pwbox_encoded_get_fixedbytes1(pwbox_encoded_t *inp);
+uint32_t pwbox_encoded_get_fixedbytes1(const pwbox_encoded_t *inp);
 /** Set the value of the fixedbytes1 field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -79,7 +79,7 @@ int pwbox_encoded_set_fixedbytes1(pwbox_encoded_t *inp, uint32_t val);
 /** Return the value of the header_len field of the pwbox_encoded_t in
  * 'inp'
  */
-uint8_t pwbox_encoded_get_header_len(pwbox_encoded_t *inp);
+uint8_t pwbox_encoded_get_header_len(const pwbox_encoded_t *inp);
 /** Set the value of the header_len field of the pwbox_encoded_t in
  * 'inp' to 'val'. Return 0 on success; return -1 and set the error
  * code on 'inp' on failure.
@@ -93,6 +93,10 @@ size_t pwbox_encoded_getlen_skey_header(const pwbox_encoded_t *inp);
  * skey_header of the pwbox_encoded_t in 'inp'.
  */
 uint8_t pwbox_encoded_get_skey_header(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_skey_header, but take and return a const
+ * pointer
+ */
+uint8_t pwbox_encoded_getconst_skey_header(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * skey_header of the pwbox_encoded_t in 'inp', so that it will hold
  * the value 'elt'.
@@ -106,6 +110,10 @@ int pwbox_encoded_add_skey_header(pwbox_encoded_t *inp, uint8_t elt);
  * 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_skey_header(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_skey_header, but take and return a const
+ * pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_skey_header(const pwbox_encoded_t *inp);
 /** Change the length of the variable-length array field skey_header
  * of 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on
  * success; return -1 and set the error code on 'inp' on failure.
@@ -118,7 +126,10 @@ size_t pwbox_encoded_getlen_iv(const pwbox_encoded_t *inp);
 /** Return the element at position 'idx' of the fixed array field iv
  * of the pwbox_encoded_t in 'inp'.
  */
-uint8_t pwbox_encoded_get_iv(const pwbox_encoded_t *inp, size_t idx);
+uint8_t pwbox_encoded_get_iv(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_iv, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_iv(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field iv
  * of the pwbox_encoded_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -127,6 +138,9 @@ int pwbox_encoded_set_iv(pwbox_encoded_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 16-element array field iv of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_iv(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_iv, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_iv(const pwbox_encoded_t *inp);
 /** Return the length of the dynamic array holding the data field of
  * the pwbox_encoded_t in 'inp'.
  */
@@ -135,6 +149,9 @@ size_t pwbox_encoded_getlen_data(const pwbox_encoded_t *inp);
  * data of the pwbox_encoded_t in 'inp'.
  */
 uint8_t pwbox_encoded_get_data(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_data, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_data(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * data of the pwbox_encoded_t in 'inp', so that it will hold the
  * value 'elt'.
@@ -147,6 +164,9 @@ int pwbox_encoded_add_data(pwbox_encoded_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field data of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_data(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_data, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_data(const pwbox_encoded_t *inp);
 /** Change the length of the variable-length array field data of 'inp'
  * to 'newlen'.Fill extra elements with 0. Return 0 on success; return
  * -1 and set the error code on 'inp' on failure.
@@ -159,7 +179,10 @@ size_t pwbox_encoded_getlen_hmac(const pwbox_encoded_t *inp);
 /** Return the element at position 'idx' of the fixed array field hmac
  * of the pwbox_encoded_t in 'inp'.
  */
-uint8_t pwbox_encoded_get_hmac(const pwbox_encoded_t *inp, size_t idx);
+uint8_t pwbox_encoded_get_hmac(pwbox_encoded_t *inp, size_t idx);
+/** As pwbox_encoded_get_hmac, but take and return a const pointer
+ */
+uint8_t pwbox_encoded_getconst_hmac(const pwbox_encoded_t *inp, size_t idx);
 /** Change the element at position 'idx' of the fixed array field hmac
  * of the pwbox_encoded_t in 'inp', so that it will hold the value
  * 'elt'.
@@ -168,6 +191,9 @@ int pwbox_encoded_set_hmac(pwbox_encoded_t *inp, size_t idx, uint8_t elt);
 /** Return a pointer to the 32-element array field hmac of 'inp'.
  */
 uint8_t * pwbox_encoded_getarray_hmac(pwbox_encoded_t *inp);
+/** As pwbox_encoded_get_hmac, but take and return a const pointer
+ */
+const uint8_t  * pwbox_encoded_getconstarray_hmac(const pwbox_encoded_t *inp);
 
 
 #endif
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index badfab7877..271b018c68 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -218,7 +218,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.2.9.15-dev"
+#define VERSION "0.3.1.10-dev"