diff options
109 files changed, 5109 insertions, 342 deletions
@@ -1,3 +1,419 @@ +Changes in version 0.4.6.6 - 2021-06-30 + Tor 0.4.6.6 makes several small fixes on 0.4.6.5, including one that + allows Tor to build correctly on older versions of GCC. You should + upgrade to this version if you were having trouble building Tor + 0.4.6.5; otherwise, there is probably no need. + + o Minor bugfixes (compilation): + - Fix a compilation error when trying to build Tor with a compiler + that does not support const variables in static initializers. + Fixes bug 40410; bugfix on 0.4.6.5. + - Suppress a strict-prototype warning when building with some + versions of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Enable the deterministic RNG for unit tests that covers the + address set bloomfilter-based API's. Fixes bug 40419; bugfix + on 0.3.3.2-alpha. + + +Changes in version 0.4.5.9 - 2021-06-14 + Tor 0.4.5.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (control, sandbox, backport from 0.4.6.4-rc): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port, backport from 0.4.6.4-rc): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + +Changes in version 0.4.4.9 - 2021-06-14 + Tor 0.4.4.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Note that the scheduled end-of-life date for the Tor 0.4.4.x series is + June 15. This is therefore the last release in its series. Everybody + still running 0.4.4.x should plan to upgrade to 0.4.5.x or later. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.15 - 2021-06-14 + Tor 0.3.5.15 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + +Changes in version 0.4.6.5 - 2021-06-14 + Tor 0.4.6.5 is the first stable release in its series. The 0.4.6.x + series includes numerous features and bugfixes, including a significant + improvement to our circuit timeout algorithm that should improve + observed client performance, and a way for relays to report when they are + overloaded. + + This release also includes security fixes for several security issues, + including a denial-of-service attack against onion service clients, + and another denial-of-service attack against relays. Everybody should + upgrade to one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Below are the changes since 0.4.6.4-rc. For a complete list of changes + since 0.4.5.8, see the ReleaseNotes file. + + o Major bugfixes (security): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor features (logging, diagnostic): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + +Changes in version 0.4.6.4-rc - 2021-05-28 + Tor 0.4.6.4-rc fixes a few bugs from previous releases. This, we hope, + the final release candidate in its series: unless major new issues are + found, the next release will be stable. + + o Minor features (compatibility): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor bugfixes (consensus handling): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (control, sandbox): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + o Removed features: + - Remove unneeded code for parsing private keys in directory + documents. This code was only used for client authentication in v2 + onion services, which are now unsupported. Closes ticket 40374. + + +Changes in version 0.4.5.8 - 2021-05-10 + Tor 0.4.5.8 fixes several bugs in earlier version, backporting fixes + from the 0.4.6.x series. + + o Minor features (compatibility, Linux seccomp sandbox, backport from 0.4.6.3-rc): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation, backport from 0.4.6.3-rc): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add warning message when connecting to now deprecated v2 onion + services. As announced, Tor 0.4.5.x is the last series that will + support v2 onions. Closes ticket 40373. + + o Minor bugfixes (bridge, pluggable transport, backport from 0.4.6.2-alpha): + - Fix a regression that made it impossible start Tor using a bridge + line with a transport name and no fingerprint. Fixes bug 40360; + bugfix on 0.4.5.4-rc. + + o Minor bugfixes (build, cross-compilation, backport from 0.4.6.3-rc): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (controller, backport from 0.4.6.1-alpha): + - Fix a "BUG" warning that would appear when a controller chooses + the first hop for a circuit, and that circuit completes. Fixes bug + 40285; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, client, memory leak, backport from 0.4.6.3-rc): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD, backport from 0.4.6.2-alpha): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + +Changes in version 0.4.6.3-rc - 2021-05-10 + Tor 0.4.6.3-rc is the first release candidate in its series. It fixes + a few small bugs from previous versions, and adds a better error + message when trying to use (no longer supported) v2 onion services. + + Though we anticipate that we'll be doing a bit more clean-up between + now and the stable release, we expect that our remaining changes will + be fairly simple. There will likely be at least one more release + candidate before 0.4.6.x is stable. + + o Major bugfixes (onion service, control port): + - Make the ADD_ONION command properly configure client authorization. + Before this fix, the created onion failed to add the client(s). + Fixes bug 40378; bugfix on 0.4.6.1-alpha. + + o Minor features (compatibility, Linux seccomp sandbox): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add a warning message when trying to connect to (no longer + supported) v2 onion services. Closes ticket 40373. + + o Minor bugfixes (build, cross-compilation): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (compiler warnings): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (logging, relay): + - Emit a warning if an Address is found to be internal and tor can't + use it. Fixes bug 40290; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (onion service, client, memory leak): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + Changes in version 0.4.6.2-alpha - 2021-04-15 Tor 0.4.6.2-alpha is the second alpha in its series. It fixes several small bugs in previous releases, and solves other issues that had diff --git a/Makefile.am b/Makefile.am index 3056b08446..7ae2133767 100644 --- a/Makefile.am +++ b/Makefile.am @@ -253,7 +253,7 @@ endif TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings if LIBFUZZER_ENABLED -TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div +TEST_CFLAGS += -fsanitize=fuzzer-no-link # not "edge" endif diff --git a/ReleaseNotes b/ReleaseNotes index 42017292c5..e4c42feefb 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,585 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.4.6.6 - 2021-06-30 + Tor 0.4.6.6 makes several small fixes on 0.4.6.5, including one that + allows Tor to build correctly on older versions of GCC. You should + upgrade to this version if you were having trouble building Tor + 0.4.6.5; otherwise, there is probably no need. + + o Minor bugfixes (compilation): + - Fix a compilation error when trying to build Tor with a compiler + that does not support const variables in static initializers. + Fixes bug 40410; bugfix on 0.4.6.5. + - Suppress a strict-prototype warning when building with some + versions of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Enable the deterministic RNG for unit tests that covers the + address set bloomfilter-based API's. Fixes bug 40419; bugfix + on 0.3.3.2-alpha. + + +Changes in version 0.4.5.9 - 2021-06-14 + Tor 0.4.5.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (control, sandbox, backport from 0.4.6.4-rc): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port, backport from 0.4.6.4-rc): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + +Changes in version 0.4.4.9 - 2021-06-14 + Tor 0.4.4.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Note that the scheduled end-of-life date for the Tor 0.4.4.x series is + June 15. This is therefore the last release in its series. Everybody + still running 0.4.4.x should plan to upgrade to 0.4.5.x or later. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.15 - 2021-06-14 + Tor 0.3.5.15 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + +Changes in version 0.4.6.5 - 2021-06-14 + Tor 0.4.6.5 is the first stable release in its series. The 0.4.6.x + series includes numerous features and bugfixes, including a significant + improvement to our circuit timeout algorithm that should improve + observed client performance, and a way for relays to report when they are + overloaded. + + This release also includes security fixes for several security issues, + including a denial-of-service attack against onion service clients, + and another denial-of-service attack against relays. Everybody should + upgrade to one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Below are the changes since 0.4.5.8. For a list of changes since + 0.4.6.4-rc, see the ChangeLog file. + + o Major bugfixes (security): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Major features (control port, onion services): + - Add controller support for creating version 3 onion services with + client authorization. Previously, only v2 onion services could be + created with client authorization. Closes ticket 40084. Patch by + Neel Chauhan. + + o Major features (directory authority): + - When voting on a relay with a Sybil-like appearance, add the Sybil + flag when clearing out the other flags. This lets a relay operator + know why their relay hasn't been included in the consensus. Closes + ticket 40255. Patch by Neel Chauhan. + + o Major features (metrics): + - Relays now report how overloaded they are in their extrainfo + documents. This information is controlled with the + OverloadStatistics torrc option, and it will be used to improve + decisions about the network's load balancing. Implements proposal + 328; closes ticket 40222. + + o Major features (relay, denial of service): + - Add a new DoS subsystem feature to control the rate of client + connections for relays. Closes ticket 40253. + + o Major features (statistics): + - Relays now publish statistics about the number of v3 onion + services and volume of v3 onion service traffic, in the same + manner they already do for v2 onions. Closes ticket 23126. + + o Major bugfixes (circuit build timeout): + - Improve the accuracy of our circuit build timeout calculation for + 60%, 70%, and 80% build rates for various guard choices. We now + use a maximum likelihood estimator for Pareto parameters of the + circuit build time distribution, instead of a "right-censored + estimator". This causes clients to ignore circuits that never + finish building in their timeout calculations. Previously, clients + were counting such unfinished circuits as having the highest + possible build time value, when in reality these circuits most + likely just contain relays that are offline. We also now wait a + bit longer to let circuits complete for measurement purposes, + lower the minimum possible effective timeout from 1.5 seconds to + 10ms, and increase the resolution of the circuit build time + histogram from 50ms bin widths to 10ms bin widths. Additionally, + we alter our estimate Xm by taking the maximum of the top 10 most + common build time values of the 10ms histogram, and compute Xm as + the average of these. Fixes bug 40168; bugfix on 0.2.2.14-alpha. + - Remove max_time calculation and associated warning from circuit + build timeout 'alpha' parameter estimation, as this is no longer + needed by our new estimator from 40168. Fixes bug 34088; bugfix + on 0.2.2.9-alpha. + + o Major bugfixes (signing key): + - In the tor-gencert utility, give an informative error message if + the passphrase given in `--create-identity-key` is too short. + Fixes bug 40189; bugfix on 0.2.0.1-alpha. Patch by Neel Chauhan. + + o Minor features (bridge): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (build system): + - New "make lsp" command to auto generate the compile_commands.json + file used by the ccls server. The "bear" program is needed for + this. Closes ticket 40227. + + o Minor features (client): + - Clients now check whether their streams are attempting to re-enter + the Tor network (i.e. to send Tor traffic over Tor), and close + them preemptively if they think exit relays will refuse them for + this reason. See ticket 2667 for details. Closes ticket 40271. + + o Minor features (command line): + - Add long format name "--torrc-file" equivalent to the existing + command-line option "-f". Closes ticket 40324. Patch by + Daniel Pinto. + + o Minor features (command-line interface): + - Add build informations to `tor --version` in order to ease + reproducible builds. Closes ticket 32102. + - When parsing command-line flags that take an optional argument, + treat the argument as absent if it would start with a '-' + character. Arguments in that form are not intelligible for any of + our optional-argument flags. Closes ticket 40223. + - Allow a relay operator to list the ed25519 keys on the command + line by adding the `rsa` and `ed25519` arguments to the + --list-fingerprint flag to show the respective RSA and ed25519 + relay fingerprint. Closes ticket 33632. Patch by Neel Chauhan. + + o Minor features (compatibility): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (control port, stream handling): + - Add the stream ID to the event line in the ADDRMAP control event. + Closes ticket 40249. Patch by Neel Chauhan. + + o Minor features (dormant mode): + - Add a new 'DormantTimeoutEnabled' option to allow coarse-grained + control over whether the client ever becomes dormant from + inactivity. Most people won't need this. Closes ticket 40228. + - Add a new 'DormantTimeoutEnabled' option for coarse-grained + control over whether the client can become dormant from + inactivity. Most people won't need this. Closes ticket 40228. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor features (logging): + - Edit heartbeat log messages so that more of them begin with the + string "Heartbeat: ". Closes ticket 40322; patch + from 'cypherpunks'. + - Change the DoS subsystem heartbeat line format to be more clear on + what has been detected/rejected, and which option is disabled (if + any). Closes ticket 40308. + - In src/core/mainloop/mainloop.c and src/core/mainloop/connection.c, + put brackets around IPv6 addresses in log messages. Closes ticket + 40232. Patch by Neel Chauhan. + + o Minor features (logging, diagnostic): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + o Minor features (onion services): + - Add a warning message when trying to connect to (no longer + supported) v2 onion services. Closes ticket 40373. + + o Minor features (performance, windows): + - Use SRWLocks to implement locking on Windows. Replaces the + "critical section" locking implementation with the faster + SRWLocks, available since Windows Vista. Closes ticket 17927. + Patch by Daniel Pinto. + + o Minor features (protocol, proxy support, defense in depth): + - Close HAProxy connections if they somehow manage to send us data + before we start reading. Closes another case of ticket 40017. + + o Minor features (tests, portability): + - Port the hs_build_address.py test script to work with recent + versions of python. Closes ticket 40213. Patch from + Samanta Navarro. + + o Minor features (vote document): + - Add a "stats" line to directory authority votes, to report various + statistics that authorities compute about the relays. This will + help us diagnose the network better. Closes ticket 40314. + + o Minor bugfixes (build): + - The configure script now shows whether or not lzma and zstd have + been used, not just if the enable flag was passed in. Fixes bug + 40236; bugfix on 0.4.3.1-alpha. + + o Minor bugfixes (compatibility): + - Fix a failure in the test cases when running on the "hppa" + architecture, along with a related test that might fail on other + architectures in the future. Fixes bug 40274; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (compilation): + - Fix a compilation warning about unused functions when building + with a libc that lacks the GLOB_ALTDIRFUNC constant. Fixes bug + 40354; bugfix on 0.4.5.1-alpha. Patch by Daniel Pinto. + + o Minor bugfixes (consensus handling): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (control, sandbox): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (directory authorities, voting): + - Add a new consensus method (31) to support any future changes that + authorities decide to make to the value of bwweightscale or + maxunmeasuredbw. Previously, there was a bug that prevented the + authorities from parsing these consensus parameters correctly under + most circumstances. Fixes bug 19011; bugfix on 0.2.2.10-alpha. + + o Minor bugfixes (ipv6): + - Allow non-SOCKSPorts to disable IPv4, IPv6, and PreferIPv4. Some + rare configurations might break, but in this case you can disable + NoIPv4Traffic and NoIPv6Traffic as needed. Fixes bug 33607; bugfix + on 0.4.1.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (key generation): + - Do not require a valid torrc when using the `--keygen` argument to + generate a signing key. This allows us to generate keys on systems + or users which may not run Tor. Fixes bug 40235; bugfix on + 0.2.7.2-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (logging, relay): + - Emit a warning if an Address is found to be internal and tor can't + use it. Fixes bug 40290; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (metrics port): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + o Minor bugfixes (onion services, logging): + - Downgrade the severity of a few rendezvous circuit-related + warnings from warning to info. Fixes bug 40207; bugfix on + 0.3.2.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. + This should reduce the CPU and memory burden for directory caches. + Fixes bug 40301; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + o Code simplification and refactoring: + - Remove the orconn_ext_or_id_map structure and related functions. + (Nothing outside of unit tests used them.) Closes ticket 33383. + Patch by Neel Chauhan. + + o Removed features: + - Remove unneeded code for parsing private keys in directory + documents. This code was only used for client authentication in v2 + onion services, which are now unsupported. Closes ticket 40374. + - As of this release, Tor no longer supports the old v2 onion + services. They were deprecated last July for security, and support + will be removed entirely later this year. We strongly encourage + everybody to migrate to v3 onion services. For more information, + see https://blog.torproject.org/v2-deprecation-timeline . Closes + ticket 40266. (NOTE: We accidentally released an earlier version + of the 0.4.6.1-alpha changelog without this entry. Sorry for + the confusion!) + + o Code simplification and refactoring (metrics, DoS): + - Move the DoS subsystem into the subsys manager, including its + configuration options. Closes ticket 40261. + + o Documentation (manual): + - Move the ServerTransport* options to the "SERVER OPTIONS" section. + Closes issue 40331. + - Indicate that the HiddenServiceStatistics option also applies to + bridges. Closes ticket 40346. + - Move the description of BridgeRecordUsageByCountry to the section + "STATISTICS OPTIONS". Closes ticket 40323. + + o Removed features (relay): + - Because DirPorts are only used on authorities, relays no longer + advertise them. Similarly, self-testing for DirPorts has been + disabled, since an unreachable DirPort is no reason for a relay + not to advertise itself. (Configuring a DirPort will still work, + for now.) Closes ticket 40282. + + +Changes in version 0.4.5.8 - 2021-05-10 + Tor 0.4.5.8 fixes several bugs in earlier version, backporting fixes + from the 0.4.6.x series. + + o Minor features (compatibility, Linux seccomp sandbox, backport from 0.4.6.3-rc): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation, backport from 0.4.6.3-rc): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add warning message when connecting to now deprecated v2 onion + services. As announced, Tor 0.4.5.x is the last series that will + support v2 onions. Closes ticket 40373. + + o Minor bugfixes (bridge, pluggable transport, backport from 0.4.6.2-alpha): + - Fix a regression that made it impossible start Tor using a bridge + line with a transport name and no fingerprint. Fixes bug 40360; + bugfix on 0.4.5.4-rc. + + o Minor bugfixes (build, cross-compilation, backport from 0.4.6.3-rc): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (controller, backport from 0.4.6.1-alpha): + - Fix a "BUG" warning that would appear when a controller chooses + the first hop for a circuit, and that circuit completes. Fixes bug + 40285; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, client, memory leak, backport from 0.4.6.3-rc): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD, backport from 0.4.6.2-alpha): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + Changes in version 0.3.5.14 - 2021-03-16 Tor 0.3.5.14 backports fixes for two important denial-of-service bugs in earlier versions of Tor. diff --git a/changes/bug40330 b/changes/bug40330 new file mode 100644 index 0000000000..8752ba5be7 --- /dev/null +++ b/changes/bug40330 @@ -0,0 +1,4 @@ + o Minor bugfixes (heartbeat): + - Adjust the heartbeat log message about distinct clients to consider + the HeartbeatPeriod rather than a flat 6-hour delay. + Fixes bug 40330; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug40365 b/changes/bug40365 new file mode 100644 index 0000000000..e4ee7b3b90 --- /dev/null +++ b/changes/bug40365 @@ -0,0 +1,3 @@ + o Minor bugfixes (tests): + - Fix a bug that prevented some tests from running with the correct names. + Fixes bug 40365; bugfix on 0.4.3.1-alpha. diff --git a/changes/bug40371 b/changes/bug40371 new file mode 100644 index 0000000000..8cc7117f9f --- /dev/null +++ b/changes/bug40371 @@ -0,0 +1,6 @@ + o Minor bugfixes (compatibility): + - Fix compatibility with the most recent Libevent versions, which + no longer have an evdns_set_random_bytes() function. Because + this function has been a no-op since Libevent 2.0.4-alpha, + it is safe for us to just stop calling it. Fixes bug 40371; + bugfix on 0.2.1.7-alpha. diff --git a/changes/bug40394 b/changes/bug40394 new file mode 100644 index 0000000000..f55167f2dd --- /dev/null +++ b/changes/bug40394 @@ -0,0 +1,4 @@ + o Minor bugfixes (statistics): + - Fix the fencepost issue when we check stability_last_downrated where + we call rep_hist_downrate_old_runs() twice. Fixes bug 40394; bugfix + on 0.2.0.5-alpha. Patch by Neel Chauhan. diff --git a/changes/bug40407 b/changes/bug40407 new file mode 100644 index 0000000000..068d278e14 --- /dev/null +++ b/changes/bug40407 @@ -0,0 +1,5 @@ + o Minor features (fuzzing): + - When building with --enable-libfuzzer, use a set of compiler flags + that works with more recent versions of the library. Previously + we were using a set of flags from 2017. + Closes ticket 40407. diff --git a/changes/bug40435 b/changes/bug40435 new file mode 100644 index 0000000000..76d0a687eb --- /dev/null +++ b/changes/bug40435 @@ -0,0 +1,4 @@ + o Minor bugfixes (circuit padding): + - Don't send STOP circuit padding cells when the other side has already + shut down the corresponding padding machine. Fixes bug 40435; bugfix on + 0.4.0.1-alpha.
\ No newline at end of file diff --git a/changes/issue40338 b/changes/issue40338 new file mode 100644 index 0000000000..bc2cafff61 --- /dev/null +++ b/changes/issue40338 @@ -0,0 +1,3 @@ + o Minor features (testing configuration): + - When TestingTorNetwork is enabled, skip the permission check on + the hidden service directory. Closes ticket 40338. diff --git a/changes/ticket33742 b/changes/ticket33742 new file mode 100644 index 0000000000..3669e76f48 --- /dev/null +++ b/changes/ticket33742 @@ -0,0 +1,3 @@ + o Documentation: + - Add links to original tor design paper and anonbib to + docs/HACKING/README.1st.md. Closes ticket 33742. Patch from Emily Bones. diff --git a/changes/ticket40209 b/changes/ticket40209 new file mode 100644 index 0000000000..a90243be8c --- /dev/null +++ b/changes/ticket40209 @@ -0,0 +1,4 @@ + o Minor features (bridge testing support): + - Let external bridge reachability testing tools discard cached + bridge descriptors when setting new bridges, so they can be sure + to get a clean reachability test. Implements ticket 40209. diff --git a/changes/ticket40337 b/changes/ticket40337 new file mode 100644 index 0000000000..1c86fc4c99 --- /dev/null +++ b/changes/ticket40337 @@ -0,0 +1,16 @@ + o Minor features (testing): + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change + the smallest amount of time over which they're willing to report + their observed maximum bandwidth. Previously, this was fixed + at 1 day. For safety, values under 2 hours are only supported on + testing networks. Part of a fix for ticket 40337. + + o Minor features (testing): + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited + until they had been running for a full day. Closes ticket + 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a fix + for ticket 40337. diff --git a/changes/ticket40363 b/changes/ticket40363 new file mode 100644 index 0000000000..713f943020 --- /dev/null +++ b/changes/ticket40363 @@ -0,0 +1,9 @@ + o Major features (Proposal 332, onion services, guard selection algorithm): + - Clients and onion services now choose four long-lived "layer 2" guard + relays for use as the middle hop in all onion circuits. These relays are + kept in place for a randomized duration averaging 1 week each. This + mitigates guard discovery attacks against clients and short-lived onion + services such as OnionShare. Long-lived onion services that need high + security should still use the Vanguards addon + (https://github.com/mikeperry-tor/vanguards). Closes ticket 40363; + implements proposal 333. diff --git a/configure.ac b/configure.ac index 788597bfdf..1b1d3a9892 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.6.7]) +AC_INIT([tor],[0.4.7.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -18,7 +18,7 @@ AC_DEFINE_UNQUOTED([CONFIG_FLAGS], ["$configure_flags"], [Flags passed to config # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2021-08-16"], # for 0.4.6.7 +AC_DEFINE(APPROX_RELEASE_DATE, ["2021-04-15"], # for 0.4.7.0-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 3dd470100f..1bf6554630 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.6.7" +!define VERSION "0.4.7.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index cd3417d0b5..c5dd6c744f 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -59,7 +59,7 @@ Some compatible licenses include: Each main development series (like 0.2.1, 0.2.2, etc) has its main work applied to a single branch. At most one series can be the development series at a time; all other series are maintenance series that get bug-fixes only. -The development series is built in a git branch called "master"; the +The development series is built in a git branch called "main"; the maintenance series are built in branches called "maint-0.2.0", "maint-0.2.1", and so on. We regularly merge the active maint branches forward. @@ -75,7 +75,7 @@ If you're working on a bugfix for a bug that occurs in a particular version, base your bugfix branch on the "maint" branch for the first supported series that has that bug. (As of June 2013, we're supporting 0.2.3 and later.) -If you're working on a new feature, base it on the master branch. If you're +If you're working on a new feature, base it on the main branch. If you're working on a new feature and it will take a while to implement and/or you'd like to avoid the possibility of unrelated bugs in Tor while you're implementing your feature, consider branching off of the latest maint- branch. diff --git a/doc/HACKING/GettingStarted.md b/doc/HACKING/GettingStarted.md index 6d61be9881..271e2d7517 100644 --- a/doc/HACKING/GettingStarted.md +++ b/doc/HACKING/GettingStarted.md @@ -41,7 +41,7 @@ Once you've reached this point, here's what you need to know. $ git clone https://git.torproject.org/git/tor ``` - This will give you a checkout of the master branch. If you're + This will give you a checkout of the main branch. If you're going to fix a bug that appears in a stable version, check out the appropriate "maint" branch, as in: diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md index 0ce59576f0..7849fc67c7 100644 --- a/doc/HACKING/HelpfulTools.md +++ b/doc/HACKING/HelpfulTools.md @@ -33,8 +33,8 @@ OFTC. If they don't, ask #tor-dev (also on OFTC). It's CI/builders. Looks like this: https://jenkins.torproject.org Runs automatically on commits merged to git.torproject.org. We CI the -master branch and all supported tor versions. We also build nightly debian -packages from master. +main branch and all supported tor versions. We also build nightly debian +packages from main. Builds Linux and Windows cross-compilation. Runs Linux tests. diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md index 4d5a7f6b76..267f6d0b58 100644 --- a/doc/HACKING/Maintaining.md +++ b/doc/HACKING/Maintaining.md @@ -7,7 +7,7 @@ The first section describes who is the current Tor maintainer and what are the responsibilities. Tor has one main single maintainer but does have many committers and subsystem maintainers. -The second third section describes how the **alpha and master** branches are +The second third section describes how the **alpha and main** branches are maintained and by whom. Finally, the last section describes how the **stable** branches are maintained @@ -25,7 +25,7 @@ protocol design. Releasing Tor falls under their responsibility. ## Alpha and Master Branches -The Tor repository always has at all times a **master** branch which contains +The Tor repository always has at all times a **main** branch which contains the upstream ongoing development. It may also contain a branch for a released feature freezed version which is @@ -37,7 +37,7 @@ Tor is separated into subsystems and some of those are maintained by other developers than the main maintainer. Those people have commit access to the code base but only commit (in most cases) into the subsystem they maintain. -Upstream merges are restricted to the alpha and master branches. Subsystem +Upstream merges are restricted to the alpha and main branches. Subsystem maintainers should never push a patch into a stable branch which is the responsibility of the [stable branch maintainer](#stable-branches). @@ -69,7 +69,7 @@ maintain the following subsystems: These are the tasks of a subsystem maintainer: 1. Regularly go over `merge_ready` tickets relevant to the related subsystem - and for the current alpha or development (master branch) Milestone. + and for the current alpha or development (main branch) Milestone. 2. A subsystem maintainer is expected to contribute to any design changes (including proposals) or large patch set about the subsystem. @@ -102,7 +102,7 @@ These are few important items to follow when merging code upstream: 4. Tor uses the "merge forward" method, that is, if a patch applies to the alpha branch, it has to be merged there first and then merged forward - into master. + into main. 5. Maintainer should always consult with the network team about any doubts, mis-understandings or unknowns of a patch. Final word will always go to the diff --git a/doc/HACKING/README.1st.md b/doc/HACKING/README.1st.md index 4bc3298c67..06a24e8300 100644 --- a/doc/HACKING/README.1st.md +++ b/doc/HACKING/README.1st.md @@ -36,6 +36,13 @@ For the latest version of the code, get a copy of git, and $ git clone https://git.torproject.org/git/tor ``` +For a copy of Tor's original design paper, see +[here](https://spec.torproject.org/tor-design). Note that Tor has changed in +many ways since 2004. + +For a large collection of security papers, many of which are related to Tor, +see [Anonbib's Selected Papers in Anonymity](https://www.freehaven.net/anonbib/). + ## Stay in touch We talk about Tor on the `tor-talk` mailing list. Design proposals and diff --git a/doc/HACKING/ReleaseSeriesLifecycle.md b/doc/HACKING/ReleaseSeriesLifecycle.md index 8536fbbd08..e47ac90fa5 100644 --- a/doc/HACKING/ReleaseSeriesLifecycle.md +++ b/doc/HACKING/ReleaseSeriesLifecycle.md @@ -87,17 +87,17 @@ they do not apply to security-related patch release versions. (Ideally, do this immediately after a release.) -1. Start a new maint-x.y.z branch based on master, and a new - release-x.y.z branch based on master. They should have the same +1. Start a new maint-x.y.z branch based on main, and a new + release-x.y.z branch based on main. They should have the same starting point. - Push both of these branches to the master git repository. + Push both of these branches to the canonical git repository. -2. In master, change the version to "0.x.y.0-alpha-dev". Run the +2. In the main branch, change the version to "0.x.y.0-alpha-dev". Run the update_versions.py script, and commit this version bump. 3. Tag the version bump with "tor-0.x.y.0-alpha-dev". Push the tag - and master. + and main branch. 4. Open tickets for connecting the new branches to various other places. See section 2 above for a list of affected locations. @@ -107,7 +107,7 @@ they do not apply to security-related patch release versions. target in the maint-x.y.z branch only. * Delete the file scripts/maint/practracker/.enable_practracker_in_hooks in the maint-x.y.z branch only. - * Merge to release-x.y.z, but do not forward-port to master. + * Merge to release-x.y.z, but do not forward-port to the main branch. 6. Finally, make sure this document is up to date with our latest process. diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 739ea38795..490c100fcb 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -1,3 +1,14 @@ +# CHECKLIST + +Here's a summary checklist, with the things that Nick messes up most often. + +Did you: + + * [ ] Copy the ChangeLog to the ReleaseNotes? + * [ ] Check that the new versions got approved? + * [ ] Check the release date in the ChangeLog? + * [ ] Update the GeoIP file? + # Putting out a new release Here are the steps that the maintainer should take when putting out a @@ -5,25 +16,13 @@ new Tor release: ## 0. Preliminaries -1. Get at least two of weasel/arma/Sebastian to put the new +1. Get at least three of weasel/arma/Sebastian/Sina to put the new version number in their approved versions list. Give them a few days to do this if you can. -2. If this is going to be an important security release, give these packagers - some advance warning: +2. If this is going to be an important security release, give the packagers + advance warning, via `tor-packagers@lists.torproject.org`. - - {weasel,sysrqb,mikeperry} at torproject dot org - - {blueness} at gentoo dot org - - {paul} at invizbox dot io - - {vincent} at invizbox dot com - - {lfleischer} at archlinux dot org - - {Nathan} at freitas dot net - - {mike} at tig dot as - - {tails-rm} at boum dot org - - {simon} at sdeziel.info - - {yuri} at freebsd.org - - {mh+tor} at scrit.ch - - {security} at brave.com 3. Given the release date for Tor, ask the TB team about the likely release date of a TB that contains it. See note below in "commit, upload, @@ -31,7 +30,9 @@ new Tor release: ## I. Make sure it works -1. Make sure that CI passes: have a look at Travis +1. Make sure that CI passes: have a look at the branches on gitlab. + + _Optionally_, have a look at Travis (https://travis-ci.org/torproject/tor/branches), Appveyor (https://ci.appveyor.com/project/torproject/tor/history), and Jenkins (https://jenkins.torproject.org/view/tor/). @@ -43,7 +44,7 @@ new Tor release: 2. Verify that there are no big outstanding issues. You might find such issues -- - * On Trac + * On Gitlab * On coverity scan @@ -51,6 +52,7 @@ new Tor release: ## II. Write a changelog + 1a. (Alpha release variant) Gather the `changes/*` files into a changelog entry, rewriting many @@ -110,7 +112,7 @@ new Tor release: note added to each section. So in this case, once you have the items from the changes files copied together, don't use them to build a new changelog: instead, look up the corrected versions that were merged - into ChangeLog in the master branch, and use those. + into ChangeLog in the main branch, and use those. Add "backport from X.Y.Z" in the section header for these entries. @@ -138,15 +140,15 @@ new Tor release: ## III. Making the source release. 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run - `make update-versions` to update version numbers in other + `./scripts/main/update_versions.py` to update version numbers in other places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. When you merge the maint branch forward to the next maint branch, or into - master, merge it with "-s ours" to avoid conflict with the version + main, merge it with `-s ours` to avoid conflict with the version bump. -2. Make distcheck, put the tarball up in somewhere (how about your - homedir on people.torproject.org?) , and tell `#tor-dev` +2. In `release-0.?.x`, run `make distcheck`, put the tarball up in somewhere + (how about your homedir on people.torproject.org?) , and tell `#tor-dev` about it. If you want, wait until at least one person has built it @@ -158,8 +160,8 @@ new Tor release: (Otherwise, users will get confused when it complains to them about its status.) - If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see - item 0.1 at the start of this document. + If it is not, you'll need to poke Roger, Weasel, Sebastian, and Sina + again: see the note at the start of the document. ## IV. Commit, upload, announce @@ -181,8 +183,8 @@ $ git push origin tag tor-0.4.x.y-<status> `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run "static-update-component dist.torproject.org" on dist-master. - In the project/web/tpo.git repository, update `databags/versions.ini` - to note the new version. Push these changes to master. + In the `project/web/tpo.git` repository, update `databags/versions.ini` + to note the new version. Push these changes to `master`. (NOTE: Due to #17805, there can only be one stable version listed at once. Nonetheless, do not call your version "alpha" if it is stable, @@ -196,7 +198,7 @@ $ git push origin tag tor-0.4.x.y-<status> Also, email tor-packagers@lists.torproject.org. - Mention where to download the tarball (https://dist.torproject.org). + Mention where to download the tarball (`https://dist.torproject.org/`). Include a link to the changelog. @@ -222,13 +224,32 @@ $ git push origin tag tor-0.4.x.y-<status> 1. If it's a stable release, bump the version number in the `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` - merge to avoid taking that change into master. + merge to avoid taking that change into main. 2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that builds the release every week. (It's ok to skip the weekly build if the branch was updated in the last 24 hours.) 3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the - master branch. + main branch. 4. Keep an eye on the blog post, to moderate comments and answer questions. + +## Appendix: An alternative means to notify packagers + +If for some reason you need to contact a bunch of packagers without +using the publicly archived tor-packagers list, you can try these +people: + + - {weasel,sysrqb,mikeperry} at torproject dot org + - {blueness} at gentoo dot org + - {paul} at invizbox dot io + - {vincent} at invizbox dot com + - {lfleischer} at archlinux dot org + - {Nathan} at freitas dot net + - {mike} at tig dot as + - {tails-rm} at boum dot org + - {simon} at sdeziel.info + - {yuri} at freebsd.org + - {mh+tor} at scrit.ch + - {security} at brave.com diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index 7222cd0548..2b5a1d9df7 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -1749,6 +1749,13 @@ The following options are useful only for clients (that is, if the guard-n-primary-guards consensus parameter, and default to 3 if the consensus parameter isn't set. (Default: 0) +[[VanguardsLiteEnabled]] **VanguardsLiteEnabled** **0**|**1**|**auto**:: + This option specifies whether clients should use the vanguards-lite + subsystem to protect against guard discovery attacks. If it's set to + 'auto', clients will do what the vanguards-lite-enabled consensus parameter + tells them to do, and will default to enable the subsystem if the consensus + parameter isn't set. (Default: auto) + [[UseMicrodescriptors]] **UseMicrodescriptors** **0**|**1**|**auto**:: Microdescriptors are a smaller version of the information that Tor needs in order to build its circuits. Using microdescriptors makes Tor clients @@ -3597,6 +3604,11 @@ The following options are used for running a testing Tor network. Minimum value for the Fast flag. Overrides the ordinary minimum taken from the consensus when TestingTorNetwork is set. (Default: 0.) +[[TestingMinTimeToReportBandwidth]] **TestingMinTimeToReportBandwidth** __N__ **seconds**|**minutes**|**hours**:: + Do not report our measurements for our maximum observed bandwidth for any + time period that has lasted for less than this amount of time. + Values over 1 day have no effect. (Default: 1 day) + [[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__:: Initial delay in seconds for when servers should download consensuses. Changing this requires that **TestingTorNetwork** is set. (Default: 0) diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index b52b956f81..95436a5a93 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -6,16 +6,16 @@ from __future__ import print_function from __future__ import unicode_literals FUZZERS = """ + address + addressPTR consensus descriptor diff diff-apply extrainfo - hsdescv2 hsdescv3 http http-connect - iptsv2 microdesc socks strops @@ -47,11 +47,10 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh index 2bcd4722b4..62547dcc5d 100755 --- a/scripts/git/git-list-tor-branches.sh +++ b/scripts/git/git-list-tor-branches.sh @@ -73,8 +73,8 @@ branch() { # location: where the branch can be found. - if [[ "$brname" == "master" ]]; then - suffix="_master" + if [[ "$brname" == "main" ]]; then + suffix="_main" location="\$GIT_PATH/\$TOR_MASTER_NAME" elif [[ "$brname" =~ ^maint- ]]; then suffix="_${brname_nodots#maint-}" @@ -139,12 +139,12 @@ finish() { branch maint-0.3.5 branch release-0.3.5 -branch maint-0.4.4 -branch release-0.4.4 - branch maint-0.4.5 branch release-0.4.5 -branch master +branch maint-0.4.6 +branch release-0.4.6 + +branch main finish diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 7c72f8478d..d5d663d558 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -12,8 +12,8 @@ function usage() echo " (default: run commands)" echo " -t: test branch mode: create new branches from the commits checked" echo " out in each maint directory. Call these branches prefix_035," - echo " prefix_040, ... , prefix_master." - echo " (default: merge forward maint-*, release-*, and master)" + echo " prefix_040, ... , prefix_main." + echo " (default: merge forward maint-*, release-*, and main)" echo " -u: in test branch mode, if a prefix_* branch already exists," echo " skip creating that branch. Use after a merge error, to" echo " restart the merge forward at the first unmerged branch." @@ -28,7 +28,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -45,7 +45,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The main branch git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -106,7 +106,7 @@ DRY_RUN=0 # Controlled by the -t <test-branch-prefix> option. The test branch base # name option makes git-merge-forward.sh create new test branches: -# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward. +# <tbbn>_035, <tbbn>_040, ... , <tbbn>_main, and merge forward. TEST_BRANCH_PREFIX= # Controlled by the -u option. The use existing option checks for existing diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 52a5c6140c..bbe2576d8e 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -20,7 +20,7 @@ usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -37,7 +37,7 @@ usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -51,7 +51,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -159,20 +159,7 @@ function goto_repo function fetch_origin { local cmd="git fetch origin" - printf " %s Fetching origin..." "$MARKER" - if [ $DRY_RUN -eq 0 ]; then - msg=$( eval "$cmd" 2>&1 ) - validate_ret $? "$msg" - else - printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" - fi -} - -# Fetch tor-github pull requests. No arguments. -function fetch_tor_github -{ - local cmd="git fetch tor-github" - printf " %s Fetching tor-github..." "$MARKER" + printf "%s Fetching origin..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -185,7 +172,7 @@ function fetch_tor_github function fetch_tor_gitlab { local cmd="git fetch tor-gitlab" - printf " %s Fetching tor-gitlab..." "$MARKER" + printf "%s Fetching tor-gitlab..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -198,11 +185,10 @@ function fetch_tor_gitlab # Entry point # ############### -# First, fetch tor-github. +# Get into our origin repository. goto_repo "$ORIGIN_PATH" -fetch_tor_github -# Then tor-gitlab +# First, fetch tor-gitlab fetch_tor_gitlab # Then, fetch the origin. diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 558ea8d01c..e5c16e615f 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -21,10 +21,10 @@ function usage() echo " -r: push to remote-name, rather than the default upstream remote." echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" echo " -t: test branch mode: push test branches to remote-name. Pushes" - echo " branches prefix_035, prefix_040, ... , prefix_master." - echo " (default: push maint-*, release-*, and master)" + echo " branches prefix_035, prefix_040, ... , prefix_main." + echo " (default: push maint-*, release-*, and main)" echo " -s: push branches whose tips match upstream maint, release, or" - echo " master branches. The default is to skip these branches," + echo " main branches. The default is to skip these branches," echo " because they do not contain any new code. Use -s to test for" echo " CI environment failures, using code that previously passed CI." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" @@ -45,7 +45,7 @@ function usage() echo " (default: use the current directory for pushes;" echo " current: $TOR_FULL_GIT_PATH)" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." @@ -55,9 +55,9 @@ function usage() echo " Overridden by <git push options> after --." echo " (default: git push --atomic; current: $GIT_PUSH)" echo " TOR_PUSH_SAME: push branches whose tips match upstream maint," - echo " release, or master branches. Inverted by -s." + echo " release, or main branches. Inverted by -s." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" - echo " TOR_PUSH_DELAY: pushes the master and maint branches separately," + echo " TOR_PUSH_DELAY: pushes the main and maint branches separately," echo " so that CI runs in a sensible order." echo " (default: push all branches immediately; current: $PUSH_DELAY)" echo " we recommend that you set these env vars in your ~/.profile" @@ -71,7 +71,7 @@ set -e # Don't change this configuration - set the env vars in your .profile # -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # Which directory do we push from? @@ -87,7 +87,7 @@ UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} # Push (1) or skip (0) test branches that are the same as an upstream -# maint/master branch. Push if you are testing that the CI environment still +# maint/main branch. Push if you are testing that the CI environment still # works on old code, skip if you are testing new code in the branch. # Default: skip unchanged branches. # Inverted by the -s option. @@ -99,7 +99,7 @@ PUSH_SAME=${TOR_PUSH_SAME:-0} # Controlled by the -t <test-branch-prefix> option. The test branch prefix # option makes git-merge-forward.sh create new test branches: -# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into +# <tbp>_035, <tbp>_040, ... , <tbp>_main, and merge each branch forward into # the next one. TEST_BRANCH_PREFIX= @@ -213,7 +213,7 @@ if [ "$TEST_BRANCH_PREFIX" ]; then # upstream branches (they have already been tested) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" else - # Skip the local maint-*, release-*, master branches that are the same as the + # Skip the local maint-*, release-*, main branches that are the same as the # current upstream branches, but ignore the default upstream # (we want to update a non-default remote, even if it matches the default) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" @@ -264,8 +264,8 @@ if [ "$PUSH_DELAY" -le 0 ]; then else # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) - MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ - || true # Skipped master branch + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep main$) \ + || true # Skipped main branch if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ || true # Skipped all maint branches @@ -273,7 +273,7 @@ else tr "\\n" " ") || true # Skipped all release branches else # Actually test branches based on maint branches - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v main$) \ || true # Skipped all maint test branches # No release branches RELEASE_BRANCHES= @@ -295,9 +295,9 @@ else # shellcheck disable=SC2086 for b in $MASTER_BRANCH $MAINT_BRANCHES; do $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" - # If we are pushing more than one branch, delay. - # In the unlikely scenario where we are pushing maint without master, - # or maint without release, there may be an extra delay + # If we are pushing more than one branch, delay. In the unlikely scenario + # where we are pushing maint branches without the main branch, or maint + # without release, there may be an extra delay if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then sleep "$PUSH_DELAY" fi diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh index 5a9ae41cbd..c502f74f58 100755 --- a/scripts/git/git-setup-dirs.sh +++ b/scripts/git/git-setup-dirs.sh @@ -22,7 +22,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -65,7 +65,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -102,7 +102,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -519,11 +519,11 @@ for ((i=0; i<COUNT; i++)); do repo_path=${!WORKTREE[$i]:1:1} printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" - # We cloned the repository, and master is the default branch - if [ "$branch" = "master" ]; then - if [ "$TOR_MASTER_NAME" != "master" ]; then - # Set up a master link in the worktree directory - make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + # We cloned the repository, and main is the default branch + if [ "$branch" = "main" ]; then + if [ "$TOR_MASTER_NAME" != "main" ]; then + # Set up a main branch link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/main" fi else # git makes worktree directories if they don't exist diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index eae4f999e7..b458630d26 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -36,8 +36,8 @@ check_for_script_update() { } cur_branch=$(git rev-parse --abbrev-ref HEAD) -if [ "$cur_branch" != "master" ]; then - echo "post-merge: Not a master branch. Skipping." +if [ "$cur_branch" != "main" ]; then + echo "post-merge: Not a main branch. Skipping." exit 0 fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f0a3a250ec..0f016df592 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -2,9 +2,9 @@ # git pre-push hook script to: # 0) Call the pre-commit hook, if it is available -# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# 1) prevent "fixup!" and "squash!" commit from ending up in main, release-* # or maint-* -# 2) Disallow pushing branches other than master, release-* +# 2) Disallow pushing branches other than main, release-* # and maint-* to origin (e.g. gitweb.torproject.org) # # To install this script, copy it into .git/hooks/pre-push path in your @@ -39,7 +39,7 @@ remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || + if [ "$1" == "refs/heads/main" ] || [[ "$1" == refs/heads/release-* ]] || [[ "$1" == refs/heads/maint-* ]]; then return 1 @@ -54,8 +54,8 @@ do : else if [ "$remote_sha" = $z40 ]; then - # New branch, examine commits not in master - range="master...$local_sha" + # New branch, examine commits not in main + range="main...$local_sha" else # Update to existing branch, examine new commits range="$remote_sha..$local_sha" diff --git a/scripts/maint/findMergedChanges.pl b/scripts/maint/findMergedChanges.pl index d6c4105b74..9b3496403d 100755 --- a/scripts/maint/findMergedChanges.pl +++ b/scripts/maint/findMergedChanges.pl @@ -9,7 +9,7 @@ sub nChanges { # requires perl 5.8. Avoids shell issues if we ever get a changes # file named by the parents of Little Johnny Tables. open F, "-|", "git", "log", "--no-merges", "--pretty=format:%H", $branches, "--", $fname - or die "$!"; + or die "$!"; my @changes = <F>; return scalar @changes } @@ -22,13 +22,13 @@ Usage: findMergedChanges.pl [--merged/--unmerged/--weird/--list] [--branch=<branchname] [--head=<branchname>] changes/* A change is "merged" if it has ever been merged to release-0.2.4 and it has had -no subsequent changes in master. +no subsequent changes in main. A change is "unmerged" if it has never been merged to release-0.2.4 and it -has had changes in master. +has had changes in main. A change is "weird" if it has been merged to release-0.2.4 and it *has* had -subsequent changes in master. +subsequent changes in main. Suggested application: findMergedChanges.pl --merged changes/* | xargs -n 1 git rm @@ -37,18 +37,18 @@ EOF } my $target_branch = "origin/release-0.2.4"; -my $head = "origin/master"; +my $head = "origin/main"; while (@ARGV and $ARGV[0] =~ /^--/) { my $flag = shift @ARGV; if ($flag =~ /^--(weird|merged|unmerged|list)/) { - $look_for_type = $1; + $look_for_type = $1; } elsif ($flag =~ /^--branch=(\S+)/) { $target_branch = $1; } elsif ($flag =~ /^--head=(\S+)/) { $head = $1; } else { - die "Unrecognized flag $flag"; + die "Unrecognized flag $flag"; } } @@ -58,16 +58,16 @@ for my $changefile (@ARGV) { my $type; if ($n_merged != 0 and $n_postmerged == 0) { - $type = "merged"; + $type = "merged"; } elsif ($n_merged == 0 and $n_postmerged != 0) { - $type = "unmerged"; + $type = "unmerged"; } else { - $type = "weird"; + $type = "weird"; } if ($type eq $look_for_type) { - print "$changefile\n"; + print "$changefile\n"; } elsif ($look_for_type eq 'list') { - printf "% 8s: %s\n", $type, $changefile; + printf "% 8s: %s\n", $type, $changefile; } } diff --git a/src/app/config/config.c b/src/app/config/config.c index bfa258c904..15b4585954 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -550,6 +550,7 @@ static const config_var_t option_vars_[] = { V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"), VPORT(MetricsPort), V(MetricsPortPolicy, LINELIST, NULL), + V(TestingMinTimeToReportBandwidth, INTERVAL, "1 day"), VAR("MyFamily", LINELIST, MyFamily_lines, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), OBSOLETE("NamingAuthoritativeDirectory"), @@ -615,6 +616,7 @@ static const config_var_t option_vars_[] = { V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(CircuitPadding, BOOL, "1"), + V(ReconfigDropsBridgeDescs, BOOL, "0"), V(ReducedCircuitPadding, BOOL, "0"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), @@ -667,6 +669,7 @@ static const config_var_t option_vars_[] = { VAR("UseEntryGuards", BOOL, UseEntryGuards_option, "1"), OBSOLETE("UseEntryGuardsAsDirGuards"), V(UseGuardFraction, AUTOBOOL, "auto"), + V(VanguardsLiteEnabled, AUTOBOOL, "auto"), V(UseMicrodescriptors, AUTOBOOL, "auto"), OBSOLETE("UseNTorHandshake"), V_IMMUTABLE(User, STRING, NULL), @@ -2308,6 +2311,8 @@ options_act,(const or_options_t *old_options)) } if (transition_affects_guards) { + if (options->ReconfigDropsBridgeDescs) + routerlist_drop_bridge_descriptors(); if (guards_update_all()) { abandon_circuits = 1; } @@ -3980,6 +3985,7 @@ options_validate_cb(const void *old_options_, void *options_, char **msg) CHECK_DEFAULT(TestingSigningKeySlop); CHECK_DEFAULT(TestingAuthKeySlop); CHECK_DEFAULT(TestingLinkKeySlop); + CHECK_DEFAULT(TestingMinTimeToReportBandwidth); or_options_free(dflt_options); } #undef CHECK_DEFAULT @@ -5466,6 +5472,77 @@ pt_parse_transport_line(const or_options_t *options, return r; } +/** + * Parse a flag describing an extra dirport for a directory authority. + * + * Right now, the supported format is exactly: + * `{upload,download,voting}=http://[IP:PORT]/`. + * Other URL schemes, and other suffixes, might be supported in the future. + * + * Only call this function if `flag` starts with one of the above strings. + * + * Return 0 on success, and -1 on failure. + * + * If `ds` is provided, then add any parsed dirport to `ds`. If `ds` is NULL, + * take no action other than parsing. + **/ +static int +parse_dirauth_dirport(dir_server_t *ds, const char *flag) +{ + tor_assert(flag); + + auth_dirport_usage_t usage; + + if (!strcasecmpstart(flag, "upload=")) { + usage = AUTH_USAGE_UPLOAD; + } else if (!strcasecmpstart(flag, "download=")) { + usage = AUTH_USAGE_DOWNLOAD; + } else if (!strcasecmpstart(flag, "vote=")) { + usage = AUTH_USAGE_VOTING; + } else { + // We shouldn't get called with a flag that we don't recognize. + tor_assert_nonfatal_unreached(); + return -1; + } + + const char *eq = strchr(flag, '='); + tor_assert(eq); + const char *target = eq + 1; + + // Find the part inside the http://{....}/ + if (strcmpstart(target, "http://")) { + log_warn(LD_CONFIG, "Unsupported URL scheme in authority flag %s", flag); + return -1; + } + const char *addr = target + strlen("http://"); + + const char *eos = strchr(addr, '/'); + size_t addr_len; + if (eos && strcmp(eos, "/")) { + log_warn(LD_CONFIG, "Unsupported URL prefix in authority flag %s", flag); + return -1; + } else if (eos) { + addr_len = eos - addr; + } else { + addr_len = strlen(addr); + } + + // Finally, parse the addr:port part. + char *addr_string = tor_strndup(addr, addr_len); + tor_addr_port_t dirport; + memset(&dirport, 0, sizeof(dirport)); + int rv = tor_addr_port_parse(LOG_WARN, addr_string, + &dirport.addr, &dirport.port, -1); + if (ds != NULL && rv == 0) { + trusted_dir_server_add_dirport(ds, usage, &dirport); + } else if (rv == -1) { + log_warn(LD_CONFIG, "Unable to parse address in authority flag %s",flag); + } + + tor_free(addr_string); + return rv; +} + /** Read the contents of a DirAuthority line from <b>line</b>. If * <b>validate_only</b> is 0, and the line is well-formed, and it * shares any bits with <b>required_type</b> or <b>required_type</b> @@ -5486,6 +5563,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char v3_digest[DIGEST_LEN]; dirinfo_type_t type = 0; double weight = 1.0; + smartlist_t *extra_dirports = smartlist_new(); memset(v3_digest, 0, sizeof(v3_digest)); @@ -5554,6 +5632,12 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } ipv6_addrport_ptr = &ipv6_addrport; } + } else if (!strcasecmpstart(flag, "upload=") || + !strcasecmpstart(flag, "download=") || + !strcasecmpstart(flag, "vote=")) { + // We'll handle these after creating the authority object. + smartlist_add(extra_dirports, flag); + flag = NULL; // prevent double-free. } else { log_warn(LD_CONFIG, "Unrecognized flag '%s' on DirAuthority line", flag); @@ -5597,6 +5681,13 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, goto err; } + if (validate_only) { + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(NULL, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); + } + if (!validate_only && (!required_type || required_type & type)) { dir_server_t *ds; if (required_type) @@ -5608,16 +5699,23 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, ipv6_addrport_ptr, digest, v3_digest, type, weight))) goto err; + + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(ds, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); dir_server_add(ds); } r = 0; goto done; - err: + err: r = -1; - done: + done: + SMARTLIST_FOREACH(extra_dirports, char*, s, tor_free(s)); + smartlist_free(extra_dirports); SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 510ece42a3..812fa92cae 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -293,6 +293,13 @@ struct or_options_t { * disabled. */ int CircuitPadding; + /** Boolean: if true, then this client will discard cached bridge + * descriptors on a setconf or other config change that impacts guards + * or bridges (see options_transition_affects_guards() for exactly which + * config changes trigger it). Useful for tools that test bridge + * reachability by fetching fresh descriptors. */ + int ReconfigDropsBridgeDescs; + /** Boolean: if true, then this client will only use circuit padding * algorithms that are known to use a low amount of overhead. If false, * we will use all available circuit padding algorithms. @@ -587,6 +594,9 @@ struct or_options_t { * If 0, use value from NumEntryGuards. */ int NumPrimaryGuards; /**< How many primary guards do we want? */ + /** Boolean: Switch to toggle the vanguards-lite subsystem */ + int VanguardsLiteEnabled; + int RephistTrackTime; /**< How many seconds do we keep rephist info? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ @@ -1060,6 +1070,10 @@ struct or_options_t { /** List of policy allowed to query the Metrics port. */ struct config_line_t *MetricsPortPolicy; + /** How far must we be into the current bandwidth-measurement period to + * report bandwidth observations from this period? */ + int TestingMinTimeToReportBandwidth; + /** * Configuration objects for individual modules. * diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc index 00b307782b..039454a0d0 100644 --- a/src/app/config/testnet.inc +++ b/src/app/config/testnet.inc @@ -19,6 +19,7 @@ { "TestingV3AuthInitialDistDelay", "20 seconds" }, { "TestingAuthDirTimeToLearnReachability", "0 minutes" }, { "MinUptimeHidServDirectoryV2", "0 minutes" }, +{ "TestingMinTimeToReportBandwidth", "0 seconds" }, { "TestingServerDownloadInitialDelay", "0" }, { "TestingClientDownloadInitialDelay", "0" }, { "TestingServerConsensusDownloadInitialDelay", "0" }, diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 7f43cd324e..667ab294b4 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -224,4 +224,4 @@ ## mechanisms like https://bridges.torproject.org/. If you want to run ## a private bridge, for example because you'll give out your bridge ## address manually to your friends, uncomment this line: -#PublishServerDescriptor 0 +#BridgeDistribution none diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 5d593871dd..edc30d043c 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -239,7 +239,7 @@ ## mechanisms like https://bridges.torproject.org/. If you want to run ## a private bridge, for example because you'll give out your bridge ## address manually to your friends, uncomment this line: -#PublishServerDescriptor 0 +#BridgeDistribution none ## Configuration options can be imported from files or folders using the %include ## option with the value being a path. This path can have wildcards. Wildcards are diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index b17d7bf2bd..79e034fb34 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -250,13 +250,13 @@ CONST_TO_LISTENER_CONN(const connection_t *c) } size_t -connection_get_inbuf_len(connection_t *conn) +connection_get_inbuf_len(const connection_t *conn) { return conn->inbuf ? buf_datalen(conn->inbuf) : 0; } size_t -connection_get_outbuf_len(connection_t *conn) +connection_get_outbuf_len(const connection_t *conn) { return conn->outbuf ? buf_datalen(conn->outbuf) : 0; } @@ -1261,7 +1261,7 @@ socket_failed_from_resource_exhaustion(void) */ if (get_max_sockets() > 65535) { /* TCP port exhaustion */ - rep_hist_note_overload(OVERLOAD_GENERAL); + rep_hist_note_tcp_exhaustion(); } else { /* File descriptor exhaustion */ rep_hist_note_overload(OVERLOAD_FD_EXHAUSTED); diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 36c94d6570..8b378b15a4 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -274,8 +274,8 @@ void connection_buf_add_compress(const char *string, size_t len, struct dir_connection_t *conn, int done); void connection_buf_add_buf(struct connection_t *conn, struct buf_t *buf); -size_t connection_get_inbuf_len(struct connection_t *conn); -size_t connection_get_outbuf_len(struct connection_t *conn); +size_t connection_get_inbuf_len(const struct connection_t *conn); +size_t connection_get_outbuf_len(const struct connection_t *conn); struct connection_t *connection_get_by_global_id(uint64_t id); struct connection_t *connection_get_by_type(int type); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 69606c0d53..37b53db92a 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1293,6 +1293,7 @@ signewnym_impl(time_t now) circuit_mark_all_dirty_circs_as_unusable(); addressmap_clear_transient(); hs_client_purge_state(); + purge_vanguards_lite(); time_of_last_signewnym = now; signewnym_is_pending = 0; @@ -1370,6 +1371,7 @@ CALLBACK(save_state); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); +CALLBACK(manage_vglite); #undef CALLBACK @@ -1392,6 +1394,9 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { CALLBACK(second_elapsed, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + /* Update vanguards-lite once per hour, if we have networking */ + CALLBACK(manage_vglite, NET_PARTICIPANT, FL(NEED_NET)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), @@ -1662,6 +1667,21 @@ mainloop_schedule_shutdown(int delay_sec) mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); } +/** + * Update vanguards-lite layer2 nodes, once every 15 minutes + */ +static int +manage_vglite_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; +#define VANGUARDS_LITE_INTERVAL (15*60) + + maintain_layer2_guards(); + + return VANGUARDS_LITE_INTERVAL; +} + /** Perform regular maintenance tasks. This function gets run once per * second. */ diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 870bcbf7cf..be6429438a 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -22,6 +22,7 @@ struct hs_token_t; struct circpad_machine_spec_t; struct circpad_machine_runtime_t; +struct congestion_control_t; /** Number of padding state machines on a circuit. */ #define CIRCPAD_MAX_MACHINES (2) @@ -244,6 +245,9 @@ struct circuit_t { * that STOP commands actually correspond to the current machine, * and not a previous one. */ uint32_t padding_machine_ctr; + + /** Congestion control fields */ + struct congestion_control_t *ccontrol; }; #endif /* !defined(CIRCUIT_ST_H) */ diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 2bcc642a97..31e3868b65 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1359,7 +1359,9 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) int routelen = DEFAULT_ROUTE_LEN; int known_purpose = 0; - if (circuit_should_use_vanguards(purpose)) { + /* If we're using L3 vanguards, we need longer paths for onion services */ + if (circuit_purpose_is_hidden_service(purpose) && + get_options()->HSLayer3Nodes) { /* Clients want an extra hop for rends to avoid linkability. * Services want it for intro points to avoid publishing their * layer3 guards. They want it for hsdir posts to use @@ -1374,14 +1376,6 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) return routelen+1; - /* If we only have Layer2 vanguards, then we do not need - * the extra hop for linkabilty reasons (see below). - * This means all hops can be of the form: - * S/C - G - L2 - M - R/HSDir/I - */ - if (get_options()->HSLayer2Nodes && !get_options()->HSLayer3Nodes) - return routelen+1; - /* For connections to hsdirs, clients want two extra hops * when using layer3 guards, to avoid linkability. * Same goes for intro points. Note that the route len @@ -1400,16 +1394,14 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) return routelen; switch (purpose) { - /* These two purposes connect to a router that we chose, so - * DEFAULT_ROUTE_LEN is safe. */ - case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: - /* hidden service connecting to introduction point */ + /* These purposes connect to a router that we chose, so DEFAULT_ROUTE_LEN + * is safe: */ case CIRCUIT_PURPOSE_TESTING: /* router reachability testing */ known_purpose = 1; break; - /* These three purposes connect to a router that someone else + /* These purposes connect to a router that someone else * might have chosen, so add an extra hop to protect anonymity. */ case CIRCUIT_PURPOSE_C_GENERAL: case CIRCUIT_PURPOSE_C_HSDIR_GET: @@ -1419,6 +1411,9 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) /* client connecting to introduction point */ case CIRCUIT_PURPOSE_S_CONNECT_REND: /* hidden service connecting to rendezvous point */ + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + /* hidden service connecting to intro point. In this case we want an extra + hop to avoid linkability attacks by the introduction point. */ known_purpose = 1; routelen++; break; @@ -2019,7 +2014,7 @@ cpath_build_state_to_crn_ipv6_extend_flag(const cpath_build_state_t *state, } /** Decide a suitable length for circ's cpath, and pick an exit - * router (or use <b>exit</b> if provided). Store these in the + * router (or use <b>exit_ei</b> if provided). Store these in the * cpath. * * If <b>is_hs_v3_rp_circuit</b> is set, then this exit should be suitable to @@ -2072,7 +2067,7 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, return 0; } -/** Give <b>circ</b> a new exit destination to <b>exit</b>, and add a +/** Give <b>circ</b> a new exit destination to <b>exit_ei</b>, and add a * hop to the cpath reflecting this. Don't send the next extend cell -- * the caller will do this if it wants to. */ @@ -2114,8 +2109,6 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) return -1; } - // XXX: Should cannibalized circuits be dirty or not? Not easy to say.. - return 0; } @@ -2261,8 +2254,14 @@ middle_node_must_be_vanguard(const or_options_t *options, return 0; } - /* If we have sticky L2 nodes, and this is an L2 pick, use vanguards */ - if (options->HSLayer2Nodes && cur_len == 1) { + /* Don't even bother if the feature is disabled */ + if (!vanguards_lite_is_enabled()) { + return 0; + } + + /* If we are a hidden service circuit, always use either vanguards-lite + * or HSLayer2Nodes for 2nd hop. */ + if (cur_len == 1) { return 1; } @@ -2286,7 +2285,8 @@ pick_vanguard_middle_node(const or_options_t *options, /* Pick the right routerset based on the current hop */ if (cur_len == 1) { - vanguard_routerset = options->HSLayer2Nodes; + vanguard_routerset = options->HSLayer2Nodes ? + options->HSLayer2Nodes : get_layer2_guards(); } else if (cur_len == 2) { vanguard_routerset = options->HSLayer3Nodes; } else { @@ -2295,6 +2295,10 @@ pick_vanguard_middle_node(const or_options_t *options, return NULL; } + if (BUG(!vanguard_routerset)) { + return NULL; + } + node = pick_restricted_middle_node(flags, vanguard_routerset, options->ExcludeNodes, excluded, cur_len+1); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 4f62284e29..35d810c660 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -100,6 +100,7 @@ #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/buf/buffers.h" +#include "core/or/congestion_control_common.h" #include "core/or/ocirc_event.h" @@ -1143,6 +1144,8 @@ circuit_free_(circuit_t *circ) * hs identifier is freed. */ hs_circ_cleanup_on_free(circ); + congestion_control_free(circ->ccontrol); + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; @@ -2586,8 +2589,10 @@ conns_compare_by_buffer_age_(const void **a_, const void **b_) /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ -void + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. + * + * Return the number of bytes removed. */ +size_t circuits_handle_oom(size_t current_allocation) { smartlist_t *circlist; @@ -2613,12 +2618,11 @@ circuits_handle_oom(size_t current_allocation) tor_zstd_get_total_allocation(), tor_lzma_get_total_allocation(), hs_cache_get_total_allocation()); - { size_t mem_target = (size_t)(get_options()->MaxMemInQueues * FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) - return; + return 0; mem_to_recover = current_allocation - mem_target; } @@ -2697,7 +2701,6 @@ circuits_handle_oom(size_t current_allocation) } SMARTLIST_FOREACH_END(circ); done_recovering_mem: - log_notice(LD_GENERAL, "Removed %"TOR_PRIuSZ" bytes by killing %d circuits; " "%d circuits remain alive. Also killed %d non-linked directory " "connections.", @@ -2705,6 +2708,8 @@ circuits_handle_oom(size_t current_allocation) n_circuits_killed, smartlist_len(circlist) - n_circuits_killed, n_dirconns_killed); + + return mem_recovered; } /** Verify that circuit <b>c</b> has all of its invariants diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index f5791d7c12..147e2cb2f8 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -232,7 +232,7 @@ int circuit_count_pending_on_channel(channel_t *chan); MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c)); void circuit_free_all(void); -void circuits_handle_oom(size_t current_allocation); +size_t circuits_handle_oom(size_t current_allocation); void circuit_clear_testing_cell_stats(circuit_t *circ); diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6dfe94de01..99dc5f9d83 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2967,6 +2967,8 @@ signed_error_t circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) { int retval = 0; + /* Should we send back a STOP cell? */ + bool respond_with_stop = true; circpad_negotiate_t *negotiate; if (CIRCUIT_IS_ORIGIN(circ)) { @@ -2992,6 +2994,12 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) negotiate->machine_type, negotiate->machine_ctr); goto done; } + + /* If we reached this point we received a STOP command from an old or + unknown machine. Don't reply with our own STOP since there is no one to + handle it on the other end */ + respond_with_stop = false; + if (negotiate->machine_ctr <= circ->padding_machine_ctr) { log_info(LD_CIRC, "Received STOP command for old machine %u, ctr %u", negotiate->machine_type, negotiate->machine_ctr); @@ -3023,10 +3031,13 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) retval = -1; done: - circpad_padding_negotiated(circ, negotiate->machine_type, - negotiate->command, - (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR, - negotiate->machine_ctr); + if (respond_with_stop) { + circpad_padding_negotiated(circ, negotiate->machine_type, + negotiate->command, + (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR, + negotiate->machine_ctr); + } + circpad_negotiate_free(negotiate); return retval; diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 044b30b8b3..2ec391eca0 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1204,25 +1204,6 @@ needs_circuits_for_build(int num) return 0; } -/** - * Launch the appropriate type of predicted circuit for hidden - * services, depending on our options. - */ -static void -circuit_launch_predicted_hs_circ(int flags) -{ - /* K.I.S.S. implementation of bug #23101: If we are using - * vanguards or pinned middles, pre-build a specific purpose - * for HS circs. */ - if (circuit_should_use_vanguards(CIRCUIT_PURPOSE_HS_VANGUARDS)) { - circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); - } else { - /* If no vanguards, then no HS-specific prebuilt circuits are needed. - * Normal GENERAL circs are fine */ - circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); - } -} - /** Determine how many circuits we have open that are clean, * Make sure it's enough for all the upcoming behaviors we predict we'll have. * But put an upper bound on the total number of circuits. @@ -1276,7 +1257,7 @@ circuit_predict_and_launch_new(void) "Have %d clean circs (%d internal), need another internal " "circ for my hidden service.", num, num_internal); - circuit_launch_predicted_hs_circ(flags); + circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); return; } @@ -1295,7 +1276,10 @@ circuit_predict_and_launch_new(void) " another hidden service circ.", num, num_uptime_internal, num_internal); - circuit_launch_predicted_hs_circ(flags); + /* Always launch vanguards purpose circuits for HS clients, + * for vanguards-lite. This prevents us from cannibalizing + * to build these circuits (and thus not use vanguards). */ + circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); return; } @@ -2022,16 +2006,12 @@ circuit_is_hs_v3(const circuit_t *circ) int circuit_should_use_vanguards(uint8_t purpose) { - const or_options_t *options = get_options(); - - /* Only hidden service circuits use vanguards */ - if (!circuit_purpose_is_hidden_service(purpose)) - return 0; - - /* Pinned middles are effectively vanguards */ - if (options->HSLayer2Nodes || options->HSLayer3Nodes) + /* All hidden service circuits use either vanguards or + * vanguards-lite. */ + if (circuit_purpose_is_hidden_service(purpose)) return 1; + /* Everything else is a normal circuit */ return 0; } @@ -2069,13 +2049,11 @@ circuit_should_cannibalize_to_build(uint8_t purpose_to_build, return 0; } - /* For vanguards, the server-side intro circ is not cannibalized - * because we pre-build 4 hop HS circuits, and it only needs a 3 hop - * circuit. It is also long-lived, so it is more important that - * it have lower latency than get built fast. + /* The server-side intro circ is not cannibalized because it only + * needs a 3 hop circuit. It is also long-lived, so it is more + * important that it have lower latency than get built fast. */ - if (circuit_should_use_vanguards(purpose_to_build) && - purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) { + if (purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) { return 0; } diff --git a/src/core/or/congestion_control_common.c b/src/core/or/congestion_control_common.c new file mode 100644 index 0000000000..9db1d7d664 --- /dev/null +++ b/src/core/or/congestion_control_common.c @@ -0,0 +1,933 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_common.c + * \brief Common code used by all congestion control algorithms. + */ + +#define TOR_CONGESTION_CONTROL_COMMON_PRIVATE + +#include "core/or/or.h" + +#include "core/or/circuitlist.h" +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "core/mainloop/connection.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_vegas.h" +#include "core/or/congestion_control_nola.h" +#include "core/or/congestion_control_westwood.h" +#include "core/or/congestion_control_st.h" +#include "lib/time/compat_time.h" +#include "feature/nodelist/networkstatus.h" + +/* Consensus parameter defaults */ +#define CIRCWINDOW_INIT (500) + +#define CWND_INC_PCT_SS_DFLT (100) + +#define SENDME_INC_DFLT (50) +#define CWND_MIN_DFLT (MAX(100, SENDME_INC_DFLT)) + +#define CWND_INC_DFLT (50) + +#define CWND_INC_RATE_DFLT (1) + +#define WESTWOOD_BDP_ALG BDP_ALG_PIECEWISE +#define VEGAS_BDP_MIX_ALG BDP_ALG_PIECEWISE +#define NOLA_BDP_ALG BDP_ALG_PIECEWISE + +#define EWMA_CWND_COUNT_DFLT 2 + +#define BWE_SENDME_MIN_DFLT 5 + +static uint64_t congestion_control_update_circuit_rtt(congestion_control_t *, + uint64_t); +static bool congestion_control_update_circuit_bdp(congestion_control_t *, + const circuit_t *, + const crypt_path_t *, + uint64_t, uint64_t); + +/** + * Set congestion control parameters on a circuit's congestion + * control object based on values from the consensus. + * + * cc_alg is the negotiated congestion control algorithm. + * + * sendme_inc is the number of packaged cells that a sendme cell + * acks. This parameter will come from circuit negotiation. + */ +static void +congestion_control_init_params(congestion_control_t *cc, + cc_alg_t cc_alg, + int sendme_inc) +{ +#define CWND_INIT_MIN 100 +#define CWND_INIT_MAX (10000) + cc->cwnd = + networkstatus_get_param(NULL, "cc_cwnd_init", + CIRCWINDOW_INIT, + CWND_INIT_MIN, + CWND_INIT_MAX); + +#define CWND_INC_PCT_SS_MIN 1 +#define CWND_INC_PCT_SS_MAX (500) + cc->cwnd_inc_pct_ss = + networkstatus_get_param(NULL, "cc_cwnd_inc_pct_ss", + CWND_INC_PCT_SS_DFLT, + CWND_INC_PCT_SS_MIN, + CWND_INC_PCT_SS_MAX); + +#define CWND_INC_MIN 1 +#define CWND_INC_MAX (1000) + cc->cwnd_inc = + networkstatus_get_param(NULL, "cc_cwnd_inc", + CWND_INC_DFLT, + CWND_INC_MIN, + CWND_INC_MAX); + +#define CWND_INC_RATE_MIN 1 +#define CWND_INC_RATE_MAX (250) + cc->cwnd_inc_rate = + networkstatus_get_param(NULL, "cc_cwnd_inc_rate", + CWND_INC_RATE_DFLT, + CWND_INC_RATE_MIN, + CWND_INC_RATE_MAX); + +#define SENDME_INC_MIN 10 +#define SENDME_INC_MAX (1000) + cc->sendme_inc = + networkstatus_get_param(NULL, "cc_sendme_inc", + sendme_inc, + SENDME_INC_MIN, + SENDME_INC_MAX); + + // XXX: this min needs to abide by sendme_inc range rules somehow +#define CWND_MIN_MIN sendme_inc +#define CWND_MIN_MAX (1000) + cc->cwnd_min = + networkstatus_get_param(NULL, "cc_cwnd_min", + CWND_MIN_DFLT, + CWND_MIN_MIN, + CWND_MIN_MAX); + +#define EWMA_CWND_COUNT_MIN 1 +#define EWMA_CWND_COUNT_MAX (100) + cc->ewma_cwnd_cnt = + networkstatus_get_param(NULL, "cc_ewma_cwnd_cnt", + EWMA_CWND_COUNT_DFLT, + EWMA_CWND_COUNT_MIN, + EWMA_CWND_COUNT_MAX); + +#define BWE_SENDME_MIN_MIN 2 +#define BWE_SENDME_MIN_MAX (20) + cc->bwe_sendme_min = + networkstatus_get_param(NULL, "cc_bwe_min", + BWE_SENDME_MIN_DFLT, + BWE_SENDME_MIN_MIN, + BWE_SENDME_MIN_MAX); + +#define CC_ALG_MIN 0 +#define CC_ALG_MAX (NUM_CC_ALGS-1) + cc->cc_alg = + networkstatus_get_param(NULL, "cc_alg", + cc_alg, + CC_ALG_MIN, + CC_ALG_MAX); + + bdp_alg_t default_bdp_alg = 0; + + switch (cc->cc_alg) { + case CC_ALG_WESTWOOD: + default_bdp_alg = WESTWOOD_BDP_ALG; + break; + case CC_ALG_VEGAS: + default_bdp_alg = VEGAS_BDP_MIX_ALG; + break; + case CC_ALG_NOLA: + default_bdp_alg = NOLA_BDP_ALG; + break; + case CC_ALG_SENDME: + default: + tor_fragile_assert(); + return; // No alg-specific params + } + + cc->bdp_alg = + networkstatus_get_param(NULL, "cc_bdp_alg", + default_bdp_alg, + 0, + NUM_BDP_ALGS-1); + + /* Algorithm-specific parameters */ + if (cc->cc_alg == CC_ALG_WESTWOOD) { + congestion_control_westwood_set_params(cc); + } else if (cc->cc_alg == CC_ALG_VEGAS) { + congestion_control_vegas_set_params(cc); + } else if (cc->cc_alg == CC_ALG_NOLA) { + congestion_control_nola_set_params(cc); + } +} + +/** + * Allocate and initialize fields in congestion control object. + * + * cc_alg is the negotiated congestion control algorithm. + * + * sendme_inc is the number of packaged cells that a sendme cell + * acks. This parameter will come from circuit negotiation. + */ +static void +congestion_control_init(congestion_control_t *cc, cc_alg_t cc_alg, + int sendme_inc) +{ + cc->sendme_pending_timestamps = smartlist_new(); + cc->sendme_arrival_timestamps = smartlist_new(); + + cc->in_slow_start = 1; + congestion_control_init_params(cc, cc_alg, sendme_inc); + + cc->next_cc_event = CWND_UPDATE_RATE(cc); +} + +/** Allocate and initialize a new congestion control object */ +congestion_control_t * +congestion_control_new(void) +{ + congestion_control_t *cc = tor_malloc_zero(sizeof(congestion_control_t)); + + // XXX: the alg and the sendme_inc need to be negotiated during + // circuit handshake + congestion_control_init(cc, CC_ALG_VEGAS, SENDME_INC_DFLT); + + return cc; +} + +/** + * Free a congestion control object and its asssociated state. + */ +void +congestion_control_free_(congestion_control_t *cc) +{ + if (!cc) + return; + + SMARTLIST_FOREACH(cc->sendme_pending_timestamps, uint64_t *, t, tor_free(t)); + SMARTLIST_FOREACH(cc->sendme_arrival_timestamps, uint64_t *, t, tor_free(t)); + smartlist_free(cc->sendme_pending_timestamps); + smartlist_free(cc->sendme_arrival_timestamps); + + tor_free(cc); +} + +/** + * Compute an N-count EWMA, aka N-EWMA. N-EWMA is defined as: + * EWMA = alpha*value + (1-alpha)*EWMA_prev + * with alpha = 2/(N+1). + * + * This works out to: + * EWMA = value*2/(N+1) + EMA_prev*(N-1)/(N+1) + * = (value*2 + EWMA_prev*(N-1))/(N+1) + */ +static inline uint64_t +n_count_ewma(uint64_t curr, uint64_t prev, uint64_t N) +{ + if (prev == 0) + return curr; + else + return (2*curr + (N-1)*prev)/(N+1); +} + +/** + * Enqueue a u64 timestamp to the end of a queue of timestamps. + */ +static inline void +enqueue_timestamp(smartlist_t *timestamps_u64, uint64_t timestamp_usec) +{ + uint64_t *timestamp_ptr = tor_malloc(sizeof(uint64_t)); + *timestamp_ptr = timestamp_usec; + + smartlist_add(timestamps_u64, timestamp_ptr); +} + +/** + * Peek at the head of a smartlist queue of u64 timestamps. + */ +static inline uint64_t +peek_timestamp(const smartlist_t *timestamps_u64_usecs) +{ + uint64_t *timestamp_ptr = smartlist_get(timestamps_u64_usecs, 0); + + if (BUG(!timestamp_ptr)) { + log_err(LD_CIRC, "Congestion control timestamp list became empty!"); + return 0; + } + + return *timestamp_ptr; +} + +/** + * Dequeue a u64 monotime usec timestamp from the front of a + * smartlist of pointers to 64. + */ +static inline uint64_t +dequeue_timestamp(smartlist_t *timestamps_u64_usecs) +{ + uint64_t *timestamp_ptr = smartlist_get(timestamps_u64_usecs, 0); + uint64_t timestamp_u64; + + if (BUG(!timestamp_ptr)) { + log_err(LD_CIRC, "Congestion control timestamp list became empty!"); + return 0; + } + + timestamp_u64 = *timestamp_ptr; + smartlist_del_keeporder(timestamps_u64_usecs, 0); + tor_free(timestamp_ptr); + + return timestamp_u64; +} + +/** + * Returns the number of sendme acks that will be recieved in the + * current congestion window size, rounded to nearest int. + */ +static inline uint64_t +sendme_acks_per_cwnd(const congestion_control_t *cc) +{ + /* We add half a sendme_inc to cwnd to round to the nearest int */ + return ((cc->cwnd + cc->sendme_inc/2)/cc->sendme_inc); +} + +/** + * Get a package window from either old sendme logic, or congestion control. + * + * A package window is how many cells you can still send. + */ +int +congestion_control_get_package_window(const circuit_t *circ, + const crypt_path_t *cpath) +{ + int package_window; + congestion_control_t *cc; + + tor_assert(circ); + + if (cpath) { + package_window = cpath->package_window; + cc = cpath->ccontrol; + } else { + package_window = circ->package_window; + cc = circ->ccontrol; + } + + if (!cc) { + return package_window; + } else { + /* Inflight can be above cwnd if cwnd was just reduced */ + if (cc->inflight > cc->cwnd) + return 0; + /* In the extremely unlikely event that cwnd-inflight is larger than + * INT32_MAX, just return that cap, so old code doesn't explode. */ + else if (cc->cwnd - cc->inflight > INT32_MAX) + return INT32_MAX; + else + return (int)(cc->cwnd - cc->inflight); + } +} + +/** + * Returns the number of cells that are acked by every sendme. + */ +int +sendme_get_inc_count(const circuit_t *circ, const crypt_path_t *layer_hint) +{ + int sendme_inc = CIRCWINDOW_INCREMENT; + congestion_control_t *cc = NULL; + + if (layer_hint) { + cc = layer_hint->ccontrol; + } else { + cc = circ->ccontrol; + } + + if (cc) { + sendme_inc = cc->sendme_inc; + } + + return sendme_inc; +} + +/** Return true iff the next cell we send will result in the other endpoint + * sending a SENDME. + * + * We are able to know that because the package or inflight window value minus + * one cell (the possible SENDME cell) should be a multiple of the + * cells-per-sendme increment value (set via consensus parameter, negotiated + * for the circuit, and passed in as sendme_inc). + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. + */ +bool +circuit_sent_cell_for_sendme(const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + congestion_control_t *cc; + int window; + + tor_assert(circ); + + if (layer_hint) { + window = layer_hint->package_window; + cc = layer_hint->ccontrol; + } else { + window = circ->package_window; + cc = circ->ccontrol; + } + + /* If we are using congestion control and the alg is not + * old-school 'fixed', then use cc->inflight to determine + * when sendmes will be sent */ + if (cc) { + if (!cc->inflight) + return false; + + /* This check must be +1 because this function is called *before* + * inflight is incremented for the sent cell */ + if ((cc->inflight+1) % cc->sendme_inc != 0) + return false; + + return true; + } + + /* At the start of the window, no SENDME will be expected. */ + if (window == CIRCWINDOW_START) { + return false; + } + + /* Are we at the limit of the increment and if not, we don't expect next + * cell is a SENDME. + * + * We test against the window minus 1 because when we are looking if the + * next cell is a SENDME, the window (either package or deliver) hasn't been + * decremented just yet so when this is called, we are currently processing + * the "window - 1" cell. + */ + if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + return false; + } + + /* Next cell is expected to be a SENDME. */ + return true; +} + +/** + * Call-in to tell congestion control code that this circuit sent a cell. + * + * This updates the 'inflight' counter, and if this is a cell that will + * cause the other end to send a SENDME, record the current time in a list + * of pending timestamps, so that we can later compute the circuit RTT when + * the SENDME comes back. */ +void +congestion_control_note_cell_sent(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *cpath) +{ + tor_assert(circ); + tor_assert(cc); + + /* Is this the last cell before a SENDME? The idea is that if the + * package_window reaches a multiple of the increment, after this cell, we + * should expect a SENDME. Note that this function must be called *before* + * we account for the sent cell. */ + if (!circuit_sent_cell_for_sendme(circ, cpath)) { + cc->inflight++; + return; + } + + cc->inflight++; + + /* Record this cell time for RTT computation when SENDME arrives */ + enqueue_timestamp(cc->sendme_pending_timestamps, + monotime_absolute_usec()); +} + +/** + * Returns true if any edge connections are active. + * + * We need to know this so that we can stop computing BDP if the + * edges are not sending on the circuit. + */ +static int +circuit_has_active_streams(const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + const edge_connection_t *streams; + + if (CIRCUIT_IS_ORIGIN(circ)) { + streams = CONST_TO_ORIGIN_CIRCUIT(circ)->p_streams; + } else { + streams = CONST_TO_OR_CIRCUIT(circ)->n_streams; + } + + /* Check linked list of streams */ + for (const edge_connection_t *conn = streams; conn != NULL; + conn = conn->next_stream) { + if (conn->base_.marked_for_close) + continue; + + if (!layer_hint || conn->cpath_layer == layer_hint) { + if (connection_get_inbuf_len(TO_CONN(conn)) > 0) { + log_info(LD_CIRC, "CC: More in edge inbuf..."); + return 1; + } + + /* If we did not reach EOF on this read, there's more */ + if (!TO_CONN(conn)->inbuf_reached_eof) { + log_info(LD_CIRC, "CC: More on edge conn..."); + return 1; + } + + if (TO_CONN(conn)->linked_conn) { + if (connection_get_inbuf_len(TO_CONN(conn)->linked_conn) > 0) { + log_info(LD_CIRC, "CC: More in linked inbuf..."); + return 1; + } + + /* If there is a linked conn, and *it* did not each EOF, + * there's more */ + if (!TO_CONN(conn)->linked_conn->inbuf_reached_eof) { + log_info(LD_CIRC, "CC: More on linked conn..."); + return 1; + } + } + } + } + + return 0; +} + +/** + * Upon receipt of a SENDME, pop the oldest timestamp off the timestamp + * list, and use this to update RTT. + * + * Returns true if circuit estimates were successfully updated, false + * otherwise. + */ +bool +congestion_control_update_circuit_estimates(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + uint64_t now_usec = monotime_absolute_usec(); + + /* Update RTT first, then BDP. BDP needs fresh RTT */ + uint64_t curr_rtt_usec = congestion_control_update_circuit_rtt(cc, now_usec); + return congestion_control_update_circuit_bdp(cc, circ, layer_hint, now_usec, + curr_rtt_usec); +} + +/** + * Returns true if we have enough time data to use heuristics + * to compare RTT to a baseline. + */ +static bool +time_delta_should_use_heuristics(const congestion_control_t *cc) +{ + + /* If we have exited slow start, we should have processed at least + * a cwnd worth of RTTs */ + if (!cc->in_slow_start) { + return true; + } + + /* If we managed to get enough acks to estimate a SENDME BDP, then + * we have enough to estimate clock jumps relative to a baseline, + * too. (This is at least 'cc_bwe_min' acks). */ + if (cc->bdp[BDP_ALG_SENDME_RATE]) { + return true; + } + + /* Not enough data to estimate clock jumps */ + return false; +} + +/** + * Returns true if the monotime delta is 0, or is significantly + * different than the previous delta. Either case indicates + * that the monotime time source stalled or jumped. + */ +static bool +time_delta_stalled_or_jumped(const congestion_control_t *cc, + uint64_t old_delta, uint64_t new_delta) +{ +#define DELTA_DISCREPENCY_RATIO_MAX 100 + /* If we have a 0 new_delta, that is definitely a monotime stall */ + if (new_delta == 0) { + static ratelim_t stall_info_limit = RATELIM_INIT(60); + log_fn_ratelim(&stall_info_limit, LOG_INFO, LD_CIRC, + "Congestion control cannot measure RTT due to monotime stall."); + return true; + } + + /* If the old_delta is 0, we have no previous values. So + * just assume this one is valid (beause it is non-zero) */ + if (old_delta == 0) + return false; + + /* + * For the heuristic cases, we need at least a few timestamps, + * to average out any previous partial stalls or jumps. So until + * than point, let's just delcare these time values "good enough + * to use". + */ + if (!time_delta_should_use_heuristics(cc)) { + return false; + } + + /* If old_delta is significantly larger than new_delta, then + * this means that the monotime clock recently stopped moving + * forward. */ + if (old_delta > new_delta * DELTA_DISCREPENCY_RATIO_MAX) { + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Sudden decrease in circuit RTT (%"PRIu64" vs %"PRIu64 + "), likely due to clock jump.", + new_delta/1000, old_delta/1000); + + return true; + } + + /* If new_delta is significantly larger than old_delta, then + * this means that the monotime clock suddenly jumped forward. */ + if (new_delta > old_delta * DELTA_DISCREPENCY_RATIO_MAX) { + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Sudden increase in circuit RTT (%"PRIu64" vs %"PRIu64 + "), likely due to clock jump.", + new_delta/1000, old_delta/1000); + + return true; + } + + return false; +} + +/** + * Called when we get a SENDME. Updates circuit RTT by pulling off a + * timestamp of when we sent the CIRCWINDOW_INCREMENT-th cell from + * the queue of such timestamps, and comparing that to current time. + * + * Also updates min, max, and EWMA of RTT. + * + * Returns the current circuit RTT in usecs, or 0 if it could not be + * measured (due to clock jump, stall, etc). + */ +static uint64_t +congestion_control_update_circuit_rtt(congestion_control_t *cc, + uint64_t now_usec) +{ + uint64_t rtt, ewma_cnt; + uint64_t sent_at_timestamp; + + tor_assert(cc); + + /* Get the time that we sent the cell that resulted in the other + * end sending this sendme. Use this to calculate RTT */ + sent_at_timestamp = dequeue_timestamp(cc->sendme_pending_timestamps); + + rtt = now_usec - sent_at_timestamp; + + /* Do not update RTT at all if it looks fishy */ + if (time_delta_stalled_or_jumped(cc, cc->ewma_rtt_usec, rtt)) { + return 0; + } + + ewma_cnt = cc->ewma_cwnd_cnt*sendme_acks_per_cwnd(cc); + ewma_cnt = MAX(ewma_cnt, 2); // Use at least 2 + + cc->ewma_rtt_usec = n_count_ewma(rtt, cc->ewma_rtt_usec, ewma_cnt); + + if (rtt > cc->max_rtt_usec) { + cc->max_rtt_usec = rtt; + } + + if (cc->min_rtt_usec == 0 || rtt < cc->min_rtt_usec) { + cc->min_rtt_usec = rtt; + } + + return rtt; +} + +/** + * Called when we get a SENDME. Updates the bandwidth-delay-product (BDP) + * estimates of a circuit. Several methods of computing BDP are used, + * depending on scenario. While some congestion control algorithms only + * use one of these methods, we update them all because it's quick and easy. + * + * - now_usec is the current monotime in usecs. + * - curr_rtt_usec is the current circuit RTT in usecs. It may be 0 if no + * RTT could bemeasured. + * + * Returns true if we were able to update BDP, false otherwise. + */ +static bool +congestion_control_update_circuit_bdp(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint, + uint64_t now_usec, + uint64_t curr_rtt_usec) +{ + int chan_q = 0; + unsigned int blocked_on_chan = 0; + uint64_t timestamp_usec; + uint64_t sendme_rate_bdp = 0; + + tor_assert(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* origin circs use n_chan */ + chan_q = circ->n_chan_cells.n; + blocked_on_chan = circ->streams_blocked_on_n_chan; + } else { + /* Both onion services and exits use or_circuit and p_chan */ + chan_q = CONST_TO_OR_CIRCUIT(circ)->p_chan_cells.n; + blocked_on_chan = circ->streams_blocked_on_p_chan; + } + + /* If we have no EWMA RTT, it is because monotime has been stalled + * or messed up the entire time so far. Set our BDP estimates directly + * to current cwnd */ + if (!cc->ewma_rtt_usec) { + uint64_t cwnd = cc->cwnd; + + /* If the channel is blocked, keep subtracting off the chan_q + * until we hit the min cwnd. */ + if (blocked_on_chan) { + cwnd = MAX(cwnd - chan_q, cc->cwnd_min); + cc->blocked_chan = 1; + } else { + cc->blocked_chan = 0; + } + + cc->bdp[BDP_ALG_CWND_RTT] = cwnd; + cc->bdp[BDP_ALG_INFLIGHT_RTT] = cwnd; + cc->bdp[BDP_ALG_SENDME_RATE] = cwnd; + cc->bdp[BDP_ALG_PIECEWISE] = cwnd; + + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Our clock has been stalled for the entire lifetime of a circuit. " + "Performance may be sub-optimal."); + + return blocked_on_chan; + } + + /* Congestion window based BDP will respond to changes in RTT only, and is + * relative to cwnd growth. It is useful for correcting for BDP + * overestimation, but if BDP is higher than the current cwnd, it will + * underestimate it. + * + * We multiply here first to avoid precision issues from min_RTT being + * close to ewma RTT. Since all fields are u64, there is plenty of + * room here to multiply first. + */ + cc->bdp[BDP_ALG_CWND_RTT] = cc->cwnd*cc->min_rtt_usec/cc->ewma_rtt_usec; + + /* + * If we have no pending streams, we do not have enough data to fill + * the BDP, so preserve our old estimates but do not make any more. + */ + if (!blocked_on_chan && !circuit_has_active_streams(circ, layer_hint)) { + log_info(LD_CIRC, + "CC: Streams drained. Spare package window: %"PRIu64 + ", no BDP update", cc->cwnd - cc->inflight); + + /* Clear SENDME timestamps; they will be wrong with intermittent data */ + SMARTLIST_FOREACH(cc->sendme_arrival_timestamps, uint64_t *, t, + tor_free(t)); + smartlist_clear(cc->sendme_arrival_timestamps); + } else if (curr_rtt_usec) { + /* Sendme-based BDP will quickly measure BDP in much less than + * a cwnd worth of data when in use (in 2-10 SENDMEs). + * + * But if the link goes idle, it will be vastly lower than true BDP. Hence + * we only compute it if we have either pending stream data, or streams + * are still blocked on the channel queued data. + * + * We also do not compute it if we do not have a current RTT passed in, + * because that means that monotime is currently stalled or just jumped. + */ + enqueue_timestamp(cc->sendme_arrival_timestamps, now_usec); + + if (smartlist_len(cc->sendme_arrival_timestamps) >= cc->bwe_sendme_min) { + /* If we have more sendmes than fit in a cwnd, trim the list. + * Those are not acurrately measuring throughput, if cwnd is + * currently smaller than BDP */ + while (smartlist_len(cc->sendme_arrival_timestamps) > + cc->bwe_sendme_min && + (uint64_t)smartlist_len(cc->sendme_arrival_timestamps) > + sendme_acks_per_cwnd(cc)) { + (void)dequeue_timestamp(cc->sendme_arrival_timestamps); + } + int sendme_cnt = smartlist_len(cc->sendme_arrival_timestamps); + + /* Calculate SENDME_BWE_COUNT pure average */ + timestamp_usec = peek_timestamp(cc->sendme_arrival_timestamps); + uint64_t delta = now_usec - timestamp_usec; + + /* The acked data is in sendme_cnt-1 chunks, because we are counting the + * data that is processed by the other endpoint *between* all of these + * sendmes. There's one less gap between the sendmes than the number + * of sendmes. */ + uint64_t cells = (sendme_cnt-1)*cc->sendme_inc; + + /* The bandwidth estimate is cells/delta, which when multiplied + * by min RTT obtains the BDP. However, we multiply first to + * avoid precision issues with the RTT being close to delta in size. */ + sendme_rate_bdp = cells*cc->min_rtt_usec/delta; + + /* Calculate BDP_EWMA_COUNT N-EWMA */ + cc->bdp[BDP_ALG_SENDME_RATE] = + n_count_ewma(sendme_rate_bdp, cc->bdp[BDP_ALG_SENDME_RATE], + cc->ewma_cwnd_cnt*sendme_acks_per_cwnd(cc)); + } + + /* In-flight BDP will cause the cwnd to drift down when underutilized. + * It is most useful when the local OR conn is blocked, so we only + * compute it if we're utilized. */ + cc->bdp[BDP_ALG_INFLIGHT_RTT] = + (cc->inflight - chan_q)*cc->min_rtt_usec/ + MAX(cc->ewma_rtt_usec, curr_rtt_usec); + } else { + /* We can still update inflight with just an EWMA RTT, but only + * if there is data flowing */ + cc->bdp[BDP_ALG_INFLIGHT_RTT] = + (cc->inflight - chan_q)*cc->min_rtt_usec/cc->ewma_rtt_usec; + } + + /* The orconn is blocked; use smaller of inflight vs SENDME */ + if (blocked_on_chan) { + log_info(LD_CIRC, "CC: Streams blocked on circ channel. Chanq: %d", + chan_q); + + /* A blocked channel is an immediate congestion signal, but it still + * happens only once per cwnd */ + if (!cc->blocked_chan) { + cc->next_cc_event = 0; + cc->blocked_chan = 1; + } + + if (cc->bdp[BDP_ALG_SENDME_RATE]) { + cc->bdp[BDP_ALG_PIECEWISE] = MIN(cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_SENDME_RATE]); + } else { + cc->bdp[BDP_ALG_PIECEWISE] = cc->bdp[BDP_ALG_INFLIGHT_RTT]; + } + } else { + /* If we were previously blocked, emit a new congestion event + * now that we are unblocked, to re-evaluate cwnd */ + if (cc->blocked_chan) { + cc->blocked_chan = 0; + cc->next_cc_event = 0; + log_info(LD_CIRC, "CC: Streams un-blocked on circ channel. Chanq: %d", + chan_q); + } + + cc->bdp[BDP_ALG_PIECEWISE] = MAX(cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_CWND_RTT]); + } + + /* We can end up with no piecewise value if we didn't have either + * a SENDME estimate or enough data for an inflight estimate. + * It also happens on the very first sendme, since we need two + * to get a BDP. In these cases, use the cwnd method. */ + if (!cc->bdp[BDP_ALG_PIECEWISE]) { + cc->bdp[BDP_ALG_PIECEWISE] = cc->bdp[BDP_ALG_CWND_RTT]; + log_info(LD_CIRC, "CC: No piecewise BDP. Using %"PRIu64, + cc->bdp[BDP_ALG_PIECEWISE]); + } + + if (cc->next_cc_event == 0) { + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: Circuit %d " + "SENDME RTT: %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", " + "BDP estimates: " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64". ", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->min_rtt_usec/1000, + curr_rtt_usec/1000, + cc->ewma_rtt_usec/1000, + cc->max_rtt_usec/1000, + cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_CWND_RTT], + sendme_rate_bdp, + cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_PIECEWISE] + ); + } else { + log_info(LD_CIRC, + "CC: Circuit %"PRIu64":%d " + "SENDME RTT: %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64". ", + // XXX: actually, is this p_chan here? This is + // an or_circuit (exit or onion) + circ->n_chan->global_identifier, circ->n_circ_id, + cc->min_rtt_usec/1000, + curr_rtt_usec/1000, + cc->ewma_rtt_usec/1000, + cc->max_rtt_usec/1000, + cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_CWND_RTT], + sendme_rate_bdp, + cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_PIECEWISE] + ); + } + } + + /* We updated BDP this round if either we had a blocked channel, or + * the curr_rtt_usec was not 0. */ + return (blocked_on_chan || curr_rtt_usec != 0); +} + +/** + * Dispatch the sendme to the appropriate congestion control algorithm. + */ +int +congestion_control_dispatch_cc_alg(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + switch (cc->cc_alg) { + case CC_ALG_WESTWOOD: + return congestion_control_westwood_process_sendme(cc, circ, layer_hint); + + case CC_ALG_VEGAS: + return congestion_control_vegas_process_sendme(cc, circ, layer_hint); + + case CC_ALG_NOLA: + return congestion_control_nola_process_sendme(cc, circ, layer_hint); + + case CC_ALG_SENDME: + default: + tor_assert(0); + } + + return -END_CIRC_REASON_INTERNAL; +} diff --git a/src/core/or/congestion_control_common.h b/src/core/or/congestion_control_common.h new file mode 100644 index 0000000000..4193d94cba --- /dev/null +++ b/src/core/or/congestion_control_common.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_common.h + * \brief Public APIs for congestion control + **/ + +#ifndef TOR_CONGESTION_CONTROL_COMMON_H +#define TOR_CONGESTION_CONTROL_COMMON_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +typedef struct congestion_control_t congestion_control_t; + +/** Wrapper for the free function, set the CC pointer to NULL after free */ +#define congestion_control_free(cc) \ + FREE_AND_NULL(congestion_control_t, congestion_control_free_, cc) + +void congestion_control_free_(congestion_control_t *cc); + +congestion_control_t *congestion_control_new(void); + +int congestion_control_dispatch_cc_alg(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); + +void congestion_control_note_cell_sent(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *cpath); + +bool congestion_control_update_circuit_estimates(congestion_control_t *, + const circuit_t *, + const crypt_path_t *); + +int congestion_control_get_package_window(const circuit_t *, + const crypt_path_t *); + +int sendme_get_inc_count(const circuit_t *, const crypt_path_t *); +bool circuit_sent_cell_for_sendme(const circuit_t *, const crypt_path_t *); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_COMMON_H) */ diff --git a/src/core/or/congestion_control_nola.c b/src/core/or/congestion_control_nola.c new file mode 100644 index 0000000000..09f88d4699 --- /dev/null +++ b/src/core/or/congestion_control_nola.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_nola.c + * \brief Code that implements the TOR_NOLA congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_NOLA_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_nola.h" +#include "core/or/circuituse.h" +#include "core/or/circuitlist.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define NOLA_BDP_OVERSHOOT 100 + +/** + * Cache NOLA consensus parameters. + */ +void +congestion_control_nola_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_NOLA); + + cc->nola_params.bdp_overshoot = + networkstatus_get_param(NULL, "cc_nola_overshoot", + NOLA_BDP_OVERSHOOT, + 0, + 1000); +} + +/** +* Process a SENDME and update the congestion window according to the +* rules specified in TOR_NOLA of Proposal #324. +* +* TOR_NOLA updates the congestion window to match the current +* BDP estimate, every sendme. Because this can result in downward +* drift, a fixed overhead is added to the BDP estimate. This will +* cause some queuing, but ensures that the algorithm always uses +* the full BDP. +* +* To handle the case where the local orconn blocks, TOR_NOLA uses +* the 'piecewise' BDP estimate, which uses more a conservative BDP +* estimate method when blocking occurrs, but a more aggressive BDP +* estimate when there is no local blocking. This minimizes local +* client queues. +*/ +int +congestion_control_nola_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + tor_assert(cc && cc->cc_alg == CC_ALG_NOLA); + tor_assert(circ); + + if (cc->next_cc_event) + cc->next_cc_event--; + + /* If we get a congestion event, the only thing NOLA + * does is note this as if we exited slow-start + * (which for NOLA just means we finished our ICW). */ + if (cc->next_cc_event == 0) + cc->in_slow_start = 0; + + /* If we did not successfully update BDP, we must return. Otherwise, + * NOLA can drift downwards */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We overshoot the BDP by the cwnd_inc param amount, because BDP + * may otherwise drift down. This helps us probe for more capacity. + * But there is no sense to do it if the local channel is blocked. */ + if (cc->blocked_chan) + cc->cwnd = cc->bdp[cc->bdp_alg]; + else + cc->cwnd = cc->bdp[cc->bdp_alg] + cc->nola_params.bdp_overshoot; + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC TOR_NOLA: Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC TOR_NOLA: Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->in_slow_start + ); + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_nola.h b/src/core/or/congestion_control_nola.h new file mode 100644 index 0000000000..9c7d6e0635 --- /dev/null +++ b/src/core/or/congestion_control_nola.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_nola.h + * \brief Private-ish APIs for the TOR_NOLA congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_NOLA_H +#define TOR_CONGESTION_CONTROL_NOLA_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_nola_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_nola_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_NOLA_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_NOLA_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_NOLA_H) */ diff --git a/src/core/or/congestion_control_st.h b/src/core/or/congestion_control_st.h new file mode 100644 index 0000000000..251ebd82e3 --- /dev/null +++ b/src/core/or/congestion_control_st.h @@ -0,0 +1,257 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_st.h + * \brief Structure definitions for congestion control. + **/ + +#ifndef CONGESTION_CONTROL_ST_H +#define CONGESTION_CONTROL_ST_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/** Signifies which sendme algorithm to use */ +typedef enum { + /** OG Tor fixed-sized circ and stream windows. It sucks, but it is important + * to make sure that the new algs can compete with the old garbage. */ + CC_ALG_SENDME = 0, + + /** + * Prop#324 TOR_WESTWOOD - Deliberately agressive. Westwood may not even + * converge to fairness in some cases because max RTT will also increase + * on congesgtion, which boosts the Westwood RTT congestion threshhold. So it + * can cause runaway queue bloat, which may or may not lead to a robot + * uprising... Ok that's Westworld, not Westwood. Still, we need to test + * Vegas and NOLA against something more agressive to ensure they do not + * starve in the presence of cheaters. We also need to make sure cheaters + * trigger the oomkiller in those cases. + */ + CC_ALG_WESTWOOD = 1, + + /** + * Prop#324 TOR_VEGAS - TCP Vegas-style BDP tracker. Because Vegas backs off + * whenever it detects queue delay, it can be beaten out by more agressive + * algs. However, in live network testing, it seems to do just fine against + * current SENDMEs. It outperforms Westwood and does not stall. */ + CC_ALG_VEGAS = 2, + + /** + * Prop#324: TOR_NOLA - NOLA looks the BDP right in the eye and uses it + * immediately as CWND. No slow start, no other congestion signals, no delay, + * no bullshit. Like TOR_VEGAS, it also uses agressive BDP estimates, to + * avoid out-competition. It seems a bit better throughput than Vegas, + * but its agressive BDP and rapid updates may lead to more queue latency. */ + CC_ALG_NOLA = 3, +} cc_alg_t; + +/* Total number of CC algs in cc_alg_t enum */ +#define NUM_CC_ALGS (CC_ALG_NOLA+1) + +/** Signifies how we estimate circuit BDP */ +typedef enum { + /* CWND-based BDP will respond to changes in RTT only, and is relative + * to cwnd growth. So in slow-start, this will under-estimate BDP */ + BDP_ALG_CWND_RTT = 0, + + /* Sendme-based BDP will quickly measure BDP in less than + * a cwnd worth of data when in use. So it should be good for slow-start. + * But if the link goes idle, it will be vastly lower than true BDP. Thus, + * this estimate gets reset when the cwnd is not fully utilized. */ + BDP_ALG_SENDME_RATE = 1, + + /* Inflight BDP is similar to the cwnd estimator, except it uses + * packets inflight minus local circuit queues instead of current cwnd. + * Because it is strictly less than or equal to the cwnd, it will cause + * the cwnd to drift downward. It is only used if the local OR connection + * is blocked. */ + BDP_ALG_INFLIGHT_RTT = 2, + + /* The Piecewise BDP estimator uses the CWND estimator before there + * are sufficient SENDMEs to calculate the SENDME estimator. At that + * point, it uses the SENDME estimator, unless the local OR connection + * becomes blocked. In that case, it switches to the inflight estimator. */ + BDP_ALG_PIECEWISE = 3, + +} bdp_alg_t; + +/** Total number of BDP algs in bdp_alg_t enum */ +#define NUM_BDP_ALGS (BDP_ALG_PIECEWISE+1) + +/** Westwood algorithm parameters */ +struct westwood_params_t { + /** Cwnd backoff multiplier upon congestion (as percent) */ + uint8_t cwnd_backoff_m; + /** Max RTT backoff multiplier upon congestion (as percent) */ + uint8_t rtt_backoff_m; + + /** Threshold between min and max RTT, to signal congestion (percent) */ + uint8_t rtt_thresh; + + /** + * If true, use minimum of BDP and backoff multiplication in backoff. + * If false, use maximum of BDP and backoff multiplication in backoff. */ + bool min_backoff; +}; + +/** Vegas algorithm parameters. */ +struct vegas_params_t { + /** The queue use allowed before we exit slow start */ + uint16_t gamma; + /** The queue use below which we increment cwnd */ + uint16_t alpha; + /** The queue use above which we decrement cwnd */ + uint16_t beta; + /** Weighted average (percent) between cwnd estimator and + * piecewise estimator. */ + uint8_t bdp_mix_pct; +}; + +/** NOLA consensus params */ +struct nola_params_t { + /** How many cells to add to BDP estimate to obtain cwnd */ + uint16_t bdp_overshoot; +}; + +/** Fields common to all congestion control algorithms */ +typedef struct congestion_control_t { + /** + * Smartlist of uint64_t monotime usec timestamps of when we sent a data + * cell that is pending a sendme. FIFO queue that is managed similar to + * sendme_last_digests. */ + smartlist_t *sendme_pending_timestamps; + + /** + * Smartlist of uint64_t monotime timestamp of when sendme's arrived. + * FIFO queue that is managed similar to sendme_last_digests. + * Used to estimate circuitbandwidth and BDP. */ + smartlist_t *sendme_arrival_timestamps; + + /** RTT time data for congestion control. */ + uint64_t ewma_rtt_usec; + uint64_t min_rtt_usec; + uint64_t max_rtt_usec; + + /* BDP estimates by algorithm */ + uint64_t bdp[NUM_BDP_ALGS]; + + /** Congestion window */ + uint64_t cwnd; + + /** Number of cells in-flight (sent but awaiting SENDME ack). */ + uint64_t inflight; + + /** + * For steady-state: the number of sendme acks until we will acknowledge + * a congestion event again. It starts out as the number of sendme acks + * in a congestion windowm and is decremented each ack. When this reaches + * 0, it means we should examine our congestion algorithm conditions. + * In this way, we only react to one congestion event per congestion window. + * + * It is also reset to 0 immediately whenever the circuit's orconn is + * blocked, and when a previously blocked orconn is unblocked. + */ + uint64_t next_cc_event; + + /** Are we in slow start? */ + bool in_slow_start; + + /** Is the local channel blocked on us? That's a congestion signal */ + bool blocked_chan; + + /* The following parameters are cached from consensus values upon + * circuit setup. */ + + /** Percent of cwnd to increment by during slow start */ + uint16_t cwnd_inc_pct_ss; + + /** Number of cells to increment cwnd by during steady state */ + uint16_t cwnd_inc; + + /** Minimum congestion window (must be at least sendme_inc) */ + uint16_t cwnd_min; + + /** + * Number of times per congestion window to update based on congestion + * signals */ + uint8_t cwnd_inc_rate; + + /** + * Number of cwnd worth of sendme acks to smooth RTT and BDP with, + * using N_EWMA */ + uint8_t ewma_cwnd_cnt; + + /** + * Minimum number of sendmes before we begin BDP estimates + */ + uint8_t bwe_sendme_min; + + /** + * Number of cells to ack with every sendme. Taken from consensus parameter + * and negotiation during circuit setup. */ + uint8_t sendme_inc; + + /** Which congestion control algorithm to use. Taken from + * consensus parameter and negotiation during circuit setup. */ + cc_alg_t cc_alg; + + /** Which algorithm to estimate circuit bandwidth with. Taken from + * consensus parameter during circuit setup. */ + bdp_alg_t bdp_alg; + + /** Algorithm-specific parameters. The specific struct that is used + * depends upon the algoritghm selected by the cc_alg parameter. + * These should not be accessed anywhere other than the algorithm-specific + * files. */ + union { + struct westwood_params_t westwood_params; + struct vegas_params_t vegas_params; + struct nola_params_t nola_params; + }; +} congestion_control_t; + +/** + * Returns the number of sendme acks we will recieve before we update cwnd. + * + * Congestion control literature recommends only one update of cwnd per + * cwnd worth of acks. However, we can also tune this to be more frequent + * by increasing the 'cc_cwnd_inc_rate' consensus parameter. + * + * If this returns 0 due to high cwnd_inc_rate, the calling code will + * update every sendme ack. + */ +static inline uint64_t CWND_UPDATE_RATE(const congestion_control_t *cc) +{ + /* We add cwnd_inc_rate*sendme_inc/2 to round to nearest integer number + * of acks */ + return ((cc->cwnd + cc->cwnd_inc_rate*cc->sendme_inc/2) + / (cc->cwnd_inc_rate*cc->sendme_inc)); +} + +/** + * Returns the amount to increment the congestion window each update, + * during slow start. + * + * Congestion control literature recommends either doubling the cwnd + * every cwnd during slow start, or some similar exponential growth + * (such as 50% more every cwnd, for Vegas). + * + * This is controlled by a consensus parameter 'cwnd_inc_pct_ss', which + * allows us to specify the percent of the current consensus window + * to update by. + */ +static inline uint64_t CWND_INC_SS(const congestion_control_t *cc) +{ + return (cc->cwnd_inc_pct_ss*cc->cwnd/100); +} + +/** + * Returns the amount to increment (and for Vegas, also decrement) the + * congestion window by, every update period. + * + * This is controlled by the cc_cwnd_inc consensus parameter. + */ +#define CWND_INC(cc) ((cc)->cwnd_inc) + +#endif /* !defined(CONGESTION_CONTROL_ST_H) */ diff --git a/src/core/or/congestion_control_vegas.c b/src/core/or/congestion_control_vegas.c new file mode 100644 index 0000000000..3206821f4c --- /dev/null +++ b/src/core/or/congestion_control_vegas.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_vegas.c + * \brief Code that implements the TOR_VEGAS congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_VEGAS_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_vegas.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define VEGAS_GAMMA(cc) (6*(cc)->sendme_inc) +#define VEGAS_ALPHA(cc) (3*(cc)->sendme_inc) +#define VEGAS_BETA(cc) (6*(cc)->sendme_inc) + +#define VEGAS_BDP_MIX_PCT 0 + +/** + * The original TCP Vegas used only a congestion window BDP estimator. We + * believe that the piecewise estimator is likely to perform better, but + * for purposes of experimentation, we might as well have a way to blend + * them. It also lets us set Vegas to its original estimator while other + * algorithms on the same network use piecewise (by setting the + * 'vegas_bdp_mix_pct' consensus parameter to 100, while leaving the + * 'cc_bdp_alg' parameter set to piecewise). + * + * Returns a percentage weighted average between the CWND estimator and + * the specified consensus BDP estimator. + */ +static inline uint64_t +vegas_bdp_mix(const congestion_control_t *cc) +{ + return cc->vegas_params.bdp_mix_pct*cc->bdp[BDP_ALG_CWND_RTT]/100 + + (100-cc->vegas_params.bdp_mix_pct)*cc->bdp[cc->bdp_alg]/100; +} + +/** + * Cache Vegas consensus parameters. + */ +void +congestion_control_vegas_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_VEGAS); + + cc->vegas_params.gamma = + networkstatus_get_param(NULL, "cc_vegas_gamma", + VEGAS_GAMMA(cc), + 0, + 1000); + + cc->vegas_params.alpha = + networkstatus_get_param(NULL, "cc_vegas_alpha", + VEGAS_ALPHA(cc), + 0, + 1000); + + cc->vegas_params.beta = + networkstatus_get_param(NULL, "cc_vegas_beta", + VEGAS_BETA(cc), + 0, + 1000); + + cc->vegas_params.bdp_mix_pct = + networkstatus_get_param(NULL, "cc_vegas_bdp_mix", + VEGAS_BDP_MIX_PCT, + 0, + 100); +} + +/** + * Process a SENDME and update the congestion window according to the + * rules specified in TOR_VEGAS of Proposal #324. + * + * Essentially, this algorithm attempts to measure queue lengths on + * the circuit by subtracting the bandwidth-delay-product estimate + * from the current congestion window. + * + * If the congestion window is larger than the bandwidth-delay-product, + * then data is assumed to be queuing. We reduce the congestion window + * in that case. + * + * If the congestion window is smaller than the bandwidth-delay-product, + * then there is spare bandwidth capacity on the circuit. We increase the + * congestion window in that case. + * + * The congestion window is updated only once every congestion window worth of + * packets, even if the signal persists. It is also updated whenever the + * upstream orcon blocks, or unblocks. This minimizes local client queues. + */ +int +congestion_control_vegas_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + uint64_t queue_use; + + tor_assert(cc && cc->cc_alg == CC_ALG_VEGAS); + tor_assert(circ); + + /* Update ack counter until next congestion signal event is allowed */ + if (cc->next_cc_event) + cc->next_cc_event--; + + /* Compute BDP and RTT. If we did not update, don't run the alg */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We only update anything once per window */ + if (cc->next_cc_event == 0) { + /* The queue use is the amount in which our cwnd is above BDP; + * if it is below, then 0 queue use. */ + if (vegas_bdp_mix(cc) > cc->cwnd) + queue_use = 0; + else + queue_use = cc->cwnd - vegas_bdp_mix(cc); + + if (cc->in_slow_start) { + if (queue_use < cc->vegas_params.gamma && !cc->blocked_chan) { + /* Grow to BDP immediately, then exponential growth until + * congestion signal */ + cc->cwnd = MAX(cc->cwnd + CWND_INC_SS(cc), + vegas_bdp_mix(cc)); + } else { + /* Congestion signal: Fall back to Vegas equilibrium (BDP) */ + cc->cwnd = vegas_bdp_mix(cc); + cc->in_slow_start = 0; + log_info(LD_CIRC, "CC: TOR_VEGAS exiting slow start"); + } + } else { + if (queue_use > cc->vegas_params.beta || cc->blocked_chan) { + cc->cwnd -= CWND_INC(cc); + } else if (queue_use < cc->vegas_params.alpha) { + cc->cwnd += CWND_INC(cc); + } + } + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + /* Schedule next update */ + cc->next_cc_event = CWND_UPDATE_RATE(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: TOR_VEGAS Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "VBDP: %"PRIu64", " + "QUSE: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + vegas_bdp_mix(cc), + queue_use, + cc->next_cc_event, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC: TOR_VEGAS Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "VBDP: %"PRIu64", " + "QUSE: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + vegas_bdp_mix(cc), + queue_use, + cc->next_cc_event, + cc->in_slow_start + ); + } + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_vegas.h b/src/core/or/congestion_control_vegas.h new file mode 100644 index 0000000000..111345081c --- /dev/null +++ b/src/core/or/congestion_control_vegas.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_vegas.h + * \brief Private-ish APIs for the TOR_VEGAS congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_VEGAS_H +#define TOR_CONGESTION_CONTROL_VEGAS_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_vegas_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_vegas_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_VEGAS_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_VEGAS_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_VEGAS_H) */ diff --git a/src/core/or/congestion_control_westwood.c b/src/core/or/congestion_control_westwood.c new file mode 100644 index 0000000000..4b24234212 --- /dev/null +++ b/src/core/or/congestion_control_westwood.c @@ -0,0 +1,231 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_westwood.c + * \brief Code that implements the TOR_WESTWOOD congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_westwood.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define USEC_ONE_MS (1000) + +#define WESTWOOD_CWND_BACKOFF_M 75 +#define WESTWOOD_RTT_BACKOFF_M 100 +#define WESTWOOD_RTT_THRESH 33 +#define WESTWOOD_MIN_BACKOFF 0 + +/** + * Cache westwood consensus parameters. + */ +void +congestion_control_westwood_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_WESTWOOD); + + cc->westwood_params.cwnd_backoff_m = + networkstatus_get_param(NULL, "cc_westwood_cwnd_m", + WESTWOOD_CWND_BACKOFF_M, + 0, + 100); + + cc->westwood_params.rtt_backoff_m = + networkstatus_get_param(NULL, "cc_westwood_rtt_m", + WESTWOOD_RTT_BACKOFF_M, + 50, + 100); + + cc->westwood_params.rtt_thresh = + networkstatus_get_param(NULL, "cc_westwood_rtt_thresh", + WESTWOOD_RTT_THRESH, + 0, + 100); + + cc->westwood_params.min_backoff = + networkstatus_get_param(NULL, "cc_westwood_min_backoff", + WESTWOOD_MIN_BACKOFF, + 0, + 1); +} + +/** + * Return the RTT threshhold that signals congestion. + * + * Computed from the threshold parameter that specifies a + * percent between the min and max RTT obseved so far. + */ +static inline uint64_t +westwood_rtt_signal(const congestion_control_t *cc) +{ + return ((100 - cc->westwood_params.rtt_thresh)*cc->min_rtt_usec + + cc->westwood_params.rtt_thresh*(cc)->max_rtt_usec)/100; +} + +/** + * Compute a backoff to reduce the max RTT. + * + * This may be necessary to ensure that westwood does not have + * a runaway condition where congestion inflates the max RTT, which + * inflates the congestion threshold. That cannot happen with one + * Westwood instance, but it may happen in aggregate. Hence, this is + * a safety parameter, in case we need it. + */ +static inline uint64_t +westwood_rtt_max_backoff(const congestion_control_t *cc) +{ + return cc->min_rtt_usec + + (cc->westwood_params.rtt_backoff_m * + (cc->max_rtt_usec - cc->min_rtt_usec))/100; +} + +/** + * Returns true if the circuit is experiencing congestion, as per + * TOR_WESTWOOD rules. + */ +static inline bool +westwood_is_congested(const congestion_control_t *cc) +{ + /* If the local channel is blocked, that is always congestion */ + if (cc->blocked_chan) + return true; + + /* If the min RTT is within 1ms of the signal, then there is not enough + * range in RTTs to signify congestion. Treat that as not congested. */ + if (westwood_rtt_signal(cc) < cc->min_rtt_usec || + westwood_rtt_signal(cc) - cc->min_rtt_usec < USEC_ONE_MS) + return false; + + /* If the EWMA-smoothed RTT exceeds the westwood RTT threshhold, + * then it is congestion. */ + if (cc->ewma_rtt_usec > westwood_rtt_signal(cc)) + return true; + + return false; +} + +/** + * Process a SENDME and update the congestion window according to the + * rules specified in TOR_WESTWOOD of Proposal #324. + * + * Essentially, this algorithm uses a threshhold of 'rtt_thresh', which + * is a midpoint between the min and max RTT. If the RTT exceeds this + * threshhold, then queue delay due to congestion is assumed to be present, + * and the algirithm reduces the congestion window. If the RTT is below the + * threshhold, the circuit is not congested (ie: queue delay is low), and we + * increase the congestion window. + * + * The congestion window is updated only once every congestion window worth of + * packets, even if the signal persists. It is also updated whenever the + * upstream orcon blocks, or unblocks. This minimizes local client queues. + */ +int +congestion_control_westwood_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + tor_assert(cc && cc->cc_alg == CC_ALG_WESTWOOD); + tor_assert(circ); + + /* Update ack counter until next congestion signal event is allowed */ + if (cc->next_cc_event) + cc->next_cc_event--; + + /* If we were unable to update our circuit estimates, Westwood must + * *not* update its cwnd, otherwise it could run to infinity, or to 0. + * Just update inflight from the sendme and return. */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We only update anything once per window */ + if (cc->next_cc_event == 0) { + if (!westwood_is_congested(cc)) { + if (cc->in_slow_start) { + cc->cwnd = MAX(cc->cwnd + CWND_INC_SS(cc), + cc->bdp[cc->bdp_alg]); + } else { + cc->cwnd = cc->cwnd + CWND_INC(cc); + } + } else { + if (cc->westwood_params.min_backoff) + cc->cwnd = MIN(cc->cwnd*cc->westwood_params.cwnd_backoff_m/100, + cc->bdp[cc->bdp_alg]); + else + cc->cwnd = MAX(cc->cwnd*cc->westwood_params.cwnd_backoff_m/100, + cc->bdp[cc->bdp_alg]); + + cc->in_slow_start = 0; + + // Because Westwood's congestion can runaway and boost max rtt, + // which increases its congestion signal, we backoff the max rtt + // too. + cc->max_rtt_usec = westwood_rtt_max_backoff(cc); + + log_info(LD_CIRC, "CC: TOR_WESTWOOD congestion. New max RTT: %"PRIu64, + cc->max_rtt_usec/1000); + } + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + /* Schedule next update */ + cc->next_cc_event = CWND_UPDATE_RATE(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: TOR_WESTWOOD Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "WRTT: %"PRIu64", " + "WSIG: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->ewma_rtt_usec/1000, + westwood_rtt_signal(cc)/1000, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC: TOR_WESTWOOD Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "WRTT: %"PRIu64", " + "WSIG: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->ewma_rtt_usec/1000, + westwood_rtt_signal(cc)/1000, + cc->in_slow_start + ); + } + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_westwood.h b/src/core/or/congestion_control_westwood.h new file mode 100644 index 0000000000..c6fd596df4 --- /dev/null +++ b/src/core/or/congestion_control_westwood.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_westwood.h + * \brief Private-ish APIs for the TOR_WESTWOOD congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_WESTWOOD_H +#define TOR_CONGESTION_CONTROL_WESTWOOD_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_westwood_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_westwood_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_WESTWOOD_H) */ diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 29356d7c2a..7673bc306f 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -27,6 +27,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/extendinfo.h" +#include "core/or/congestion_control_common.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_util.h" @@ -165,6 +166,7 @@ cpath_free(crypt_path_t *victim) onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); + congestion_control_free(victim->ccontrol); memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 2529b6ee41..ddc85eec14 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -29,6 +29,8 @@ struct onion_handshake_state_t { } u; }; +struct congestion_control_t; + /** Macro to encapsulate private members of a struct. * * Renames 'x' to 'x_crypt_path_private_field'. @@ -80,6 +82,9 @@ struct crypt_path_t { int deliver_window; /**< How many cells are we willing to deliver originating * at this step? */ + /** Congestion control info */ + struct congestion_control_t *ccontrol; + /*********************** Private members ****************************/ /** Private member: Cryptographic state used for encrypting and diff --git a/src/core/or/include.am b/src/core/or/include.am index b578b75673..d142062216 100644 --- a/src/core/or/include.am +++ b/src/core/or/include.am @@ -35,6 +35,10 @@ LIBTOR_APP_A_SOURCES += \ src/core/or/scheduler_kist.c \ src/core/or/scheduler_vanilla.c \ src/core/or/sendme.c \ + src/core/or/congestion_control_common.c \ + src/core/or/congestion_control_vegas.c \ + src/core/or/congestion_control_nola.c \ + src/core/or/congestion_control_westwood.c \ src/core/or/status.c \ src/core/or/versions.c @@ -57,6 +61,7 @@ noinst_HEADERS += \ src/core/or/circuitpadding_machines.h \ src/core/or/circuituse.h \ src/core/or/command.h \ + src/core/or/congestion_control_st.h \ src/core/or/connection_edge.h \ src/core/or/connection_or.h \ src/core/or/connection_st.h \ @@ -97,6 +102,10 @@ noinst_HEADERS += \ src/core/or/relay_crypto_st.h \ src/core/or/scheduler.h \ src/core/or/sendme.h \ + src/core/or/congestion_control_common.h \ + src/core/or/congestion_control_vegas.h \ + src/core/or/congestion_control_nola.h \ + src/core/or/congestion_control_westwood.h \ src/core/or/server_port_cfg_st.h \ src/core/or/socks_request_st.h \ src/core/or/status.h \ diff --git a/src/core/or/relay.c b/src/core/or/relay.c index f5a9e73856..e3d41d7bf0 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -97,6 +97,7 @@ #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" #include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -1574,6 +1575,7 @@ process_sendme_cell(const relay_header_t *rh, const cell_t *cell, } /* Stream level SENDME cell. */ + // TODO: Turn this off for cc_alg=1,2,3; use XON/XOFF instead ret = sendme_process_stream_level(conn, circ, rh->length); if (ret < 0) { /* Means we need to close the circuit with reason ret. */ @@ -2091,6 +2093,7 @@ void circuit_reset_sendme_randomness(circuit_t *circ) { circ->have_sent_sufficiently_random_cell = 0; + // XXX: do we need to change this check for congestion control? circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 + crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2); } @@ -2350,7 +2353,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, /* How many cells do we have space for? It will be the minimum of * the number needed to exhaust the package window, and the minimum * needed to fill the cell queue. */ - max_to_package = circ->package_window; + + max_to_package = congestion_control_get_package_window(circ, layer_hint); if (CIRCUIT_IS_ORIGIN(circ)) { cells_on_queue = circ->n_chan_cells.n; } else { @@ -2495,7 +2499,7 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); log_debug(domain,"considering circ->package_window %d", circ->package_window); - if (circ->package_window <= 0) { + if (congestion_control_get_package_window(circ, layer_hint) <= 0) { log_debug(domain,"yes, not-at-origin. stopped."); for (conn = or_circ->n_streams; conn; conn=conn->next_stream) connection_stop_reading(TO_CONN(conn)); @@ -2506,7 +2510,7 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) /* else, layer hint is defined, use it */ log_debug(domain,"considering layer_hint->package_window %d", layer_hint->package_window); - if (layer_hint->package_window <= 0) { + if (congestion_control_get_package_window(circ, layer_hint) <= 0) { log_debug(domain,"yes, at-origin. stopped."); for (conn = TO_ORIGIN_CIRCUIT(circ)->p_streams; conn; conn=conn->next_stream) { @@ -2722,11 +2726,18 @@ cell_queues_get_total_allocation(void) /** The time at which we were last low on memory. */ static time_t last_time_under_memory_pressure = 0; +/** Statistics on how many bytes were removed by the OOM per type. */ +uint64_t oom_stats_n_bytes_removed_dns = 0; +uint64_t oom_stats_n_bytes_removed_cell = 0; +uint64_t oom_stats_n_bytes_removed_geoip = 0; +uint64_t oom_stats_n_bytes_removed_hsdir = 0; + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ STATIC int cell_queues_check_size(void) { + size_t removed = 0; time_t now = time(NULL); size_t alloc = cell_queues_get_total_allocation(); alloc += half_streams_get_total_allocation(); @@ -2751,20 +2762,27 @@ cell_queues_check_size(void) if (hs_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = hs_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= hs_cache_handle_oom(now, bytes_to_remove); + removed = hs_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_hsdir += removed; + alloc -= removed; } if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = geoip_client_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= geoip_client_cache_handle_oom(now, bytes_to_remove); + removed = geoip_client_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_geoip += removed; + alloc -= removed; } if (dns_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = dns_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= dns_cache_handle_oom(now, bytes_to_remove); + removed = dns_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_dns += removed; + alloc -= removed; } - circuits_handle_oom(alloc); + removed = circuits_handle_oom(alloc); + oom_stats_n_bytes_removed_cell += removed; return 1; } } diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 2f337d5d16..eac920f491 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -49,6 +49,11 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; +extern uint64_t oom_stats_n_bytes_removed_dns; +extern uint64_t oom_stats_n_bytes_removed_cell; +extern uint64_t oom_stats_n_bytes_removed_geoip; +extern uint64_t oom_stats_n_bytes_removed_hsdir; + void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index ce3385ae98..900490a892 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -21,6 +21,7 @@ #include "core/or/or_circuit_st.h" #include "core/or/relay.h" #include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" #include "feature/nodelist/networkstatus.h" #include "lib/ctime/di_ops.h" #include "trunnel/sendme_cell.h" @@ -64,13 +65,6 @@ pop_first_cell_digest(const circuit_t *circ) return NULL; } - /* More cell digest than the SENDME window is never suppose to happen. The - * cell should have been rejected before reaching this point due to its - * package_window down to 0 leading to a circuit close. Scream loudly but - * still pop the element so we don't memory leak. */ - tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <= - CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT); - circ_digest = smartlist_get(circ->sendme_last_digests, 0); smartlist_del_keeporder(circ->sendme_last_digests, 0); return circ_digest; @@ -334,17 +328,18 @@ record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest) /** Return true iff the next cell for the given cell window is expected to be * a SENDME. * - * We are able to know that because the package or deliver window value minus - * one cell (the possible SENDME cell) should be a multiple of the increment - * window value. */ + * We are able to know that because the package or inflight window value minus + * one cell (the possible SENDME cell) should be a multiple of the + * cells-per-sendme increment value (set via consensus parameter, negotiated + * for the circuit, and passed in as sendme_inc). + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. + */ static bool -circuit_sendme_cell_is_next(int window) +circuit_sendme_cell_is_next(int window, int sendme_inc) { - /* At the start of the window, no SENDME will be expected. */ - if (window == CIRCWINDOW_START) { - return false; - } - /* Are we at the limit of the increment and if not, we don't expect next * cell is a SENDME. * @@ -352,11 +347,8 @@ circuit_sendme_cell_is_next(int window) * next cell is a SENDME, the window (either package or deliver) hasn't been * decremented just yet so when this is called, we are currently processing * the "window - 1" cell. - * - * This function is used when recording a cell digest and this is done quite - * low in the stack when decrypting or encrypting a cell. The window is only - * updated once the cell is actually put in the outbuf. */ - if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + */ + if (((window - 1) % sendme_inc) != 0) { return false; } @@ -419,15 +411,16 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) { bool sent_one_sendme = false; const uint8_t *digest; + int sendme_inc = sendme_get_inc_count(circ, layer_hint); while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= - CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { + CIRCWINDOW_START - sendme_inc) { log_debug(LD_CIRC,"Queuing circuit sendme."); if (layer_hint) { - layer_hint->deliver_window += CIRCWINDOW_INCREMENT; + layer_hint->deliver_window += sendme_inc; digest = cpath_get_sendme_digest(layer_hint); } else { - circ->deliver_window += CIRCWINDOW_INCREMENT; + circ->deliver_window += sendme_inc; digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); } if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { @@ -448,6 +441,9 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) * the length of the SENDME cell payload (excluding the header). The * cell_payload is the payload. * + * This function validates the SENDME's digest, and then dispatches to + * the appropriate congestion control algorithm in use on the circuit. + * * Return 0 on success (the SENDME is valid and the package window has * been updated properly). * @@ -460,6 +456,7 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, { tor_assert(circ); tor_assert(cell_payload); + congestion_control_t *cc; /* Validate the SENDME cell. Depending on the version, different validation * can be done. An invalid SENDME requires us to close the circuit. */ @@ -467,6 +464,34 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, return -END_CIRC_REASON_TORPROTOCOL; } + // Get CC + if (layer_hint) { + cc = layer_hint->ccontrol; + + /* origin circuits need to count valid sendmes as valid protocol data */ + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); + } else { + cc = circ->ccontrol; + } + + /* If there is no CC object, assume fixed alg */ + if (!cc) { + return sendme_process_circuit_level_impl(layer_hint, circ); + } + + return congestion_control_dispatch_cc_alg(cc, circ, layer_hint); +} + +/** + * Process a SENDME for Tor's original fixed window circuit-level flow control. + * Updates the package_window and ensures that it does not exceed the max. + * + * Returns -END_CIRC_REASON_TORPROTOCOL if the max is exceeded, otherwise + * returns 0. + */ +int +sendme_process_circuit_level_impl(crypt_path_t *layer_hint, circuit_t *circ) +{ /* If we are the origin of the circuit, we are the Client so we use the * layer hint (the Exit hop) for the package window tracking. */ if (CIRCUIT_IS_ORIGIN(circ)) { @@ -486,10 +511,6 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, layer_hint->package_window += CIRCWINDOW_INCREMENT; log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.", layer_hint->package_window); - - /* We count circuit-level sendme's as valid delivered data because they - * are rate limited. */ - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); } else { /* We aren't the origin of this circuit so we are the Exit and thus we * track the package window with the circuit object. */ @@ -592,25 +613,39 @@ int sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) { int package_window, domain; + congestion_control_t *cc; tor_assert(circ); - if (CIRCUIT_IS_ORIGIN(circ)) { - /* Client side. */ - tor_assert(layer_hint); - --layer_hint->package_window; - package_window = layer_hint->package_window; + if (layer_hint) { + cc = layer_hint->ccontrol; domain = LD_APP; } else { - /* Exit side. */ - tor_assert(!layer_hint); - --circ->package_window; - package_window = circ->package_window; + cc = circ->ccontrol; domain = LD_EXIT; } - log_debug(domain, "Circuit package_window now %d.", package_window); - return package_window; + if (cc) { + congestion_control_note_cell_sent(cc, circ, layer_hint); + } else { + /* Fixed alg uses package_window and must update it */ + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* Client side. */ + tor_assert(layer_hint); + --layer_hint->package_window; + package_window = layer_hint->package_window; + } else { + /* Exit side. */ + tor_assert(!layer_hint); + --circ->package_window; + package_window = circ->package_window; + } + log_debug(domain, "Circuit package_window now %d.", package_window); + } + + /* Return appropriate number designating how many cells can still be sent */ + return congestion_control_get_package_window(circ, layer_hint); } /* Called when a relay DATA cell is packaged for the given edge connection @@ -631,20 +666,14 @@ sendme_note_stream_data_packaged(edge_connection_t *conn) void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath) { - int package_window; uint8_t *sendme_digest; tor_assert(circ); - package_window = circ->package_window; - if (cpath) { - package_window = cpath->package_window; - } - /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (!circuit_sendme_cell_is_next(package_window)) { + if (!circuit_sent_cell_for_sendme(circ, cpath)) { return; } @@ -670,7 +699,8 @@ sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath) /* Only record if the next cell is expected to be a SENDME. */ if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window : - circ->deliver_window)) { + circ->deliver_window, + sendme_get_inc_count(circ, cpath))) { return; } @@ -692,8 +722,7 @@ sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath) tor_assert(circ); /* Only record if the next cell is expected to be a SENDME. */ - if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window : - circ->package_window)) { + if (!circuit_sent_cell_for_sendme(circ, cpath)) { goto end; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index a008940905..c224d0a921 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -22,6 +22,7 @@ void sendme_circuit_consider_sending(circuit_t *circ, int sendme_process_circuit_level(crypt_path_t *layer_hint, circuit_t *circ, const uint8_t *cell_payload, uint16_t cell_payload_len); +int sendme_process_circuit_level_impl(crypt_path_t *, circuit_t *); int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, uint16_t cell_body_len); diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 502cb99690..9583a4db71 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -3930,6 +3930,253 @@ guard_selection_free_(guard_selection_t *gs) tor_free(gs); } +/**********************************************************************/ + +/** Layer2 guard subsystem (vanguards-lite) used for onion service circuits */ + +/** A simple representation of a layer2 guard. We just need its identity so + * that we feed it into a routerset, and a sampled timestamp to do expiration + * checks. */ +typedef struct layer2_guard_t { + /** Identity of the guard */ + char identity[DIGEST_LEN]; + /** When does this guard expire? (randomized timestamp) */ + time_t expire_on_date; +} layer2_guard_t; + +#define layer2_guard_free(val) \ + FREE_AND_NULL(layer2_guard_t, layer2_guard_free_, (val)) + +/** Return true if the vanguards-lite subsystem is enabled */ +bool +vanguards_lite_is_enabled(void) +{ + /* First check torrc option and then maybe also the consensus parameter. */ + const or_options_t *options = get_options(); + + /* If the option is explicitly disabled, that's the final word here */ + if (options->VanguardsLiteEnabled == 0) { + return false; + } + + /* If the option is set to auto, then check the consensus parameter */ + if (options->VanguardsLiteEnabled == -1) { + return networkstatus_get_param(NULL, "vanguards-lite-enabled", + 1, /* default to "on" */ + 0, 1); + } + + /* else it's enabled */ + tor_assert_nonfatal(options->VanguardsLiteEnabled == 1); + return options->VanguardsLiteEnabled; +} + +static void +layer2_guard_free_(layer2_guard_t *l2) +{ + if (!l2) { + return; + } + + tor_free(l2); +} + +/** Global list and routerset of L2 guards. They are both synced and they get + * updated periodically. We need both the list and the routerset: we use the + * smartlist to keep track of expiration times and the routerset is what we + * return to the users of this subsystem. */ +static smartlist_t *layer2_guards = NULL; +static routerset_t *layer2_routerset = NULL; + +/** Number of L2 guards */ +#define NUMBER_SECOND_GUARDS 4 +/** Make sure that the number of L2 guards is less than the number of + * MAX_SANE_RESTRICTED_NODES */ +CTASSERT(NUMBER_SECOND_GUARDS < 20); + +/** Lifetime of L2 guards: + * 1 to 12 days, for an average of a week using the max(x,x) distribution */ +#define MIN_SECOND_GUARD_LIFETIME (3600*24) +#define MAX_SECOND_GUARD_LIFETIME (3600*24*12) + +/** Return the number of guards our L2 guardset should have */ +static int +get_number_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-number", + NUMBER_SECOND_GUARDS, + 1, 19); +} + +/** Return the minimum lifetime of L2 guards */ +static int +get_min_lifetime_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-lifetime-min", + MIN_SECOND_GUARD_LIFETIME, + 1, INT32_MAX); +} + +/** Return the maximum lifetime of L2 guards */ +static int +get_max_lifetime_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-lifetime-max", + MAX_SECOND_GUARD_LIFETIME, + 1, INT32_MAX); +} + +/** + * Sample and return a lifetime for an L2 guard. + * + * Lifetime randomized uniformly between min and max consensus params. + */ +static int +get_layer2_hs_guard_lifetime(void) +{ + int min = get_min_lifetime_of_layer2_hs_guards(); + int max = get_max_lifetime_of_layer2_hs_guards(); + + if (BUG(min >= max)) { + return min; + } + + return crypto_rand_int_range(min, max); +} + +/** Maintain the L2 guard list. Make sure the list contains enough guards, do + * expirations as necessary, and keep all the data structures of this + * subsystem synchronized */ +void +maintain_layer2_guards(void) +{ + if (!router_have_minimum_dir_info()) { + return; + } + + /* Create the list if it doesn't exist */ + if (!layer2_guards) { + layer2_guards = smartlist_new(); + } + + /* Go through the list and perform any needed expirations */ + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + /* Expire based on expiration date */ + if (g->expire_on_date <= approx_time()) { + log_info(LD_GENERAL, "Removing expired Layer2 guard %s", + safe_str_client(hex_str(g->identity, DIGEST_LEN))); + // Nickname may be gone from consensus and doesn't matter anyway + control_event_guard("None", g->identity, "BAD_L2"); + layer2_guard_free(g); + SMARTLIST_DEL_CURRENT_KEEPORDER(layer2_guards, g); + continue; + } + + /* Expire if relay has left consensus */ + if (router_get_consensus_status_by_id(g->identity) == NULL) { + log_info(LD_GENERAL, "Removing missing Layer2 guard %s", + safe_str_client(hex_str(g->identity, DIGEST_LEN))); + // Nickname may be gone from consensus and doesn't matter anyway + control_event_guard("None", g->identity, "BAD_L2"); + layer2_guard_free(g); + SMARTLIST_DEL_CURRENT_KEEPORDER(layer2_guards, g); + continue; + } + } SMARTLIST_FOREACH_END(g); + + /* Find out how many guards we need to add */ + int new_guards_needed_n = + get_number_of_layer2_hs_guards() - smartlist_len(layer2_guards); + if (new_guards_needed_n <= 0) { + return; + } + + log_info(LD_GENERAL, "Adding %d guards to Layer2 routerset", + new_guards_needed_n); + + /* Add required guards to the list */ + smartlist_t *excluded = smartlist_new(); + for (int i = 0; i < new_guards_needed_n; i++) { + const node_t *choice = NULL; + const or_options_t *options = get_options(); + /* Pick Stable nodes */ + router_crn_flags_t flags = CRN_NEED_DESC|CRN_NEED_UPTIME; + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + if (!choice) { + break; + } + + /* We found our node: create an L2 guard out of it */ + layer2_guard_t *layer2_guard = tor_malloc_zero(sizeof(layer2_guard_t)); + memcpy(layer2_guard->identity, choice->identity, DIGEST_LEN); + layer2_guard->expire_on_date = approx_time() + + get_layer2_hs_guard_lifetime(); + smartlist_add(layer2_guards, layer2_guard); + log_info(LD_GENERAL, "Adding Layer2 guard %s", + safe_str_client(hex_str(layer2_guard->identity, DIGEST_LEN))); + // Nickname can also be None here because it is looked up later + control_event_guard("None", layer2_guard->identity, + "GOOD_L2"); + /* Exclude this node and its family so that we don't double-pick. */ + nodelist_add_node_and_family(excluded, choice); + } + + /* Some cleanup */ + smartlist_free(excluded); + + /* Now that the list is up to date, synchronize the routerset */ + routerset_free(layer2_routerset); + layer2_routerset = routerset_new(); + + SMARTLIST_FOREACH_BEGIN (layer2_guards, layer2_guard_t *, g) { + routerset_parse(layer2_routerset, + hex_str(g->identity, DIGEST_LEN), + "l2 guards"); + } SMARTLIST_FOREACH_END(g); +} + +/** + * Reset vanguards-lite list(s). + * + * Used for SIGNAL NEWNYM. + */ +void +purge_vanguards_lite(void) +{ + if (!layer2_guards) + return; + + /* Go through the list and perform any needed expirations */ + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + layer2_guard_free(g); + } SMARTLIST_FOREACH_END(g); + + smartlist_clear(layer2_guards); + + /* Pick new l2 guards */ + maintain_layer2_guards(); +} + +/** Return a routerset containing the L2 guards or NULL if it's not yet + * initialized. Callers must not free the routerset. Designed for use in + * pick_vanguard_middle_node() and should not be used anywhere else. Do not + * store this pointer -- any future calls to maintain_layer2_guards() and + * purge_vanguards_lite() can invalidate it. */ +const routerset_t * +get_layer2_guards(void) +{ + if (!layer2_guards) { + maintain_layer2_guards(); + } + + return layer2_routerset; +} + +/*****************************************************************************/ + /** Release all storage held by the list of entry guards and related * memory structs. */ void @@ -3946,4 +4193,15 @@ entry_guards_free_all(void) guard_contexts = NULL; } circuit_build_times_free_timeouts(get_circuit_build_times_mutable()); + + if (!layer2_guards) { + return; + } + + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + layer2_guard_free(g); + } SMARTLIST_FOREACH_END(g); + + smartlist_free(layer2_guards); + routerset_free(layer2_routerset); } diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h index 88ed8f649e..08fd7cf745 100644 --- a/src/feature/client/entrynodes.h +++ b/src/feature/client/entrynodes.h @@ -651,4 +651,9 @@ guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, int orig_bandwidth, uint32_t guardfraction_percentage); +bool vanguards_lite_is_enabled(void); +const routerset_t *get_layer2_guards(void); +void maintain_layer2_guards(void); +void purge_vanguards_lite(void); + #endif /* !defined(TOR_ENTRYNODES_H) */ diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index bd0d41d29e..2950a1c0cc 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1075,7 +1075,7 @@ static const control_cmd_syntax_t redirectstream_syntax = { .max_args = UINT_MAX, // XXX should be 3. }; -/** Called when we receive a REDIRECTSTERAM command. Try to change the target +/** Called when we receive a REDIRECTSTREAM command. Try to change the target * address of the named AP stream, and report success or failure. */ static int handle_control_redirectstream(control_connection_t *conn, diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h index ed6b00647e..ac45f3787b 100644 --- a/src/feature/dirclient/dir_server_st.h +++ b/src/feature/dirclient/dir_server_st.h @@ -16,6 +16,8 @@ #include "core/or/or.h" #include "feature/nodelist/routerstatus_st.h" +struct smartlist_t; + /** Represents information about a single trusted or fallback directory * server. */ struct dir_server_t { @@ -48,6 +50,10 @@ struct dir_server_t { time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ + /** Authority only. Can be null. If present, a list of auth_dirport_t + * representing HTTP dirports for this authority. */ + struct smartlist_t *auth_dirports; + routerstatus_t fake_status; /**< Used when we need to pass this trusted * dir_server_t to * directory_request_set_routerstatus. diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index fd677d33fe..a2b20621a0 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -1119,6 +1119,7 @@ directory_request_set_routerstatus(directory_request_t *req, { req->routerstatus = status; } + /** * Helper: update the addresses, ports, and identities in <b>req</b> * from the routerstatus object in <b>req</b>. Return 0 on success. @@ -1161,7 +1162,7 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } - /* At this point, if we are a client making a direct connection to a + /* At this point, if we are a client making a direct connection to a * directory server, we have selected a server that has at least one address * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if @@ -1176,6 +1177,37 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } + /* One last thing: If we're talking to an authority, we might want to use + * a special HTTP port for it based on our purpose. + */ + if (req->indirection == DIRIND_DIRECT_CONN && status->is_authority) { + const dir_server_t *ds = router_get_trusteddirserver_by_digest( + status->identity_digest); + if (ds) { + const tor_addr_port_t *v4 = NULL; + if (authdir_mode_v3(get_options())) { + // An authority connecting to another authority should always + // prefer the VOTING usage, if one is specifically configured. + v4 = trusted_dir_server_get_dirport_exact( + ds, AUTH_USAGE_VOTING, AF_INET); + } + if (! v4) { + // Everybody else should prefer a usage dependent on their + // the dir_purpose. + auth_dirport_usage_t usage = + auth_dirport_usage_for_purpose(req->dir_purpose); + v4 = trusted_dir_server_get_dirport(ds, usage, AF_INET); + } + tor_assert_nonfatal(v4); + if (v4) { + // XXXX We could, if we wanted, also select a v6 address. But a v4 + // address must exist here, and we as a relay are required to support + // ipv4. So we just that. + tor_addr_port_copy(&use_dir_ap, v4); + } + } + } + directory_request_set_or_addr_port(req, &use_or_ap); directory_request_set_dir_addr_port(req, &use_dir_ap); directory_request_set_directory_id_digest(req, status->identity_digest); @@ -1194,7 +1226,7 @@ directory_initiate_request,(directory_request_t *request)) tor_assert_nonfatal( ! directory_request_dir_contact_info_specified(request)); if (directory_request_set_dir_from_routerstatus(request) < 0) { - return; + return; // or here XXXX } } @@ -1309,6 +1341,8 @@ directory_initiate_request,(directory_request_t *request)) entry_guard_cancel(&guard_state); } + // XXXX This is the case where we replace. + switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 68ed932701..f9f71c78f2 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -544,15 +544,19 @@ config_service(config_line_t *line, const or_options_t *options, tor_assert(service->config.version <= HS_VERSION_MAX); - /* Check permission on service directory that was just parsed. And this must - * be done regardless of the service version. Do not ask for the directory - * to be created, this is done when the keys are loaded because we could be - * in validation mode right now. */ - if (hs_check_service_private_dir(options->User, - service->config.directory_path, - service->config.dir_group_readable, - 0) < 0) { - goto err; + /* If we're running with TestingTorNetwork enabled, we relax the permissions + * check on the hs directory. */ + if (!options->TestingTorNetwork) { + /* Check permission on service directory that was just parsed. And this + * must be done regardless of the service version. Do not ask for the + * directory to be created, this is done when the keys are loaded because + * we could be in validation mode right now. */ + if (hs_check_service_private_dir(options->User, + service->config.directory_path, + service->config.dir_group_readable, + 0) < 0) { + goto err; + } } /* We'll try to learn the service version here by loading the key(s) if diff --git a/src/feature/hs/hs_metrics.c b/src/feature/hs/hs_metrics.c index e023eab90c..0f1824c51c 100644 --- a/src/feature/hs/hs_metrics.c +++ b/src/feature/hs/hs_metrics.c @@ -29,18 +29,6 @@ port_to_str(const uint16_t port) return buf; } -/** Return a static buffer pointer that contains a formatted label on the form - * of key=value. - * - * Subsequent call to this function invalidates the previous buffer. */ -static const char * -format_label(const char *key, const char *value) -{ - static char buf[128]; - tor_snprintf(buf, sizeof(buf), "%s=%s", key, value); - return buf; -} - /** Initialize a metrics store for the given service. * * Essentially, this goes over the base_metrics array and adds them all to the @@ -61,12 +49,12 @@ init_store(hs_service_t *service) /* Add labels to the entry. */ metrics_store_entry_add_label(entry, - format_label("onion", service->onion_address)); + metrics_format_label("onion", service->onion_address)); if (base_metrics[i].port_as_label && service->config.ports) { SMARTLIST_FOREACH_BEGIN(service->config.ports, const hs_port_config_t *, p) { metrics_store_entry_add_label(entry, - format_label("port", port_to_str(p->virtual_port))); + metrics_format_label("port", port_to_str(p->virtual_port))); } SMARTLIST_FOREACH_END(p); } } @@ -96,7 +84,7 @@ hs_metrics_update_by_service(const hs_metrics_key_t key, SMARTLIST_FOREACH_BEGIN(entries, metrics_store_entry_t *, entry) { if (port == 0 || metrics_store_entry_has_label(entry, - format_label("port", port_to_str(port)))) { + metrics_format_label("port", port_to_str(port)))) { metrics_store_entry_update(entry, n); break; } diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index 1f18bd71a2..1f1ac4d106 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -43,6 +43,14 @@ #include "feature/dirclient/dir_server_st.h" #include "feature/nodelist/node_st.h" +/** Information about an (HTTP) dirport for a directory authority. */ +struct auth_dirport_t { + /** What is the intended usage for this dirport? One of AUTH_USAGE_* */ + auth_dirport_usage_t usage; + /** What is the correct address/port ? */ + tor_addr_port_t dirport; +}; + /** Global list of a dir_server_t object for each directory * authority. */ static smartlist_t *trusted_dir_servers = NULL; @@ -66,6 +74,11 @@ add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir) /* IPv6 DirPort is not a thing yet for authorities. */ nodelist_add_addr_to_address_set(&dir->ipv6_addr, dir->ipv6_orport, 0); } + if (dir->auth_dirports) { + SMARTLIST_FOREACH_BEGIN(dir->auth_dirports, const auth_dirport_t *, p) { + nodelist_add_addr_to_address_set(&p->dirport.addr, 0, p->dirport.port); + } SMARTLIST_FOREACH_END(p); + } } /** Go over the trusted directory server list and add their address(es) to the @@ -256,7 +269,10 @@ MOCK_IMPL(int, router_digest_is_trusted_dir_type, /** Return true iff the given address matches a trusted directory that matches * at least one bit of type. * - * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. */ + * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. + * + * Only ORPorts' addresses are considered. + */ bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) { @@ -281,6 +297,39 @@ router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) return false; } +/** Return an appropriate usage value describing which authdir port to use + * for a given directory connection purpose. + */ +auth_dirport_usage_t +auth_dirport_usage_for_purpose(int purpose) +{ + switch (purpose) { + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + case DIR_PURPOSE_FETCH_CONSENSUS: + case DIR_PURPOSE_FETCH_CERTIFICATE: + case DIR_PURPOSE_FETCH_MICRODESC: + return AUTH_USAGE_DOWNLOAD; + + case DIR_PURPOSE_UPLOAD_DIR: + return AUTH_USAGE_UPLOAD; + + case DIR_PURPOSE_UPLOAD_VOTE: + case DIR_PURPOSE_UPLOAD_SIGNATURES: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_STATUS_VOTE: + return AUTH_USAGE_VOTING; + + case DIR_PURPOSE_SERVER: + case DIR_PURPOSE_UPLOAD_HSDESC: + case DIR_PURPOSE_FETCH_HSDESC: + case DIR_PURPOSE_HAS_FETCHED_HSDESC: + default: + tor_assert_nonfatal_unreached(); + return AUTH_USAGE_LEGACY; + } +} + /** Create a directory server at <b>address</b>:<b>port</b>, with OR identity * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, * add ourself. If <b>is_authority</b>, this is a directory authority. Return @@ -357,6 +406,7 @@ dir_server_new(int is_authority, ent->fake_status.ipv4_dirport = ent->ipv4_dirport; ent->fake_status.ipv4_orport = ent->ipv4_orport; ent->fake_status.ipv6_orport = ent->ipv6_orport; + ent->fake_status.is_authority = !! is_authority; return ent; } @@ -404,10 +454,98 @@ trusted_dir_server_new(const char *nickname, const char *address, ipv6_addrport, digest, v3_auth_digest, type, weight); + + if (ipv4_dirport) { + tor_addr_port_t p; + memset(&p, 0, sizeof(p)); + tor_addr_copy(&p.addr, &ipv4_addr); + p.port = ipv4_dirport; + trusted_dir_server_add_dirport(result, AUTH_USAGE_LEGACY, &p); + } tor_free(hostname); return result; } +/** + * Add @a dirport as an HTTP DirPort contact point for the directory authority + * @a ds, for use when contacting that authority for the given @a usage. + * + * Multiple ports of the same usage are allowed; if present, then only + * the first one of each address family is currently used. + */ +void +trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport) +{ + tor_assert(ds); + tor_assert(dirport); + + if (BUG(! ds->is_authority)) { + return; + } + + if (ds->auth_dirports == NULL) { + ds->auth_dirports = smartlist_new(); + } + + auth_dirport_t *port = tor_malloc_zero(sizeof(auth_dirport_t)); + port->usage = usage; + tor_addr_port_copy(&port->dirport, dirport); + smartlist_add(ds->auth_dirports, port); +} + +/** + * Helper for trusted_dir_server_get_dirport: only return the exact requested + * usage type. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport_exact(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + tor_assert(ds); + tor_assert_nonfatal(addr_family == AF_INET || addr_family == AF_INET6); + if (ds->auth_dirports == NULL) + return NULL; + + SMARTLIST_FOREACH_BEGIN(ds->auth_dirports, const auth_dirport_t *, port) { + if (port->usage == usage && + tor_addr_family(&port->dirport.addr) == addr_family) { + return &port->dirport; + } + } SMARTLIST_FOREACH_END(port); + + return NULL; +} + +/** + * Return the DirPort of the authority @a ds for with the usage type + * @a usage and address family @a addr_family. If none is found, try + * again with an AUTH_USAGE_LEGACY dirport, if there is one. Return NULL + * if no port can be found. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + const tor_addr_port_t *port; + + while (1) { + port = trusted_dir_server_get_dirport_exact(ds, usage, addr_family); + if (port) + return port; + + // If we tried LEGACY, there is no fallback from this point. + if (usage == AUTH_USAGE_LEGACY) + return NULL; + + // Try again with LEGACY. + usage = AUTH_USAGE_LEGACY; + } +} + /** Return a new dir_server_t for a fallback directory server at * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest * <b>id_digest</b> */ @@ -447,6 +585,10 @@ dir_server_free_(dir_server_t *ds) if (!ds) return; + if (ds->auth_dirports) { + SMARTLIST_FOREACH(ds->auth_dirports, auth_dirport_t *, p, tor_free(p)); + smartlist_free(ds->auth_dirports); + } tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h index f744fecf92..3b4faf07af 100644 --- a/src/feature/nodelist/dirlist.h +++ b/src/feature/nodelist/dirlist.h @@ -11,6 +11,28 @@ #ifndef TOR_DIRLIST_H #define TOR_DIRLIST_H +typedef struct auth_dirport_t auth_dirport_t; +/** + * Different usages for an authority's HTTP directory port. + * + * Historically, only legacy ports existed; proposal 330 added multiple types + * of dirport to better enable authorities to offload work and resist DoS + * attacks. + **/ +typedef enum auth_dirport_usage_t { + /** Flag for an authority's dirport that is intended for misc/legacy + * usage. May be used when no other dirport is available. */ + AUTH_USAGE_LEGACY, + /** Flag for an authority's dirport that is intended for descriptor uploads + * only. */ + AUTH_USAGE_UPLOAD, + /** Flag for an authority's dirport that is intended for voting only */ + AUTH_USAGE_VOTING, + /** Flag for an authority's dirport that is intended for relay downloads + * only. */ + AUTH_USAGE_DOWNLOAD, +} auth_dirport_usage_t; + int get_n_authorities(dirinfo_type_t type); const smartlist_t *router_get_trusted_dir_servers(void); const smartlist_t *router_get_fallback_dir_servers(void); @@ -18,6 +40,8 @@ smartlist_t *router_get_trusted_dir_servers_mutable(void); smartlist_t *router_get_fallback_dir_servers_mutable(void); void mark_all_dirservers_up(smartlist_t *server_list); +auth_dirport_usage_t auth_dirport_usage_for_purpose(int purpose); + dir_server_t *router_get_trusteddirserver_by_digest(const char *d); dir_server_t *router_get_fallback_dirserver_by_digest( const char *digest); @@ -28,6 +52,14 @@ MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, MOCK_DECL(int, router_digest_is_trusted_dir_type, (const char *digest, dirinfo_type_t type)); +const tor_addr_port_t *trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); +const tor_addr_port_t *trusted_dir_server_get_dirport_exact( + const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); + bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type); #define router_addr_is_trusted_dir(d) \ @@ -41,6 +73,9 @@ dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, const tor_addr_port_t *addrport_ipv6, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight); +void trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport); dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, uint16_t dir_port, uint16_t or_port, const tor_addr_port_t *addrport_ipv6, diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index 2ffa6da1a3..7a1e73ef60 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -1699,6 +1699,9 @@ notify_after_networkstatus_changes(void) channelpadding_new_consensus_params(c); circpad_new_consensus_params(c); router_new_consensus_params(c); + + /* Maintenance of our L2 guard list */ + maintain_layer2_guards(); } /** Copy all the ancillary information (like router download status and so on) diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 565d4596d4..fbf9e026c2 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -2012,6 +2012,30 @@ routerlist_remove_old_routers(void) router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store); } +/* Drop every bridge descriptor in our routerlist. Used by the external + * 'bridgestrap' tool to discard bridge descriptors so that it can then + * do a clean reachability test. */ +void +routerlist_drop_bridge_descriptors(void) +{ + routerinfo_t *router; + int i; + + if (!routerlist) + return; + + for (i = 0; i < smartlist_len(routerlist->routers); ++i) { + router = smartlist_get(routerlist->routers, i); + if (router->purpose == ROUTER_PURPOSE_BRIDGE) { + log_notice(LD_DIR, + "Dropping existing bridge descriptor for %s", + router_describe(router)); + routerlist_remove(routerlist, router, 0, time(NULL)); + i--; + } + } +} + /** We just added a new set of descriptors. Take whatever extra steps * we need. */ void diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index 7dc748c94b..7ba305baf6 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -145,6 +145,7 @@ was_router_added_t router_add_extrainfo_to_routerlist( int from_cache, int from_fetch); void routerlist_descriptors_added(smartlist_t *sl, int from_cache); void routerlist_remove_old_routers(void); +void routerlist_drop_bridge_descriptors(void); int router_load_single_router(const char *s, uint8_t purpose, int cache, const char **msg); int router_load_routers_from_string(const char *s, const char *eos, diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 22f929808e..c6e0439338 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -212,20 +212,11 @@ evdns_log_cb(int warn, const char *msg) tor_log(severity, LD_EXIT, "eventdns: %s", msg); } -/** Helper: passed to eventdns.c as a callback so it can generate random - * numbers for transaction IDs and 0x20-hack coding. */ -static void -dns_randfn_(char *b, size_t n) -{ - crypto_rand(b,n); -} - /** Initialize the DNS subsystem; called by the OR process. */ int dns_init(void) { init_cache_map(); - evdns_set_random_bytes_fn(dns_randfn_); if (server_mode(get_options())) { int r = configure_nameservers(1); return r; @@ -1659,6 +1650,10 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses, dns_found_answer(string_address, orig_query_type, result, &addr, hostname, ttl); + /* The result can be changed within this function thus why we note the result + * at the end. */ + rep_hist_note_dns_error(type, result); + tor_free(arg_); } @@ -1677,6 +1672,9 @@ launch_one_resolve(const char *address, uint8_t query_type, addr[0] = (char) query_type; memcpy(addr+1, address, addr_len + 1); + /* Note the query for our statistics. */ + rep_hist_note_dns_request(query_type); + switch (query_type) { case DNS_IPv4_A: req = evdns_base_resolve_ipv4(the_evdns_base, diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am index 84bb1ff35e..8a121cef01 100644 --- a/src/feature/relay/include.am +++ b/src/feature/relay/include.am @@ -15,6 +15,7 @@ MODULE_RELAY_SOURCES = \ src/feature/relay/routermode.c \ src/feature/relay/relay_config.c \ src/feature/relay/relay_handshake.c \ + src/feature/relay/relay_metrics.c \ src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ src/feature/relay/routerkeys.c \ @@ -30,6 +31,7 @@ noinst_HEADERS += \ src/feature/relay/onion_queue.h \ src/feature/relay/relay_config.h \ src/feature/relay/relay_handshake.h \ + src/feature/relay/relay_metrics.h \ src/feature/relay/relay_periodic.h \ src/feature/relay/relay_sys.h \ src/feature/relay/relay_find_addr.h \ diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 85ec0dc74a..c09f4d5b9b 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -164,6 +164,7 @@ onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin) #define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + rep_hist_note_circuit_handshake_dropped(onionskin->handshake_type); if (onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR) { char *m; /* Note this ntor onionskin drop as an overload */ diff --git a/src/feature/relay/relay_metrics.c b/src/feature/relay/relay_metrics.c new file mode 100644 index 0000000000..07f89fc043 --- /dev/null +++ b/src/feature/relay/relay_metrics.c @@ -0,0 +1,378 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.c + * @brief Relay metrics exposed through the MetricsPort + **/ + +#define RELAY_METRICS_ENTRY_PRIVATE + +#include "orconfig.h" + +#include "core/or/or.h" +#include "core/or/relay.h" + +#include "lib/malloc/malloc.h" +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_store.h" +#include "lib/log/util_bug.h" + +#include "feature/relay/relay_metrics.h" +#include "feature/stats/rephist.h" + +#include <event2/dns.h> + +/** Declarations of each fill function for metrics defined in base_metrics. */ +static void fill_dns_error_values(void); +static void fill_dns_query_values(void); +static void fill_global_bw_limit_values(void); +static void fill_socket_values(void); +static void fill_onionskins_values(void); +static void fill_oom_values(void); +static void fill_tcp_exhaustion_values(void); + +/** The base metrics that is a static array of metrics added to the metrics + * store. + * + * The key member MUST be also the index of the entry in the array. */ +static const relay_metrics_entry_t base_metrics[] = +{ + { + .key = RELAY_METRICS_NUM_OOM_BYTES, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_oom_bytes_total), + .help = "Total number of bytes the OOM has freed by subsystem", + .fill_fn = fill_oom_values, + }, + { + .key = RELAY_METRICS_NUM_ONIONSKINS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_onionskins_total), + .help = "Total number of onionskins handled", + .fill_fn = fill_onionskins_values, + }, + { + .key = RELAY_METRICS_NUM_SOCKETS, + .type = METRICS_TYPE_GAUGE, + .name = METRICS_NAME(relay_load_socket_total), + .help = "Total number of sockets", + .fill_fn = fill_socket_values, + }, + { + .key = RELAY_METRICS_NUM_GLOBAL_RW_LIMIT, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_global_rate_limit_reached_total), + .help = "Total number of global connection bucket limit reached", + .fill_fn = fill_global_bw_limit_values, + }, + { + .key = RELAY_METRICS_NUM_DNS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_query_total), + .help = "Total number of DNS queries done by this relay", + .fill_fn = fill_dns_query_values, + }, + { + .key = RELAY_METRICS_NUM_DNS_ERRORS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_error_total), + .help = "Total number of DNS errors encountered by this relay", + .fill_fn = fill_dns_error_values, + }, + { + .key = RELAY_METRICS_NUM_TCP_EXHAUSTION, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_tcp_exhaustion_total), + .help = "Total number of times we ran out of TCP ports", + .fill_fn = fill_tcp_exhaustion_values, + }, +}; +static const size_t num_base_metrics = ARRAY_LENGTH(base_metrics); + +/** The only and single store of all the relay metrics. */ +static metrics_store_t *the_store; + +/** Helper function to convert an handshake type into a string. */ +static inline const char * +handshake_type_to_str(const uint16_t type) +{ + switch (type) { + case ONION_HANDSHAKE_TYPE_TAP: + return "tap"; + case ONION_HANDSHAKE_TYPE_FAST: + return "fast"; + case ONION_HANDSHAKE_TYPE_NTOR: + return "ntor"; + default: + // LCOV_EXCL_START + tor_assert_unreached(); + // LCOV_EXCL_STOP + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_tcp_exhaustion_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_TCP_EXHAUSTION]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, rep_hist_get_n_tcp_exhaustion()); +} + +/** Helper array containing mapping for the name of the different DNS records + * and their corresponding libevent values. */ +static struct dns_type { + const char *name; + uint8_t type; +} dns_types[] = { + { .name = "A", .type = DNS_IPv4_A }, + { .name = "PTR", .type = DNS_PTR }, + { .name = "AAAA", .type = DNS_IPv6_AAAA }, +}; +static const size_t num_dns_types = ARRAY_LENGTH(dns_types); + +/** Fill function for the RELAY_METRICS_NUM_DNS_ERRORS metrics. */ +static void +fill_dns_error_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS_ERRORS]; + + /* Helper array to map libeven DNS errors to their names and so we can + * iterate over this array to add all metrics. */ + static struct dns_error { + const char *name; + uint8_t key; + } errors[] = { + { .name = "success", .key = DNS_ERR_NONE }, + { .name = "format", .key = DNS_ERR_FORMAT }, + { .name = "serverfailed", .key = DNS_ERR_SERVERFAILED }, + { .name = "notexist", .key = DNS_ERR_NOTEXIST }, + { .name = "notimpl", .key = DNS_ERR_NOTIMPL }, + { .name = "refused", .key = DNS_ERR_REFUSED }, + { .name = "truncated", .key = DNS_ERR_TRUNCATED }, + { .name = "unknown", .key = DNS_ERR_UNKNOWN }, + { .name = "timeout", .key = DNS_ERR_TIMEOUT }, + { .name = "shutdown", .key = DNS_ERR_SHUTDOWN }, + { .name = "cancel", .key = DNS_ERR_CANCEL }, + { .name = "nodata", .key = DNS_ERR_NODATA }, + }; + static const size_t num_errors = ARRAY_LENGTH(errors); + + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + + for (size_t j = 0; j < num_errors; j++) { + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("reason", errors[j].name)); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_error(dns_types[i].type, errors[j].key)); + } + tor_free(record_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_dns_query_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS]; + + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_request(dns_types[i].type)); + tor_free(record_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_GLOBAL_RW_LIMIT metrics. */ +static void +fill_global_bw_limit_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_GLOBAL_RW_LIMIT]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "read")); + metrics_store_entry_update(sentry, rep_hist_get_n_read_limit_reached()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "write")); + metrics_store_entry_update(sentry, rep_hist_get_n_write_limit_reached()); +} + +/** Fill function for the RELAY_METRICS_NUM_SOCKETS metrics. */ +static void +fill_socket_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_SOCKETS]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("state", "opened")); + metrics_store_entry_update(sentry, get_n_open_sockets()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, get_max_sockets()); +} + +/** Fill function for the RELAY_METRICS_NUM_ONIONSKINS metrics. */ +static void +fill_onionskins_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_ONIONSKINS]; + + for (uint16_t t = 0; t <= MAX_ONION_HANDSHAKE_TYPE; t++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *type_label = + tor_strdup(metrics_format_label("type", handshake_type_to_str(t))); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "processed")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_n_handshake_assigned(t)); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "dropped")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_n_handshake_dropped(t)); + tor_free(type_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_OOM_BYTES metrics. */ +static void +fill_oom_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_OOM_BYTES]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "cell")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_cell); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "dns")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_dns); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "geoip")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_geoip); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "hsdir")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_hsdir); +} + +/** Reset the global store and fill it with all the metrics from base_metrics + * and their associated values. + * + * To pull this off, every metrics has a "fill" function that is called and in + * charge of adding the metrics to the store, appropriate labels and finally + * updating the value to report. */ +static void +fill_store(void) +{ + /* Reset the current store, we are about to fill it with all the things. */ + metrics_store_reset(the_store); + + /* Call the fill function for each metrics. */ + for (size_t i = 0; i < num_base_metrics; i++) { + if (BUG(!base_metrics[i].fill_fn)) { + continue; + } + base_metrics[i].fill_fn(); + } +} + +/** Return a list of all the relay metrics stores. This is the + * function attached to the .get_metrics() member of the subsys_t. */ +const smartlist_t * +relay_metrics_get_stores(void) +{ + /* We can't have the caller to free the returned list so keep it static, + * simply update it. */ + static smartlist_t *stores_list = NULL; + + /* We dynamically fill the store with all the metrics upon a request. The + * reason for this is because the exposed metrics of a relay are often + * internal counters in the fast path and thus we fetch the value when a + * metrics port request arrives instead of keeping a local metrics store of + * those values. */ + fill_store(); + + if (!stores_list) { + stores_list = smartlist_new(); + smartlist_add(stores_list, the_store); + } + + return stores_list; +} + +/** Initialize the relay metrics. */ +void +relay_metrics_init(void) +{ + if (BUG(the_store)) { + return; + } + the_store = metrics_store_new(); +} + +/** Free the relay metrics. */ +void +relay_metrics_free(void) +{ + if (!the_store) { + return; + } + /* NULL is set with this call. */ + metrics_store_free(the_store); +} diff --git a/src/feature/relay/relay_metrics.h b/src/feature/relay/relay_metrics.h new file mode 100644 index 0000000000..00dfeaa624 --- /dev/null +++ b/src/feature/relay/relay_metrics.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.h + * @brief Header for feature/relay/relay_metrics.c + **/ + +#ifndef TOR_FEATURE_RELAY_RELAY_METRICS_H +#define TOR_FEATURE_RELAY_RELAY_METRICS_H + +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_common.h" + +/** Metrics key for each reported metrics. This key is also used as an index in + * the base_metrics array. */ +typedef enum { + /** Number of OOM invocation. */ + RELAY_METRICS_NUM_OOM_BYTES = 0, + /** Number of onionskines handled. */ + RELAY_METRICS_NUM_ONIONSKINS = 1, + /** Number of sockets. */ + RELAY_METRICS_NUM_SOCKETS = 2, + /** Number of global connection rate limit. */ + RELAY_METRICS_NUM_GLOBAL_RW_LIMIT = 3, + /** Number of DNS queries. */ + RELAY_METRICS_NUM_DNS = 4, + /** Number of DNS query errors. */ + RELAY_METRICS_NUM_DNS_ERRORS = 5, + /** Number of TCP exhaustion reached. */ + RELAY_METRICS_NUM_TCP_EXHAUSTION = 6, +} relay_metrics_key_t; + +/** The metadata of a relay metric. */ +typedef struct relay_metrics_entry_t { + /* Metric key used as a static array index. */ + relay_metrics_key_t key; + /* Metric type. */ + metrics_type_t type; + /* Metrics output name. */ + const char *name; + /* Metrics output help comment. */ + const char *help; + /* Update value function. */ + void (*fill_fn)(void); +} relay_metrics_entry_t; + +/* Init. */ +void relay_metrics_init(void); +void relay_metrics_free(void); + +/* Accessors. */ +const smartlist_t *relay_metrics_get_stores(void); + +#endif /* !defined(TOR_FEATURE_RELAY_RELAY_METRICS_H) */ diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c index 25fc0bbd32..9c43734b84 100644 --- a/src/feature/relay/relay_sys.c +++ b/src/feature/relay/relay_sys.c @@ -14,6 +14,7 @@ #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" +#include "feature/relay/relay_metrics.h" #include "feature/relay/onion_queue.h" #include "feature/relay/relay_periodic.h" #include "feature/relay/relay_sys.h" @@ -25,6 +26,7 @@ static int subsys_relay_initialize(void) { + relay_metrics_init(); relay_register_periodic_events(); return 0; } @@ -37,6 +39,7 @@ subsys_relay_shutdown(void) clear_pending_onions(); routerkeys_free_all(); router_free_all(); + relay_metrics_free(); } const struct subsys_fns_t sys_relay = { @@ -46,4 +49,6 @@ const struct subsys_fns_t sys_relay = { .level = RELAY_SUBSYS_LEVEL, .initialize = subsys_relay_initialize, .shutdown = subsys_relay_shutdown, + + .get_metrics = relay_metrics_get_stores, }; diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 67d3e3ee75..a2ca472307 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2622,7 +2622,10 @@ check_descriptor_bandwidth_changed(time_t now) if ((prev != cur && (!prev || !cur)) || cur > (prev * BANDWIDTH_CHANGE_FACTOR) || cur < (prev / BANDWIDTH_CHANGE_FACTOR) ) { - if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) { + const bool change_recent_enough = + last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now; + const bool testing_network = get_options()->TestingTorNetwork; + if (change_recent_enough || testing_network || !prev) { log_info(LD_GENERAL, "Measured bandwidth has changed; rebuilding descriptor."); mark_my_descriptor_dirty("bandwidth has changed"); diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index d5a9b73605..552dc7ad74 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -206,16 +206,24 @@ bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when) add_obs(dir_read_array, when, num_bytes); } -/** Helper: Return the largest value in b->maxima. (This is equal to the +/** + * Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) + * + * Also include the current period if we have been observing it for + * at least min_observation_time seconds. */ STATIC uint64_t -find_largest_max(bw_array_t *b) +find_largest_max(bw_array_t *b, int min_observation_time) { int i; uint64_t max; - max=0; + time_t period_start = b->next_period - NUM_SECS_BW_SUM_INTERVAL; + if (b->cur_obs_time > period_start + min_observation_time) + max = b->max_total; + else + max = 0; for (i=0; i<NUM_TOTALS; ++i) { if (b->maxima[i]>max) max = b->maxima[i]; @@ -233,8 +241,9 @@ MOCK_IMPL(int, bwhist_bandwidth_assess,(void)) { uint64_t w,r; - r = find_largest_max(read_array); - w = find_largest_max(write_array); + int min_obs_time = get_options()->TestingMinTimeToReportBandwidth; + r = find_largest_max(read_array, min_obs_time); + w = find_largest_max(write_array, min_obs_time); if (r>w) return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); else diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h index e7fc60fdee..d61c442e5d 100644 --- a/src/feature/stats/bwhist.h +++ b/src/feature/stats/bwhist.h @@ -28,7 +28,7 @@ int bwhist_load_state(struct or_state_t *state, char **err); #ifdef BWHIST_PRIVATE typedef struct bw_array_t bw_array_t; -STATIC uint64_t find_largest_max(bw_array_t *b); +STATIC uint64_t find_largest_max(bw_array_t *b, int min_observation_time); STATIC void commit_max(bw_array_t *b); STATIC void advance_obs(bw_array_t *b); STATIC bw_array_t *bw_array_new(void); diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index b4b107c3f7..a0fe8597c1 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -1206,11 +1206,11 @@ format_bridge_stats_controller(time_t now) char * format_client_stats_heartbeat(time_t now) { - const int n_hours = 6; + const int n_seconds = get_options()->HeartbeatPeriod; char *out = NULL; int n_clients = 0; clientmap_entry_t **ent; - unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + unsigned cutoff = (unsigned)( (now-n_seconds)/60 ); if (!start_of_bridge_stats_interval) return NULL; /* Not initialized. */ @@ -1226,8 +1226,7 @@ format_client_stats_heartbeat(time_t now) } tor_asprintf(&out, "Heartbeat: " - "In the last %d hours, I have seen %d unique clients.", - n_hours, + "Since last heartbeat message, I have seen %d unique clients.", n_clients); return out; diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index e25c01331d..cb3ccdc91e 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -84,6 +84,8 @@ #include "feature/nodelist/networkstatus_st.h" #include "core/or/or_circuit_st.h" +#include <event2/dns.h> + #ifdef HAVE_FCNTL_H #include <fcntl.h> #endif @@ -207,6 +209,183 @@ typedef struct { /** Current state of overload stats */ static overload_stats_t overload_stats; +/** Counters to count the number of times we've reached an overload for the + * global connection read/write limit. Reported on the MetricsPort. */ +static uint64_t stats_n_read_limit_reached = 0; +static uint64_t stats_n_write_limit_reached = 0; + +/** Total number of times we've reached TCP port exhaustion. */ +static uint64_t stats_n_tcp_exhaustion = 0; + +/***** DNS statistics *****/ + +/** Represents the statistics of DNS queries seen if it is an Exit. */ +typedef struct { + /* Total number of DNS errors found in RFC 1035 (from 0 to 5 code). */ + uint64_t stats_n_error_none; /* 0 */ + uint64_t stats_n_error_format; /* 1 */ + uint64_t stats_n_error_serverfailed; /* 2 */ + uint64_t stats_n_error_notexist; /* 3 */ + uint64_t stats_n_error_notimpl; /* 4 */ + uint64_t stats_n_error_refused; /* 5 */ + + /* Total number of DNS errors specific to libevent. */ + uint64_t stats_n_error_truncated; /* 65 */ + uint64_t stats_n_error_unknown; /* 66 */ + uint64_t stats_n_error_timeout; /* 67 */ + uint64_t stats_n_error_shutdown; /* 68 */ + uint64_t stats_n_error_cancel; /* 69 */ + uint64_t stats_n_error_nodata; /* 70 */ + + /* Total number of DNS request seen at an Exit. They might not all end + * successfully or might even be lost by tor. This counter is incremented + * right before the DNS request is initiated. */ + uint64_t stats_n_request; +} dns_stats_t; + +/** DNS statistics store for each DNS record type for which tor supports only + * three at the moment: A, PTR and AAAA. */ +static dns_stats_t dns_A_stats; +static dns_stats_t dns_PTR_stats; +static dns_stats_t dns_AAAA_stats; + +/** From a libevent record type, return a pointer to the corresponding DNS + * statistics store. NULL is returned if the type is unhandled. */ +static inline dns_stats_t * +get_dns_stats_by_type(const int type) +{ + switch (type) { + case DNS_IPv4_A: + return &dns_A_stats; + case DNS_PTR: + return &dns_PTR_stats; + case DNS_IPv6_AAAA: + return &dns_AAAA_stats; + default: + return NULL; + } +} + +/** Return the DNS error count for the given libevent DNS type and error code. + * The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_error(int type, uint8_t error) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; + } + + switch (error) { + case DNS_ERR_NONE: + return dns_stats->stats_n_error_none; + case DNS_ERR_FORMAT: + return dns_stats->stats_n_error_format; + case DNS_ERR_SERVERFAILED: + return dns_stats->stats_n_error_serverfailed; + case DNS_ERR_NOTEXIST: + return dns_stats->stats_n_error_notexist; + case DNS_ERR_NOTIMPL: + return dns_stats->stats_n_error_notimpl; + case DNS_ERR_REFUSED: + return dns_stats->stats_n_error_refused; + case DNS_ERR_TRUNCATED: + return dns_stats->stats_n_error_truncated; + case DNS_ERR_UNKNOWN: + return dns_stats->stats_n_error_unknown; + case DNS_ERR_TIMEOUT: + return dns_stats->stats_n_error_timeout; + case DNS_ERR_SHUTDOWN: + return dns_stats->stats_n_error_shutdown; + case DNS_ERR_CANCEL: + return dns_stats->stats_n_error_cancel; + case DNS_ERR_NODATA: + return dns_stats->stats_n_error_nodata; + default: + /* Unhandled code sent back by libevent. */ + return 0; + } +} + +/** Return the total number of DNS request seen for the given libevent DNS + * record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_request(int type) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; + } + return dns_stats->stats_n_request; +} + +/** Note a DNS error for the given given libevent DNS record type and error + * code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +void +rep_hist_note_dns_error(int type, uint8_t error) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + /* Unsupported DNS query type. */ + if (!dns_stats) { + return; + } + + switch (error) { + case DNS_ERR_NONE: + dns_stats->stats_n_error_none++; + break; + case DNS_ERR_FORMAT: + dns_stats->stats_n_error_format++; + break; + case DNS_ERR_SERVERFAILED: + dns_stats->stats_n_error_serverfailed++; + break; + case DNS_ERR_NOTEXIST: + dns_stats->stats_n_error_notexist++; + break; + case DNS_ERR_NOTIMPL: + dns_stats->stats_n_error_notimpl++; + break; + case DNS_ERR_REFUSED: + dns_stats->stats_n_error_refused++; + break; + case DNS_ERR_TRUNCATED: + dns_stats->stats_n_error_truncated++; + break; + case DNS_ERR_UNKNOWN: + dns_stats->stats_n_error_unknown++; + break; + case DNS_ERR_TIMEOUT: + dns_stats->stats_n_error_timeout++; + break; + case DNS_ERR_SHUTDOWN: + dns_stats->stats_n_error_shutdown++; + break; + case DNS_ERR_CANCEL: + dns_stats->stats_n_error_cancel++; + break; + case DNS_ERR_NODATA: + dns_stats->stats_n_error_nodata++; + break; + default: + /* Unhandled code sent back by libevent. */ + break; + } +} + +/** Note a DNS request for the given given libevent DNS record type. */ +void +rep_hist_note_dns_request(int type) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return; + } + dns_stats->stats_n_request++; +} + +/***** END of DNS statistics *****/ + /** Return true if this overload happened within the last `n_hours`. */ static bool overload_happened_recently(time_t overload_time, int n_hours) @@ -221,6 +400,20 @@ overload_happened_recently(time_t overload_time, int n_hours) /* The current version of the overload stats version */ #define OVERLOAD_STATS_VERSION 1 +/** Return the stats_n_read_limit_reached counter. */ +uint64_t +rep_hist_get_n_read_limit_reached(void) +{ + return stats_n_read_limit_reached; +} + +/** Return the stats_n_write_limit_reached counter. */ +uint64_t +rep_hist_get_n_write_limit_reached(void) +{ + return stats_n_write_limit_reached; +} + /** Returns an allocated string for server descriptor for publising information * on whether we are overloaded or not. */ char * @@ -299,6 +492,7 @@ rep_hist_note_overload(overload_type_t overload) SET_TO_START_OF_HOUR(overload_stats.overload_general_time); break; case OVERLOAD_READ: { + stats_n_read_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_read_counted + 60) { /* Count once a minute */ overload_stats.overload_read_count++; @@ -307,6 +501,7 @@ rep_hist_note_overload(overload_type_t overload) break; } case OVERLOAD_WRITE: { + stats_n_write_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_write_counted + 60) { /* Count once a minute */ overload_stats.overload_write_count++; @@ -321,6 +516,22 @@ rep_hist_note_overload(overload_type_t overload) } } +/** Note down that we've reached a TCP port exhaustion. This triggers an + * overload general event. */ +void +rep_hist_note_tcp_exhaustion(void) +{ + stats_n_tcp_exhaustion++; + rep_hist_note_overload(OVERLOAD_GENERAL); +} + +/** Return the total number of TCP exhaustion times we've reached. */ +uint64_t +rep_hist_get_n_tcp_exhaustion(void) +{ + return stats_n_tcp_exhaustion; +} + /** Return the or_history_t for the OR with identity digest <b>id</b>, * creating it if necessary. */ static or_history_t * @@ -520,7 +731,7 @@ rep_hist_downrate_old_runs(time_t now) return stability_last_downrated + STABILITY_INTERVAL; /* Okay, we should downrate the data. By how much? */ - while (stability_last_downrated + STABILITY_INTERVAL < now) { + while (stability_last_downrated + STABILITY_INTERVAL <= now) { stability_last_downrated += STABILITY_INTERVAL; alpha *= STABILITY_ALPHA; } @@ -1787,11 +1998,18 @@ rep_hist_note_desc_served(const char * desc) /** Internal statistics to track how many requests of each type of * handshake we've received, and how many we've assigned to cpuworkers. * Useful for seeing trends in cpu load. + * + * They are reset at every heartbeat. * @{ */ STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ +/** Counters keeping the same stats as above but for the entire duration of the + * process (not reset). */ +static uint64_t stats_n_onionskin_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +static uint64_t stats_n_onionskin_dropped[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; + /** A new onionskin (using the <b>type</b> handshake) has arrived. */ void rep_hist_note_circuit_handshake_requested(uint16_t type) @@ -1805,8 +2023,20 @@ rep_hist_note_circuit_handshake_requested(uint16_t type) void rep_hist_note_circuit_handshake_assigned(uint16_t type) { - if (type <= MAX_ONION_HANDSHAKE_TYPE) + if (type <= MAX_ONION_HANDSHAKE_TYPE) { onion_handshakes_assigned[type]++; + stats_n_onionskin_assigned[type]++; + } +} + +/** We've just drop an onionskin (using the <b>type</b> handshake) due to being + * overloaded. */ +void +rep_hist_note_circuit_handshake_dropped(uint16_t type) +{ + if (type <= MAX_ONION_HANDSHAKE_TYPE) { + stats_n_onionskin_dropped[type]++; + } } /** Get the circuit handshake value that is requested. */ @@ -1829,6 +2059,26 @@ rep_hist_get_circuit_handshake_assigned, (uint16_t type)) return onion_handshakes_assigned[type]; } +/** Get the total number of circuit handshake value that is assigned. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_n_handshake_assigned, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return stats_n_onionskin_assigned[type]; +} + +/** Get the total number of circuit handshake value that is dropped. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_n_handshake_dropped, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return stats_n_onionskin_dropped[type]; +} + /** Log our onionskin statistics since the last time we were called. */ void rep_hist_log_circuit_handshake_stats(time_t now) diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index d4a2f301cf..7f414de4c8 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -58,11 +58,17 @@ time_t rep_hist_desc_stats_write(time_t now); void rep_hist_note_circuit_handshake_requested(uint16_t type); void rep_hist_note_circuit_handshake_assigned(uint16_t type); +void rep_hist_note_circuit_handshake_dropped(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); MOCK_DECL(int, rep_hist_get_circuit_handshake_requested, (uint16_t type)); MOCK_DECL(int, rep_hist_get_circuit_handshake_assigned, (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_n_handshake_assigned, + (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_n_handshake_dropped, + (uint16_t type)); + void rep_hist_hs_stats_init(time_t now); void rep_hist_hs_stats_term(void); time_t rep_hist_hs_stats_write(time_t now, bool is_v3); @@ -78,6 +84,11 @@ void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here); void rep_hist_log_link_protocol_counts(void); +uint64_t rep_hist_get_n_dns_error(int type, uint8_t error); +uint64_t rep_hist_get_n_dns_request(int type); +void rep_hist_note_dns_request(int type); +void rep_hist_note_dns_error(int type, uint8_t error); + extern uint64_t rephist_total_alloc; extern uint32_t rephist_total_num; #ifdef TOR_UNIT_TESTS @@ -159,6 +170,12 @@ void rep_hist_note_overload(overload_type_t overload); char *rep_hist_get_overload_general_line(void); char *rep_hist_get_overload_stats_lines(void); +void rep_hist_note_tcp_exhaustion(void); +uint64_t rep_hist_get_n_tcp_exhaustion(void); + +uint64_t rep_hist_get_n_read_limit_reached(void); +uint64_t rep_hist_get_n_write_limit_reached(void); + #ifdef TOR_UNIT_TESTS struct hs_v2_stats_t; const struct hs_v2_stats_t *rep_hist_get_hs_v2_stats(void); diff --git a/src/lib/metrics/metrics_common.c b/src/lib/metrics/metrics_common.c index 5941a4d892..f3f7e22d88 100644 --- a/src/lib/metrics/metrics_common.c +++ b/src/lib/metrics/metrics_common.c @@ -11,6 +11,7 @@ #include "orconfig.h" #include "lib/log/util_bug.h" +#include "lib/string/printf.h" #include "lib/metrics/metrics_common.h" @@ -27,3 +28,15 @@ metrics_type_to_str(const metrics_type_t type) tor_assert_unreached(); } } + +/** Return a static buffer pointer that contains a formatted label on the form + * of key=value. + * + * Subsequent call to this function invalidates the previous buffer. */ +const char * +metrics_format_label(const char *key, const char *value) +{ + static char buf[128]; + tor_snprintf(buf, sizeof(buf), "%s=\"%s\"", key, value); + return buf; +} diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h index 59aa9c0e90..3644ad3d50 100644 --- a/src/lib/metrics/metrics_common.h +++ b/src/lib/metrics/metrics_common.h @@ -42,4 +42,7 @@ typedef struct metrics_gauge_t { const char *metrics_type_to_str(const metrics_type_t type); +/* Helpers. */ +const char *metrics_format_label(const char *key, const char *value); + #endif /* !defined(TOR_LIB_METRICS_METRICS_COMMON_H) */ diff --git a/src/lib/metrics/metrics_store.c b/src/lib/metrics/metrics_store.c index 4cab5245f3..33b1780438 100644 --- a/src/lib/metrics/metrics_store.c +++ b/src/lib/metrics/metrics_store.c @@ -34,7 +34,8 @@ struct metrics_store_t { }; /** Function pointer to the format function of a specific driver. */ -typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *); +typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *, + bool no_comment); /** Helper: Free a single entry in a metrics_store_t taking a void pointer * parameter. */ @@ -47,6 +48,8 @@ metrics_store_free_void(void *p) smartlist_free(list); } +#include <stdio.h> + /** Put the given store output in the buffer data and use the format function * given in fmt to get it for each entry. */ static void @@ -57,8 +60,11 @@ get_output(const metrics_store_t *store, buf_t *data, fmt_driver_fn_t fmt) tor_assert(fmt); STRMAP_FOREACH(store->entries, key, const smartlist_t *, entries) { + /* Indicate that we've formatted the coment already for the entries. */ + bool comment_formatted = false; SMARTLIST_FOREACH_BEGIN(entries, const metrics_store_entry_t *, entry) { - fmt(entry, data); + fmt(entry, data, comment_formatted); + comment_formatted = true; } SMARTLIST_FOREACH_END(entry); } STRMAP_FOREACH_END; } @@ -138,3 +144,14 @@ metrics_store_get_output(const metrics_format_t fmt, // LCOV_EXCL_STOP } } + +/** Reset a store as in free its content. */ +void +metrics_store_reset(metrics_store_t *store) +{ + if (store == NULL) { + return; + } + strmap_free(store->entries, metrics_store_free_void); + store->entries = strmap_new(); +} diff --git a/src/lib/metrics/metrics_store.h b/src/lib/metrics/metrics_store.h index 42bc56e8fd..d85f484bd6 100644 --- a/src/lib/metrics/metrics_store.h +++ b/src/lib/metrics/metrics_store.h @@ -28,6 +28,7 @@ metrics_store_t *metrics_store_new(void); metrics_store_entry_t *metrics_store_add(metrics_store_t *store, metrics_type_t type, const char *name, const char *help); +void metrics_store_reset(metrics_store_t *store); /* Accessors. */ smartlist_t *metrics_store_get_all(const metrics_store_t *store, diff --git a/src/lib/metrics/prometheus.c b/src/lib/metrics/prometheus.c index 65241ed6c1..aac23ac92e 100644 --- a/src/lib/metrics/prometheus.c +++ b/src/lib/metrics/prometheus.c @@ -42,14 +42,17 @@ format_labels(smartlist_t *labels) /** Format the given entry in to the buffer data. */ void -prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data) +prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data, + bool no_comment) { tor_assert(entry); tor_assert(data); - buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); - buf_add_printf(data, "# TYPE %s %s\n", entry->name, - metrics_type_to_str(entry->type)); + if (!no_comment) { + buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); + buf_add_printf(data, "# TYPE %s %s\n", entry->name, + metrics_type_to_str(entry->type)); + } buf_add_printf(data, "%s%s %" PRIi64 "\n", entry->name, format_labels(entry->labels), metrics_store_entry_get_value(entry)); diff --git a/src/lib/metrics/prometheus.h b/src/lib/metrics/prometheus.h index 19770e7911..faa7681daa 100644 --- a/src/lib/metrics/prometheus.h +++ b/src/lib/metrics/prometheus.h @@ -13,6 +13,6 @@ #include "lib/metrics/metrics_store_entry.h" void prometheus_format_store_entry(const metrics_store_entry_t *entry, - buf_t *data); + buf_t *data, bool no_comment); #endif /* !defined(TOR_LIB_METRICS_PROMETHEUS_H) */ diff --git a/src/test/fuzz/fuzz_address.c b/src/test/fuzz/fuzz_address.c new file mode 100644 index 0000000000..6dccd65e9d --- /dev/null +++ b/src/test/fuzz/fuzz_address.c @@ -0,0 +1,26 @@ +#include "lib/net/address.h" +#include "lib/malloc/malloc.h" + +#include "test/fuzz/fuzzing.h" + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + tor_addr_t addr; + char *fuzzing_data = tor_memdup_nulterm(data, sz); + tor_addr_parse(&addr, fuzzing_data); + tor_free(fuzzing_data); + return 0; +} diff --git a/src/test/fuzz/fuzz_addressPTR.c b/src/test/fuzz/fuzz_addressPTR.c new file mode 100644 index 0000000000..b503d53666 --- /dev/null +++ b/src/test/fuzz/fuzz_addressPTR.c @@ -0,0 +1,32 @@ +#include "lib/net/address.h" +#include "lib/net/socket.h" +#include "lib/cc/ctassert.h" +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/log/log.h" +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/net/address.h" +#include "test/fuzz/fuzzing.h" + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + tor_addr_t addr_result; + char *fuzzing_data = tor_memdup_nulterm(data, sz); + tor_addr_parse_PTR_name(&addr_result, fuzzing_data, AF_UNSPEC, 1); + tor_free(fuzzing_data); + return 0; +} diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 9bdced9e6f..951eb04e6b 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -23,17 +23,36 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) # ===== AFL fuzzers if UNITTESTS_ENABLED +src_test_fuzz_fuzz_address_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_address.c +src_test_fuzz_fuzz_address_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_address_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_address_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_address_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_addressPTR_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_addressPTR.c +src_test_fuzz_fuzz_addressPTR_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_addressPTR_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_addressPTR_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_addressPTR_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_consensus_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_consensus.c @@ -155,6 +174,8 @@ endif if UNITTESTS_ENABLED FUZZERS = \ + src/test/fuzz/fuzz-address \ + src/test/fuzz/fuzz-addressPTR \ src/test/fuzz/fuzz-consensus \ src/test/fuzz/fuzz-descriptor \ src/test/fuzz/fuzz-diff \ @@ -173,6 +194,24 @@ endif if LIBFUZZER_ENABLED if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_address_SOURCES = \ + $(src_test_fuzz_fuzz_address_SOURCES) +src_test_fuzz_lf_fuzz_address_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_address_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_address_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_address_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_addressPTR_SOURCES = \ + $(src_test_fuzz_fuzz_addressPTR_SOURCES) +src_test_fuzz_lf_fuzz_addressPTR_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_addressPTR_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_addressPTR_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_addressPTR_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_consensus_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -281,6 +320,8 @@ src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS) endif LIBFUZZER_FUZZERS = \ + src/test/fuzz/lf-fuzz-address \ + src/test/fuzz/lf-fuzz-addressPTR \ src/test/fuzz/lf-fuzz-consensus \ src/test/fuzz/lf-fuzz-descriptor \ src/test/fuzz/lf-fuzz-diff \ @@ -302,6 +343,20 @@ endif if OSS_FUZZ_ENABLED if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_address_a_SOURCES = \ + $(src_test_fuzz_fuzz_address_SOURCES) +src_test_fuzz_liboss_fuzz_address_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_address_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_addressPTR_a_SOURCES = \ + $(src_test_fuzz_fuzz_addressPTR_SOURCES) +src_test_fuzz_liboss_fuzz_addressPTR_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_addressPTR_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -386,6 +441,8 @@ src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif OSS_FUZZ_FUZZERS = \ + src/test/fuzz/liboss-fuzz-address.a \ + src/test/fuzz/liboss-fuzz-addressPTR.a \ src/test/fuzz/liboss-fuzz-consensus.a \ src/test/fuzz/liboss-fuzz-descriptor.a \ src/test/fuzz/liboss-fuzz-diff.a \ diff --git a/src/test/include.am b/src/test/include.am index d5dcebfaea..b5e121d1ad 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -170,6 +170,7 @@ src_test_test_SOURCES += \ src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ + src/test/test_dirauth_ports.c \ src/test/test_dirvote.c \ src/test/test_dir_common.c \ src/test/test_dir_handle_get.c \ diff --git a/src/test/test.c b/src/test/test.c index 40c053a660..9543a24376 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1,5 +1,4 @@ /* Copyright (c) 2001-2004, Roger Dingledine. -->a * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2021, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -673,6 +672,7 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, + { "dir/auth/ports/", dirauth_port_tests }, { "dir/auth/process_descs/", process_descs_tests }, { "dir/md/", microdesc_tests }, { "dirauth/dirvote/", dirvote_tests}, diff --git a/src/test/test.h b/src/test/test.h index f88bc98498..700aa70a4b 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -120,6 +120,7 @@ extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; extern struct testcase_t crypto_rng_tests[]; extern struct testcase_t crypto_tests[]; +extern struct testcase_t dirauth_port_tests[]; extern struct testcase_t dir_handle_get_tests[]; extern struct testcase_t dir_tests[]; extern struct testcase_t dirvote_tests[]; diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 873391a84f..0a5c3530bd 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -113,7 +113,7 @@ test_new_route_len_safe_exit(void *arg) /* hidden service connecting to introduction point */ r = new_route_len(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, &dummy_ei, &dummy_nodes); - tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r); + tt_int_op(DEFAULT_ROUTE_LEN+1, OP_EQ, r); /* router testing its own reachability */ r = new_route_len(CIRCUIT_PURPOSE_TESTING, &dummy_ei, &dummy_nodes); diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 86baf54f40..6ced3f4111 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1367,7 +1367,7 @@ test_circuitpadding_wronghop(void *arg) tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); - tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); tt_int_op(n_client_cells, OP_EQ, 2); /* 6. Sending negotiated command to relay does nothing */ @@ -1396,11 +1396,9 @@ test_circuitpadding_wronghop(void *arg) /* verify no padding was negotiated */ tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); - tt_int_op(n_relay_cells, OP_EQ, 3); - tt_int_op(n_client_cells, OP_EQ, 2); /* verify no echo was sent */ - tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); tt_int_op(n_client_cells, OP_EQ, 2); /* Finish circuit */ diff --git a/src/test/test_connection.c b/src/test/test_connection.c index 9c726c07f8..87940f71e6 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -992,12 +992,12 @@ test_conn_describe(void *arg) #define STR(x) #x /* where arg is an expression (constant, variable, compound expression) */ -#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg) \ - { #name "_" STR(x), \ +#define CONNECTION_TESTCASE_ARG(name, extra, fork, setup, arg) \ + { STR(name)"/"extra, \ test_conn_##name, \ - fork, \ - &setup, \ - (void *)arg } + (fork), \ + &(setup), \ + (void *)(arg) } #endif /* !defined(COCCI) */ static const unsigned int PROXY_CONNECT_ARG = PROXY_CONNECT; @@ -1007,14 +1007,14 @@ struct testcase_t connection_tests[] = { CONNECTION_TESTCASE(get_basic, TT_FORK, test_conn_get_basic_st), CONNECTION_TESTCASE(get_rsrc, TT_FORK, test_conn_get_rsrc_st), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "microdesc", TT_FORK, test_conn_download_status_st, "microdesc"), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "ns", TT_FORK, test_conn_download_status_st, "ns"), - CONNECTION_TESTCASE_ARG(https_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(https_proxy_connect, "https", TT_FORK, test_conn_proxy_connect_st, &PROXY_CONNECT_ARG), - CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, "haproxy", TT_FORK, test_conn_proxy_connect_st, &PROXY_HAPROXY_ARG), //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair), diff --git a/src/test/test_dirauth_ports.c b/src/test/test_dirauth_ports.c new file mode 100644 index 0000000000..5dc0b0b631 --- /dev/null +++ b/src/test/test_dirauth_ports.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONFIG_PRIVATE + +#include "core/or/or.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/dirlist.h" +#include "app/config/config.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +static void +test_dirauth_port_parsing(void *arg) +{ + (void)arg; + + // This one is okay. + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/ " + "download=http://128.31.0.39:9131 " + "vote=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + + // These have bad syntax. + setup_capture_of_logs(LOG_WARN); + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "uploadx=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + expect_log_msg_containing("Unrecognized flag"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=https://128.31.0.39:9131/ " // https is not recognized + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL scheme"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/tor " // suffix is not supported + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL prefix"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.256:9131/ " // "256" is not ipv4. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://xyz.example.com/ " // hostnames not supported. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + done: + teardown_capture_of_logs(); +} + +static void +test_dirauth_port_lookup(void *arg) +{ + (void)arg; + + clear_dir_servers(); + + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.40:9132/ " + "download=http://128.31.0.41:9133 " + "vote=http://128.31.0.42:9134/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + rv = parse_dir_authority_line( + "morgoth orport=9101 " + "v3ident=D586D18309DED4CDFFFFFFFFDB97EFA96D330566 " + "upload=http://128.31.0.43:9140/ " + "128.31.0.44:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + const smartlist_t *servers = router_get_trusted_dir_servers(); + tt_assert(servers); + tt_int_op(smartlist_len(servers), OP_EQ, 2); + const dir_server_t *moria = smartlist_get(servers, 0); + const dir_server_t *morgoth = smartlist_get(servers, 1); + tt_str_op(moria->nickname, OP_EQ, "moria1"); + tt_str_op(morgoth->nickname, OP_EQ, "morgoth"); + + const tor_addr_port_t *dirport; + + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9132); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9133); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9134); + + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9140); + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + + done: + ; +} + +#define T(name) \ + { #name, test_dirauth_port_ ## name, TT_FORK, NULL, NULL } + +struct testcase_t dirauth_port_tests[] = { + T(parsing), + T(lookup), + END_OF_TESTCASES +}; diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index c94b5d6a23..118b66dfa7 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -92,6 +92,12 @@ bfn_mock_node_get_by_id(const char *id) return NULL; } +static int +mock_router_have_minimum_dir_info(void) +{ + return 1; +} + /* Helper function to free a test node. */ static void test_node_free(node_t *n) @@ -3087,6 +3093,38 @@ test_entry_guard_vanguard_path_selection(void *arg) circuit_free_(circ); } +static void +test_entry_guard_layer2_guards(void *arg) +{ + (void) arg; + MOCK(router_have_minimum_dir_info, mock_router_have_minimum_dir_info); + + /* First check the enable/disable switch */ + get_options_mutable()->VanguardsLiteEnabled = 0; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 0); + + get_options_mutable()->VanguardsLiteEnabled = 1; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 1); + + get_options_mutable()->VanguardsLiteEnabled = -1; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 1); + + /* OK now let's move to actual testing */ + + /* Remove restrictions to route around Big Fake Network restrictions */ + get_options_mutable()->EnforceDistinctSubnets = 0; + + /* Create the L2 guardset */ + maintain_layer2_guards(); + + const routerset_t *l2_guards = get_layer2_guards(); + tt_assert(l2_guards); + tt_int_op(routerset_len(l2_guards), OP_EQ, 4); + + done: + UNMOCK(router_have_minimum_dir_info); +} + static const struct testcase_setup_t big_fake_network = { big_fake_network_setup, big_fake_network_cleanup }; @@ -3152,6 +3190,8 @@ struct testcase_t entrynodes_tests[] = { BFN_TEST(manage_primary), BFN_TEST(correct_cascading_order), + BFN_TEST(layer2_guards), + EN_TEST_FORK(guard_preferred), BFN_TEST(select_for_circuit_no_confirmed), diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index b036c5eada..c32803b380 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -798,7 +798,7 @@ test_hs_control_add_onion_helper_add_service(void *arg) hs_service_ht *global_map; hs_port_config_t *portcfg; smartlist_t *portcfgs; - char *address_out_good, *address_out_bad; + char *address_out_good = NULL, *address_out_bad = NULL; hs_service_t *service_good = NULL; hs_service_t *service_bad = NULL; diff --git a/src/test/test_hs_ob.c b/src/test/test_hs_ob.c index 3485655c2e..2f69bf31e0 100644 --- a/src/test/test_hs_ob.c +++ b/src/test/test_hs_ob.c @@ -174,6 +174,7 @@ test_get_subcredentials(void *arg) hs_subcredential_t *subcreds = NULL; (void) arg; + memset(&config, 0, sizeof(config)); MOCK(networkstatus_get_live_consensus, mock_networkstatus_get_live_consensus); diff --git a/src/test/test_relay.c b/src/test/test_relay.c index 7338340e25..dbedc021e4 100644 --- a/src/test/test_relay.c +++ b/src/test/test_relay.c @@ -98,7 +98,7 @@ test_relay_close_circuit(void *arg) tt_int_op(new_count, OP_EQ, old_count + 1); /* Ensure our write totals are 0 */ - tt_u64_op(find_largest_max(write_array), OP_EQ, 0); + tt_u64_op(find_largest_max(write_array, 86400), OP_EQ, 0); /* Mark the circuit for close */ circuit_mark_for_close(TO_CIRCUIT(orcirc), 0); @@ -107,7 +107,7 @@ test_relay_close_circuit(void *arg) advance_obs(write_array); commit_max(write_array); /* Check for two cells plus overhead */ - tt_u64_op(find_largest_max(write_array), OP_EQ, + tt_u64_op(find_largest_max(write_array, 86400), OP_EQ, 2*(get_cell_network_size(nchan->wide_circ_ids) +TLS_PER_CELL_OVERHEAD)); diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 09ff8df4ab..ad52fdaa22 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -253,7 +253,7 @@ build_socks_resolve_request(uint8_t **out, } static void -onion_warning(const char *hostname) +onion_hs_warning(const char *hostname) { log_warn(LD_NET, "%s is a hidden service; those don't have IP addresses. " @@ -264,6 +264,15 @@ onion_warning(const char *hostname) hostname); } +static void +onion_exit_warning(const char *hostname) +{ + log_warn(LD_NET, + "%s is a link pointing to an exit node; however, .exit domains" + "have been long defunct and are not valid anymore.", + hostname); +} + /** Given a <b>len</b>-byte SOCKS4a response in <b>response</b>, set * *<b>addr_out</b> to the address it contains (in host order). * Return 0 on success, -1 on error. @@ -306,9 +315,15 @@ parse_socks4a_resolve_response(const char *hostname, if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); result = -1; goto cleanup; } + + if (!strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); + result = -1; goto cleanup; + } + result = -1; goto cleanup; } @@ -493,7 +508,11 @@ do_resolve(const char *hostname, (unsigned)reply_field, socks5_reason_to_string(reply_field)); if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); + } + + if (reply_field == 4 && !strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); } socks5_server_reply_free(reply); diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index cc70a3ab19..841d227a43 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -217,7 +217,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.6.7" +#define VERSION "0.4.7.0-alpha-dev" #define HAVE_STRUCT_SOCKADDR_IN6 #define HAVE_STRUCT_IN6_ADDR |