diff options
71 files changed, 2418 insertions, 187 deletions
@@ -1,3 +1,400 @@ +Changes in version 0.4.5.9 - 2021-06-14 + Tor 0.4.5.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (control, sandbox, backport from 0.4.6.4-rc): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port, backport from 0.4.6.4-rc): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + +Changes in version 0.4.4.9 - 2021-06-14 + Tor 0.4.4.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Note that the scheduled end-of-life date for the Tor 0.4.4.x series is + June 15. This is therefore the last release in its series. Everybody + still running 0.4.4.x should plan to upgrade to 0.4.5.x or later. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.15 - 2021-06-14 + Tor 0.3.5.15 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + +Changes in version 0.4.6.5 - 2021-06-14 + Tor 0.4.6.5 is the first stable release in its series. The 0.4.6.x + series includes numerous features and bugfixes, including a significant + improvement to our circuit timeout algorithm that should improve + observed client performance, and a way for relays to report when they are + overloaded. + + This release also includes security fixes for several security issues, + including a denial-of-service attack against onion service clients, + and another denial-of-service attack against relays. Everybody should + upgrade to one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Below are the changes since 0.4.6.4-rc. For a complete list of changes + since 0.4.5.8, see the ReleaseNotes file. + + o Major bugfixes (security): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor features (logging, diagnostic): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + +Changes in version 0.4.6.4-rc - 2021-05-28 + Tor 0.4.6.4-rc fixes a few bugs from previous releases. This, we hope, + the final release candidate in its series: unless major new issues are + found, the next release will be stable. + + o Minor features (compatibility): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor bugfixes (consensus handling): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (control, sandbox): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + o Removed features: + - Remove unneeded code for parsing private keys in directory + documents. This code was only used for client authentication in v2 + onion services, which are now unsupported. Closes ticket 40374. + + +Changes in version 0.4.5.8 - 2021-05-10 + Tor 0.4.5.8 fixes several bugs in earlier version, backporting fixes + from the 0.4.6.x series. + + o Minor features (compatibility, Linux seccomp sandbox, backport from 0.4.6.3-rc): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation, backport from 0.4.6.3-rc): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add warning message when connecting to now deprecated v2 onion + services. As announced, Tor 0.4.5.x is the last series that will + support v2 onions. Closes ticket 40373. + + o Minor bugfixes (bridge, pluggable transport, backport from 0.4.6.2-alpha): + - Fix a regression that made it impossible start Tor using a bridge + line with a transport name and no fingerprint. Fixes bug 40360; + bugfix on 0.4.5.4-rc. + + o Minor bugfixes (build, cross-compilation, backport from 0.4.6.3-rc): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (controller, backport from 0.4.6.1-alpha): + - Fix a "BUG" warning that would appear when a controller chooses + the first hop for a circuit, and that circuit completes. Fixes bug + 40285; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, client, memory leak, backport from 0.4.6.3-rc): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD, backport from 0.4.6.2-alpha): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + +Changes in version 0.4.6.3-rc - 2021-05-10 + Tor 0.4.6.3-rc is the first release candidate in its series. It fixes + a few small bugs from previous versions, and adds a better error + message when trying to use (no longer supported) v2 onion services. + + Though we anticipate that we'll be doing a bit more clean-up between + now and the stable release, we expect that our remaining changes will + be fairly simple. There will likely be at least one more release + candidate before 0.4.6.x is stable. + + o Major bugfixes (onion service, control port): + - Make the ADD_ONION command properly configure client authorization. + Before this fix, the created onion failed to add the client(s). + Fixes bug 40378; bugfix on 0.4.6.1-alpha. + + o Minor features (compatibility, Linux seccomp sandbox): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add a warning message when trying to connect to (no longer + supported) v2 onion services. Closes ticket 40373. + + o Minor bugfixes (build, cross-compilation): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (compiler warnings): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (logging, relay): + - Emit a warning if an Address is found to be internal and tor can't + use it. Fixes bug 40290; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (onion service, client, memory leak): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + Changes in version 0.4.6.2-alpha - 2021-04-15 Tor 0.4.6.2-alpha is the second alpha in its series. It fixes several small bugs in previous releases, and solves other issues that had diff --git a/Makefile.am b/Makefile.am index 3056b08446..7ae2133767 100644 --- a/Makefile.am +++ b/Makefile.am @@ -253,7 +253,7 @@ endif TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings if LIBFUZZER_ENABLED -TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div +TEST_CFLAGS += -fsanitize=fuzzer-no-link # not "edge" endif diff --git a/ReleaseNotes b/ReleaseNotes index 42017292c5..cc275a7452 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,566 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.4.5.9 - 2021-06-14 + Tor 0.4.5.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (control, sandbox, backport from 0.4.6.4-rc): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port, backport from 0.4.6.4-rc): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + +Changes in version 0.4.4.9 - 2021-06-14 + Tor 0.4.4.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Note that the scheduled end-of-life date for the Tor 0.4.4.x series is + June 15. This is therefore the last release in its series. Everybody + still running 0.4.4.x should plan to upgrade to 0.4.5.x or later. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.15 - 2021-06-14 + Tor 0.3.5.15 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + +Changes in version 0.4.6.5 - 2021-06-14 + Tor 0.4.6.5 is the first stable release in its series. The 0.4.6.x + series includes numerous features and bugfixes, including a significant + improvement to our circuit timeout algorithm that should improve + observed client performance, and a way for relays to report when they are + overloaded. + + This release also includes security fixes for several security issues, + including a denial-of-service attack against onion service clients, + and another denial-of-service attack against relays. Everybody should + upgrade to one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Below are the changes since 0.4.5.8. For a list of changes since + 0.4.6.4-rc, see the ChangeLog file. + + o Major bugfixes (security): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Major features (control port, onion services): + - Add controller support for creating version 3 onion services with + client authorization. Previously, only v2 onion services could be + created with client authorization. Closes ticket 40084. Patch by + Neel Chauhan. + + o Major features (directory authority): + - When voting on a relay with a Sybil-like appearance, add the Sybil + flag when clearing out the other flags. This lets a relay operator + know why their relay hasn't been included in the consensus. Closes + ticket 40255. Patch by Neel Chauhan. + + o Major features (metrics): + - Relays now report how overloaded they are in their extrainfo + documents. This information is controlled with the + OverloadStatistics torrc option, and it will be used to improve + decisions about the network's load balancing. Implements proposal + 328; closes ticket 40222. + + o Major features (relay, denial of service): + - Add a new DoS subsystem feature to control the rate of client + connections for relays. Closes ticket 40253. + + o Major features (statistics): + - Relays now publish statistics about the number of v3 onion + services and volume of v3 onion service traffic, in the same + manner they already do for v2 onions. Closes ticket 23126. + + o Major bugfixes (circuit build timeout): + - Improve the accuracy of our circuit build timeout calculation for + 60%, 70%, and 80% build rates for various guard choices. We now + use a maximum likelihood estimator for Pareto parameters of the + circuit build time distribution, instead of a "right-censored + estimator". This causes clients to ignore circuits that never + finish building in their timeout calculations. Previously, clients + were counting such unfinished circuits as having the highest + possible build time value, when in reality these circuits most + likely just contain relays that are offline. We also now wait a + bit longer to let circuits complete for measurement purposes, + lower the minimum possible effective timeout from 1.5 seconds to + 10ms, and increase the resolution of the circuit build time + histogram from 50ms bin widths to 10ms bin widths. Additionally, + we alter our estimate Xm by taking the maximum of the top 10 most + common build time values of the 10ms histogram, and compute Xm as + the average of these. Fixes bug 40168; bugfix on 0.2.2.14-alpha. + - Remove max_time calculation and associated warning from circuit + build timeout 'alpha' parameter estimation, as this is no longer + needed by our new estimator from 40168. Fixes bug 34088; bugfix + on 0.2.2.9-alpha. + + o Major bugfixes (signing key): + - In the tor-gencert utility, give an informative error message if + the passphrase given in `--create-identity-key` is too short. + Fixes bug 40189; bugfix on 0.2.0.1-alpha. Patch by Neel Chauhan. + + o Minor features (bridge): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (build system): + - New "make lsp" command to auto generate the compile_commands.json + file used by the ccls server. The "bear" program is needed for + this. Closes ticket 40227. + + o Minor features (client): + - Clients now check whether their streams are attempting to re-enter + the Tor network (i.e. to send Tor traffic over Tor), and close + them preemptively if they think exit relays will refuse them for + this reason. See ticket 2667 for details. Closes ticket 40271. + + o Minor features (command line): + - Add long format name "--torrc-file" equivalent to the existing + command-line option "-f". Closes ticket 40324. Patch by + Daniel Pinto. + + o Minor features (command-line interface): + - Add build informations to `tor --version` in order to ease + reproducible builds. Closes ticket 32102. + - When parsing command-line flags that take an optional argument, + treat the argument as absent if it would start with a '-' + character. Arguments in that form are not intelligible for any of + our optional-argument flags. Closes ticket 40223. + - Allow a relay operator to list the ed25519 keys on the command + line by adding the `rsa` and `ed25519` arguments to the + --list-fingerprint flag to show the respective RSA and ed25519 + relay fingerprint. Closes ticket 33632. Patch by Neel Chauhan. + + o Minor features (compatibility): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (control port, stream handling): + - Add the stream ID to the event line in the ADDRMAP control event. + Closes ticket 40249. Patch by Neel Chauhan. + + o Minor features (dormant mode): + - Add a new 'DormantTimeoutEnabled' option to allow coarse-grained + control over whether the client ever becomes dormant from + inactivity. Most people won't need this. Closes ticket 40228. + - Add a new 'DormantTimeoutEnabled' option for coarse-grained + control over whether the client can become dormant from + inactivity. Most people won't need this. Closes ticket 40228. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor features (logging): + - Edit heartbeat log messages so that more of them begin with the + string "Heartbeat: ". Closes ticket 40322; patch + from 'cypherpunks'. + - Change the DoS subsystem heartbeat line format to be more clear on + what has been detected/rejected, and which option is disabled (if + any). Closes ticket 40308. + - In src/core/mainloop/mainloop.c and src/core/mainloop/connection.c, + put brackets around IPv6 addresses in log messages. Closes ticket + 40232. Patch by Neel Chauhan. + + o Minor features (logging, diagnostic): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + o Minor features (onion services): + - Add a warning message when trying to connect to (no longer + supported) v2 onion services. Closes ticket 40373. + + o Minor features (performance, windows): + - Use SRWLocks to implement locking on Windows. Replaces the + "critical section" locking implementation with the faster + SRWLocks, available since Windows Vista. Closes ticket 17927. + Patch by Daniel Pinto. + + o Minor features (protocol, proxy support, defense in depth): + - Close HAProxy connections if they somehow manage to send us data + before we start reading. Closes another case of ticket 40017. + + o Minor features (tests, portability): + - Port the hs_build_address.py test script to work with recent + versions of python. Closes ticket 40213. Patch from + Samanta Navarro. + + o Minor features (vote document): + - Add a "stats" line to directory authority votes, to report various + statistics that authorities compute about the relays. This will + help us diagnose the network better. Closes ticket 40314. + + o Minor bugfixes (build): + - The configure script now shows whether or not lzma and zstd have + been used, not just if the enable flag was passed in. Fixes bug + 40236; bugfix on 0.4.3.1-alpha. + + o Minor bugfixes (compatibility): + - Fix a failure in the test cases when running on the "hppa" + architecture, along with a related test that might fail on other + architectures in the future. Fixes bug 40274; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (compilation): + - Fix a compilation warning about unused functions when building + with a libc that lacks the GLOB_ALTDIRFUNC constant. Fixes bug + 40354; bugfix on 0.4.5.1-alpha. Patch by Daniel Pinto. + + o Minor bugfixes (consensus handling): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (control, sandbox): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (directory authorities, voting): + - Add a new consensus method (31) to support any future changes that + authorities decide to make to the value of bwweightscale or + maxunmeasuredbw. Previously, there was a bug that prevented the + authorities from parsing these consensus parameters correctly under + most circumstances. Fixes bug 19011; bugfix on 0.2.2.10-alpha. + + o Minor bugfixes (ipv6): + - Allow non-SOCKSPorts to disable IPv4, IPv6, and PreferIPv4. Some + rare configurations might break, but in this case you can disable + NoIPv4Traffic and NoIPv6Traffic as needed. Fixes bug 33607; bugfix + on 0.4.1.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (key generation): + - Do not require a valid torrc when using the `--keygen` argument to + generate a signing key. This allows us to generate keys on systems + or users which may not run Tor. Fixes bug 40235; bugfix on + 0.2.7.2-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (logging, relay): + - Emit a warning if an Address is found to be internal and tor can't + use it. Fixes bug 40290; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (metrics port): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + o Minor bugfixes (onion services, logging): + - Downgrade the severity of a few rendezvous circuit-related + warnings from warning to info. Fixes bug 40207; bugfix on + 0.3.2.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. + This should reduce the CPU and memory burden for directory caches. + Fixes bug 40301; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + o Code simplification and refactoring: + - Remove the orconn_ext_or_id_map structure and related functions. + (Nothing outside of unit tests used them.) Closes ticket 33383. + Patch by Neel Chauhan. + + o Removed features: + - Remove unneeded code for parsing private keys in directory + documents. This code was only used for client authentication in v2 + onion services, which are now unsupported. Closes ticket 40374. + - As of this release, Tor no longer supports the old v2 onion + services. They were deprecated last July for security, and support + will be removed entirely later this year. We strongly encourage + everybody to migrate to v3 onion services. For more information, + see https://blog.torproject.org/v2-deprecation-timeline . Closes + ticket 40266. (NOTE: We accidentally released an earlier version + of the 0.4.6.1-alpha changelog without this entry. Sorry for + the confusion!) + + o Code simplification and refactoring (metrics, DoS): + - Move the DoS subsystem into the subsys manager, including its + configuration options. Closes ticket 40261. + + o Documentation (manual): + - Move the ServerTransport* options to the "SERVER OPTIONS" section. + Closes issue 40331. + - Indicate that the HiddenServiceStatistics option also applies to + bridges. Closes ticket 40346. + - Move the description of BridgeRecordUsageByCountry to the section + "STATISTICS OPTIONS". Closes ticket 40323. + + o Removed features (relay): + - Because DirPorts are only used on authorities, relays no longer + advertise them. Similarly, self-testing for DirPorts has been + disabled, since an unreachable DirPort is no reason for a relay + not to advertise itself. (Configuring a DirPort will still work, + for now.) Closes ticket 40282. + + +Changes in version 0.4.5.8 - 2021-05-10 + Tor 0.4.5.8 fixes several bugs in earlier version, backporting fixes + from the 0.4.6.x series. + + o Minor features (compatibility, Linux seccomp sandbox, backport from 0.4.6.3-rc): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation, backport from 0.4.6.3-rc): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add warning message when connecting to now deprecated v2 onion + services. As announced, Tor 0.4.5.x is the last series that will + support v2 onions. Closes ticket 40373. + + o Minor bugfixes (bridge, pluggable transport, backport from 0.4.6.2-alpha): + - Fix a regression that made it impossible start Tor using a bridge + line with a transport name and no fingerprint. Fixes bug 40360; + bugfix on 0.4.5.4-rc. + + o Minor bugfixes (build, cross-compilation, backport from 0.4.6.3-rc): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (controller, backport from 0.4.6.1-alpha): + - Fix a "BUG" warning that would appear when a controller chooses + the first hop for a circuit, and that circuit completes. Fixes bug + 40285; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, client, memory leak, backport from 0.4.6.3-rc): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD, backport from 0.4.6.2-alpha): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + Changes in version 0.3.5.14 - 2021-03-16 Tor 0.3.5.14 backports fixes for two important denial-of-service bugs in earlier versions of Tor. diff --git a/changes/bug40330 b/changes/bug40330 new file mode 100644 index 0000000000..8752ba5be7 --- /dev/null +++ b/changes/bug40330 @@ -0,0 +1,4 @@ + o Minor bugfixes (heartbeat): + - Adjust the heartbeat log message about distinct clients to consider + the HeartbeatPeriod rather than a flat 6-hour delay. + Fixes bug 40330; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug40365 b/changes/bug40365 new file mode 100644 index 0000000000..e4ee7b3b90 --- /dev/null +++ b/changes/bug40365 @@ -0,0 +1,3 @@ + o Minor bugfixes (tests): + - Fix a bug that prevented some tests from running with the correct names. + Fixes bug 40365; bugfix on 0.4.3.1-alpha. diff --git a/changes/bug40371 b/changes/bug40371 new file mode 100644 index 0000000000..8cc7117f9f --- /dev/null +++ b/changes/bug40371 @@ -0,0 +1,6 @@ + o Minor bugfixes (compatibility): + - Fix compatibility with the most recent Libevent versions, which + no longer have an evdns_set_random_bytes() function. Because + this function has been a no-op since Libevent 2.0.4-alpha, + it is safe for us to just stop calling it. Fixes bug 40371; + bugfix on 0.2.1.7-alpha. diff --git a/changes/bug40394 b/changes/bug40394 new file mode 100644 index 0000000000..f55167f2dd --- /dev/null +++ b/changes/bug40394 @@ -0,0 +1,4 @@ + o Minor bugfixes (statistics): + - Fix the fencepost issue when we check stability_last_downrated where + we call rep_hist_downrate_old_runs() twice. Fixes bug 40394; bugfix + on 0.2.0.5-alpha. Patch by Neel Chauhan. diff --git a/changes/bug40407 b/changes/bug40407 new file mode 100644 index 0000000000..068d278e14 --- /dev/null +++ b/changes/bug40407 @@ -0,0 +1,5 @@ + o Minor features (fuzzing): + - When building with --enable-libfuzzer, use a set of compiler flags + that works with more recent versions of the library. Previously + we were using a set of flags from 2017. + Closes ticket 40407. diff --git a/changes/ticket33742 b/changes/ticket33742 new file mode 100644 index 0000000000..3669e76f48 --- /dev/null +++ b/changes/ticket33742 @@ -0,0 +1,3 @@ + o Documentation: + - Add links to original tor design paper and anonbib to + docs/HACKING/README.1st.md. Closes ticket 33742. Patch from Emily Bones. diff --git a/changes/ticket40337 b/changes/ticket40337 new file mode 100644 index 0000000000..1c86fc4c99 --- /dev/null +++ b/changes/ticket40337 @@ -0,0 +1,16 @@ + o Minor features (testing): + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change + the smallest amount of time over which they're willing to report + their observed maximum bandwidth. Previously, this was fixed + at 1 day. For safety, values under 2 hours are only supported on + testing networks. Part of a fix for ticket 40337. + + o Minor features (testing): + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited + until they had been running for a full day. Closes ticket + 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a fix + for ticket 40337. diff --git a/configure.ac b/configure.ac index dff93a1fa3..912c2f2dfe 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.6.5-dev]) +AC_INIT([tor],[0.4.7.0-alpha-dev]) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -18,7 +18,7 @@ AC_DEFINE_UNQUOTED([CONFIG_FLAGS], ["$configure_flags"], [Flags passed to config # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2021-06-14"], # for 0.4.6.5-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2021-04-15"], # for 0.4.7.0-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 6d9a9d6cc6..1bf6554630 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.6.5-dev" +!define VERSION "0.4.7.0-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index cd3417d0b5..c5dd6c744f 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -59,7 +59,7 @@ Some compatible licenses include: Each main development series (like 0.2.1, 0.2.2, etc) has its main work applied to a single branch. At most one series can be the development series at a time; all other series are maintenance series that get bug-fixes only. -The development series is built in a git branch called "master"; the +The development series is built in a git branch called "main"; the maintenance series are built in branches called "maint-0.2.0", "maint-0.2.1", and so on. We regularly merge the active maint branches forward. @@ -75,7 +75,7 @@ If you're working on a bugfix for a bug that occurs in a particular version, base your bugfix branch on the "maint" branch for the first supported series that has that bug. (As of June 2013, we're supporting 0.2.3 and later.) -If you're working on a new feature, base it on the master branch. If you're +If you're working on a new feature, base it on the main branch. If you're working on a new feature and it will take a while to implement and/or you'd like to avoid the possibility of unrelated bugs in Tor while you're implementing your feature, consider branching off of the latest maint- branch. diff --git a/doc/HACKING/GettingStarted.md b/doc/HACKING/GettingStarted.md index 6d61be9881..271e2d7517 100644 --- a/doc/HACKING/GettingStarted.md +++ b/doc/HACKING/GettingStarted.md @@ -41,7 +41,7 @@ Once you've reached this point, here's what you need to know. $ git clone https://git.torproject.org/git/tor ``` - This will give you a checkout of the master branch. If you're + This will give you a checkout of the main branch. If you're going to fix a bug that appears in a stable version, check out the appropriate "maint" branch, as in: diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md index 0ce59576f0..7849fc67c7 100644 --- a/doc/HACKING/HelpfulTools.md +++ b/doc/HACKING/HelpfulTools.md @@ -33,8 +33,8 @@ OFTC. If they don't, ask #tor-dev (also on OFTC). It's CI/builders. Looks like this: https://jenkins.torproject.org Runs automatically on commits merged to git.torproject.org. We CI the -master branch and all supported tor versions. We also build nightly debian -packages from master. +main branch and all supported tor versions. We also build nightly debian +packages from main. Builds Linux and Windows cross-compilation. Runs Linux tests. diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md index 4d5a7f6b76..267f6d0b58 100644 --- a/doc/HACKING/Maintaining.md +++ b/doc/HACKING/Maintaining.md @@ -7,7 +7,7 @@ The first section describes who is the current Tor maintainer and what are the responsibilities. Tor has one main single maintainer but does have many committers and subsystem maintainers. -The second third section describes how the **alpha and master** branches are +The second third section describes how the **alpha and main** branches are maintained and by whom. Finally, the last section describes how the **stable** branches are maintained @@ -25,7 +25,7 @@ protocol design. Releasing Tor falls under their responsibility. ## Alpha and Master Branches -The Tor repository always has at all times a **master** branch which contains +The Tor repository always has at all times a **main** branch which contains the upstream ongoing development. It may also contain a branch for a released feature freezed version which is @@ -37,7 +37,7 @@ Tor is separated into subsystems and some of those are maintained by other developers than the main maintainer. Those people have commit access to the code base but only commit (in most cases) into the subsystem they maintain. -Upstream merges are restricted to the alpha and master branches. Subsystem +Upstream merges are restricted to the alpha and main branches. Subsystem maintainers should never push a patch into a stable branch which is the responsibility of the [stable branch maintainer](#stable-branches). @@ -69,7 +69,7 @@ maintain the following subsystems: These are the tasks of a subsystem maintainer: 1. Regularly go over `merge_ready` tickets relevant to the related subsystem - and for the current alpha or development (master branch) Milestone. + and for the current alpha or development (main branch) Milestone. 2. A subsystem maintainer is expected to contribute to any design changes (including proposals) or large patch set about the subsystem. @@ -102,7 +102,7 @@ These are few important items to follow when merging code upstream: 4. Tor uses the "merge forward" method, that is, if a patch applies to the alpha branch, it has to be merged there first and then merged forward - into master. + into main. 5. Maintainer should always consult with the network team about any doubts, mis-understandings or unknowns of a patch. Final word will always go to the diff --git a/doc/HACKING/README.1st.md b/doc/HACKING/README.1st.md index 4bc3298c67..06a24e8300 100644 --- a/doc/HACKING/README.1st.md +++ b/doc/HACKING/README.1st.md @@ -36,6 +36,13 @@ For the latest version of the code, get a copy of git, and $ git clone https://git.torproject.org/git/tor ``` +For a copy of Tor's original design paper, see +[here](https://spec.torproject.org/tor-design). Note that Tor has changed in +many ways since 2004. + +For a large collection of security papers, many of which are related to Tor, +see [Anonbib's Selected Papers in Anonymity](https://www.freehaven.net/anonbib/). + ## Stay in touch We talk about Tor on the `tor-talk` mailing list. Design proposals and diff --git a/doc/HACKING/ReleaseSeriesLifecycle.md b/doc/HACKING/ReleaseSeriesLifecycle.md index 8536fbbd08..e47ac90fa5 100644 --- a/doc/HACKING/ReleaseSeriesLifecycle.md +++ b/doc/HACKING/ReleaseSeriesLifecycle.md @@ -87,17 +87,17 @@ they do not apply to security-related patch release versions. (Ideally, do this immediately after a release.) -1. Start a new maint-x.y.z branch based on master, and a new - release-x.y.z branch based on master. They should have the same +1. Start a new maint-x.y.z branch based on main, and a new + release-x.y.z branch based on main. They should have the same starting point. - Push both of these branches to the master git repository. + Push both of these branches to the canonical git repository. -2. In master, change the version to "0.x.y.0-alpha-dev". Run the +2. In the main branch, change the version to "0.x.y.0-alpha-dev". Run the update_versions.py script, and commit this version bump. 3. Tag the version bump with "tor-0.x.y.0-alpha-dev". Push the tag - and master. + and main branch. 4. Open tickets for connecting the new branches to various other places. See section 2 above for a list of affected locations. @@ -107,7 +107,7 @@ they do not apply to security-related patch release versions. target in the maint-x.y.z branch only. * Delete the file scripts/maint/practracker/.enable_practracker_in_hooks in the maint-x.y.z branch only. - * Merge to release-x.y.z, but do not forward-port to master. + * Merge to release-x.y.z, but do not forward-port to the main branch. 6. Finally, make sure this document is up to date with our latest process. diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 739ea38795..8ee57b0cb5 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -1,3 +1,14 @@ +# CHECKLIST + +Here's a summary checklist, with the things that Nick messes up most often. + +Did you: + + * [ ] Copy the ChangeLog to the ReleaseNotes? + * [ ] Check that the new versions got approved? + * [ ] Check the release date in the ChangeLog? + * [ ] Update the GeoIP file? + # Putting out a new release Here are the steps that the maintainer should take when putting out a @@ -110,7 +121,7 @@ new Tor release: note added to each section. So in this case, once you have the items from the changes files copied together, don't use them to build a new changelog: instead, look up the corrected versions that were merged - into ChangeLog in the master branch, and use those. + into ChangeLog in the main branch, and use those. Add "backport from X.Y.Z" in the section header for these entries. @@ -142,7 +153,7 @@ new Tor release: places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. When you merge the maint branch forward to the next maint branch, or into - master, merge it with "-s ours" to avoid conflict with the version + main, merge it with "-s ours" to avoid conflict with the version bump. 2. Make distcheck, put the tarball up in somewhere (how about your @@ -222,13 +233,13 @@ $ git push origin tag tor-0.4.x.y-<status> 1. If it's a stable release, bump the version number in the `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` - merge to avoid taking that change into master. + merge to avoid taking that change into main. 2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that builds the release every week. (It's ok to skip the weekly build if the branch was updated in the last 24 hours.) 3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the - master branch. + main branch. 4. Keep an eye on the blog post, to moderate comments and answer questions. diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index 7222cd0548..89afe59582 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -3597,6 +3597,11 @@ The following options are used for running a testing Tor network. Minimum value for the Fast flag. Overrides the ordinary minimum taken from the consensus when TestingTorNetwork is set. (Default: 0.) +[[TestingMinTimeToReportBandwidth]] **TestingMinTimeToReportBandwidth** __N__ **seconds**|**minutes**|**hours**:: + Do not report our measurements for our maximum observed bandwidth for any + time period that has lasted for less than this amount of time. + Values over 1 day have no effect. (Default: 1 day) + [[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__:: Initial delay in seconds for when servers should download consensuses. Changing this requires that **TestingTorNetwork** is set. (Default: 0) diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index b52b956f81..ddad8e6c3b 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -11,11 +11,9 @@ FUZZERS = """ diff diff-apply extrainfo - hsdescv2 hsdescv3 http http-connect - iptsv2 microdesc socks strops @@ -47,11 +45,10 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh index 2bcd4722b4..62547dcc5d 100755 --- a/scripts/git/git-list-tor-branches.sh +++ b/scripts/git/git-list-tor-branches.sh @@ -73,8 +73,8 @@ branch() { # location: where the branch can be found. - if [[ "$brname" == "master" ]]; then - suffix="_master" + if [[ "$brname" == "main" ]]; then + suffix="_main" location="\$GIT_PATH/\$TOR_MASTER_NAME" elif [[ "$brname" =~ ^maint- ]]; then suffix="_${brname_nodots#maint-}" @@ -139,12 +139,12 @@ finish() { branch maint-0.3.5 branch release-0.3.5 -branch maint-0.4.4 -branch release-0.4.4 - branch maint-0.4.5 branch release-0.4.5 -branch master +branch maint-0.4.6 +branch release-0.4.6 + +branch main finish diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 7c72f8478d..d5d663d558 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -12,8 +12,8 @@ function usage() echo " (default: run commands)" echo " -t: test branch mode: create new branches from the commits checked" echo " out in each maint directory. Call these branches prefix_035," - echo " prefix_040, ... , prefix_master." - echo " (default: merge forward maint-*, release-*, and master)" + echo " prefix_040, ... , prefix_main." + echo " (default: merge forward maint-*, release-*, and main)" echo " -u: in test branch mode, if a prefix_* branch already exists," echo " skip creating that branch. Use after a merge error, to" echo " restart the merge forward at the first unmerged branch." @@ -28,7 +28,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -45,7 +45,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The main branch git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -106,7 +106,7 @@ DRY_RUN=0 # Controlled by the -t <test-branch-prefix> option. The test branch base # name option makes git-merge-forward.sh create new test branches: -# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward. +# <tbbn>_035, <tbbn>_040, ... , <tbbn>_main, and merge forward. TEST_BRANCH_PREFIX= # Controlled by the -u option. The use existing option checks for existing diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 52a5c6140c..bbe2576d8e 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -20,7 +20,7 @@ usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -37,7 +37,7 @@ usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -51,7 +51,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -159,20 +159,7 @@ function goto_repo function fetch_origin { local cmd="git fetch origin" - printf " %s Fetching origin..." "$MARKER" - if [ $DRY_RUN -eq 0 ]; then - msg=$( eval "$cmd" 2>&1 ) - validate_ret $? "$msg" - else - printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" - fi -} - -# Fetch tor-github pull requests. No arguments. -function fetch_tor_github -{ - local cmd="git fetch tor-github" - printf " %s Fetching tor-github..." "$MARKER" + printf "%s Fetching origin..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -185,7 +172,7 @@ function fetch_tor_github function fetch_tor_gitlab { local cmd="git fetch tor-gitlab" - printf " %s Fetching tor-gitlab..." "$MARKER" + printf "%s Fetching tor-gitlab..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -198,11 +185,10 @@ function fetch_tor_gitlab # Entry point # ############### -# First, fetch tor-github. +# Get into our origin repository. goto_repo "$ORIGIN_PATH" -fetch_tor_github -# Then tor-gitlab +# First, fetch tor-gitlab fetch_tor_gitlab # Then, fetch the origin. diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 558ea8d01c..e5c16e615f 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -21,10 +21,10 @@ function usage() echo " -r: push to remote-name, rather than the default upstream remote." echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" echo " -t: test branch mode: push test branches to remote-name. Pushes" - echo " branches prefix_035, prefix_040, ... , prefix_master." - echo " (default: push maint-*, release-*, and master)" + echo " branches prefix_035, prefix_040, ... , prefix_main." + echo " (default: push maint-*, release-*, and main)" echo " -s: push branches whose tips match upstream maint, release, or" - echo " master branches. The default is to skip these branches," + echo " main branches. The default is to skip these branches," echo " because they do not contain any new code. Use -s to test for" echo " CI environment failures, using code that previously passed CI." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" @@ -45,7 +45,7 @@ function usage() echo " (default: use the current directory for pushes;" echo " current: $TOR_FULL_GIT_PATH)" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." @@ -55,9 +55,9 @@ function usage() echo " Overridden by <git push options> after --." echo " (default: git push --atomic; current: $GIT_PUSH)" echo " TOR_PUSH_SAME: push branches whose tips match upstream maint," - echo " release, or master branches. Inverted by -s." + echo " release, or main branches. Inverted by -s." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" - echo " TOR_PUSH_DELAY: pushes the master and maint branches separately," + echo " TOR_PUSH_DELAY: pushes the main and maint branches separately," echo " so that CI runs in a sensible order." echo " (default: push all branches immediately; current: $PUSH_DELAY)" echo " we recommend that you set these env vars in your ~/.profile" @@ -71,7 +71,7 @@ set -e # Don't change this configuration - set the env vars in your .profile # -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # Which directory do we push from? @@ -87,7 +87,7 @@ UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} # Push (1) or skip (0) test branches that are the same as an upstream -# maint/master branch. Push if you are testing that the CI environment still +# maint/main branch. Push if you are testing that the CI environment still # works on old code, skip if you are testing new code in the branch. # Default: skip unchanged branches. # Inverted by the -s option. @@ -99,7 +99,7 @@ PUSH_SAME=${TOR_PUSH_SAME:-0} # Controlled by the -t <test-branch-prefix> option. The test branch prefix # option makes git-merge-forward.sh create new test branches: -# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into +# <tbp>_035, <tbp>_040, ... , <tbp>_main, and merge each branch forward into # the next one. TEST_BRANCH_PREFIX= @@ -213,7 +213,7 @@ if [ "$TEST_BRANCH_PREFIX" ]; then # upstream branches (they have already been tested) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" else - # Skip the local maint-*, release-*, master branches that are the same as the + # Skip the local maint-*, release-*, main branches that are the same as the # current upstream branches, but ignore the default upstream # (we want to update a non-default remote, even if it matches the default) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" @@ -264,8 +264,8 @@ if [ "$PUSH_DELAY" -le 0 ]; then else # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) - MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ - || true # Skipped master branch + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep main$) \ + || true # Skipped main branch if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ || true # Skipped all maint branches @@ -273,7 +273,7 @@ else tr "\\n" " ") || true # Skipped all release branches else # Actually test branches based on maint branches - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v main$) \ || true # Skipped all maint test branches # No release branches RELEASE_BRANCHES= @@ -295,9 +295,9 @@ else # shellcheck disable=SC2086 for b in $MASTER_BRANCH $MAINT_BRANCHES; do $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" - # If we are pushing more than one branch, delay. - # In the unlikely scenario where we are pushing maint without master, - # or maint without release, there may be an extra delay + # If we are pushing more than one branch, delay. In the unlikely scenario + # where we are pushing maint branches without the main branch, or maint + # without release, there may be an extra delay if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then sleep "$PUSH_DELAY" fi diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh index 5a9ae41cbd..c502f74f58 100755 --- a/scripts/git/git-setup-dirs.sh +++ b/scripts/git/git-setup-dirs.sh @@ -22,7 +22,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -65,7 +65,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -102,7 +102,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -519,11 +519,11 @@ for ((i=0; i<COUNT; i++)); do repo_path=${!WORKTREE[$i]:1:1} printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" - # We cloned the repository, and master is the default branch - if [ "$branch" = "master" ]; then - if [ "$TOR_MASTER_NAME" != "master" ]; then - # Set up a master link in the worktree directory - make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + # We cloned the repository, and main is the default branch + if [ "$branch" = "main" ]; then + if [ "$TOR_MASTER_NAME" != "main" ]; then + # Set up a main branch link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/main" fi else # git makes worktree directories if they don't exist diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index eae4f999e7..b458630d26 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -36,8 +36,8 @@ check_for_script_update() { } cur_branch=$(git rev-parse --abbrev-ref HEAD) -if [ "$cur_branch" != "master" ]; then - echo "post-merge: Not a master branch. Skipping." +if [ "$cur_branch" != "main" ]; then + echo "post-merge: Not a main branch. Skipping." exit 0 fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f0a3a250ec..0f016df592 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -2,9 +2,9 @@ # git pre-push hook script to: # 0) Call the pre-commit hook, if it is available -# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# 1) prevent "fixup!" and "squash!" commit from ending up in main, release-* # or maint-* -# 2) Disallow pushing branches other than master, release-* +# 2) Disallow pushing branches other than main, release-* # and maint-* to origin (e.g. gitweb.torproject.org) # # To install this script, copy it into .git/hooks/pre-push path in your @@ -39,7 +39,7 @@ remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || + if [ "$1" == "refs/heads/main" ] || [[ "$1" == refs/heads/release-* ]] || [[ "$1" == refs/heads/maint-* ]]; then return 1 @@ -54,8 +54,8 @@ do : else if [ "$remote_sha" = $z40 ]; then - # New branch, examine commits not in master - range="master...$local_sha" + # New branch, examine commits not in main + range="main...$local_sha" else # Update to existing branch, examine new commits range="$remote_sha..$local_sha" diff --git a/scripts/maint/findMergedChanges.pl b/scripts/maint/findMergedChanges.pl index d6c4105b74..9b3496403d 100755 --- a/scripts/maint/findMergedChanges.pl +++ b/scripts/maint/findMergedChanges.pl @@ -9,7 +9,7 @@ sub nChanges { # requires perl 5.8. Avoids shell issues if we ever get a changes # file named by the parents of Little Johnny Tables. open F, "-|", "git", "log", "--no-merges", "--pretty=format:%H", $branches, "--", $fname - or die "$!"; + or die "$!"; my @changes = <F>; return scalar @changes } @@ -22,13 +22,13 @@ Usage: findMergedChanges.pl [--merged/--unmerged/--weird/--list] [--branch=<branchname] [--head=<branchname>] changes/* A change is "merged" if it has ever been merged to release-0.2.4 and it has had -no subsequent changes in master. +no subsequent changes in main. A change is "unmerged" if it has never been merged to release-0.2.4 and it -has had changes in master. +has had changes in main. A change is "weird" if it has been merged to release-0.2.4 and it *has* had -subsequent changes in master. +subsequent changes in main. Suggested application: findMergedChanges.pl --merged changes/* | xargs -n 1 git rm @@ -37,18 +37,18 @@ EOF } my $target_branch = "origin/release-0.2.4"; -my $head = "origin/master"; +my $head = "origin/main"; while (@ARGV and $ARGV[0] =~ /^--/) { my $flag = shift @ARGV; if ($flag =~ /^--(weird|merged|unmerged|list)/) { - $look_for_type = $1; + $look_for_type = $1; } elsif ($flag =~ /^--branch=(\S+)/) { $target_branch = $1; } elsif ($flag =~ /^--head=(\S+)/) { $head = $1; } else { - die "Unrecognized flag $flag"; + die "Unrecognized flag $flag"; } } @@ -58,16 +58,16 @@ for my $changefile (@ARGV) { my $type; if ($n_merged != 0 and $n_postmerged == 0) { - $type = "merged"; + $type = "merged"; } elsif ($n_merged == 0 and $n_postmerged != 0) { - $type = "unmerged"; + $type = "unmerged"; } else { - $type = "weird"; + $type = "weird"; } if ($type eq $look_for_type) { - print "$changefile\n"; + print "$changefile\n"; } elsif ($look_for_type eq 'list') { - printf "% 8s: %s\n", $type, $changefile; + printf "% 8s: %s\n", $type, $changefile; } } diff --git a/src/app/config/config.c b/src/app/config/config.c index bfa258c904..7c92694178 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -550,6 +550,7 @@ static const config_var_t option_vars_[] = { V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"), VPORT(MetricsPort), V(MetricsPortPolicy, LINELIST, NULL), + V(TestingMinTimeToReportBandwidth, INTERVAL, "1 day"), VAR("MyFamily", LINELIST, MyFamily_lines, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), OBSOLETE("NamingAuthoritativeDirectory"), @@ -3980,6 +3981,7 @@ options_validate_cb(const void *old_options_, void *options_, char **msg) CHECK_DEFAULT(TestingSigningKeySlop); CHECK_DEFAULT(TestingAuthKeySlop); CHECK_DEFAULT(TestingLinkKeySlop); + CHECK_DEFAULT(TestingMinTimeToReportBandwidth); or_options_free(dflt_options); } #undef CHECK_DEFAULT @@ -5466,6 +5468,77 @@ pt_parse_transport_line(const or_options_t *options, return r; } +/** + * Parse a flag describing an extra dirport for a directory authority. + * + * Right now, the supported format is exactly: + * `{upload,download,voting}=http://[IP:PORT]/`. + * Other URL schemes, and other suffixes, might be supported in the future. + * + * Only call this function if `flag` starts with one of the above strings. + * + * Return 0 on success, and -1 on failure. + * + * If `ds` is provided, then add any parsed dirport to `ds`. If `ds` is NULL, + * take no action other than parsing. + **/ +static int +parse_dirauth_dirport(dir_server_t *ds, const char *flag) +{ + tor_assert(flag); + + auth_dirport_usage_t usage; + + if (!strcasecmpstart(flag, "upload=")) { + usage = AUTH_USAGE_UPLOAD; + } else if (!strcasecmpstart(flag, "download=")) { + usage = AUTH_USAGE_DOWNLOAD; + } else if (!strcasecmpstart(flag, "vote=")) { + usage = AUTH_USAGE_VOTING; + } else { + // We shouldn't get called with a flag that we don't recognize. + tor_assert_nonfatal_unreached(); + return -1; + } + + const char *eq = strchr(flag, '='); + tor_assert(eq); + const char *target = eq + 1; + + // Find the part inside the http://{....}/ + if (strcmpstart(target, "http://")) { + log_warn(LD_CONFIG, "Unsupported URL scheme in authority flag %s", flag); + return -1; + } + const char *addr = target + strlen("http://"); + + const char *eos = strchr(addr, '/'); + size_t addr_len; + if (eos && strcmp(eos, "/")) { + log_warn(LD_CONFIG, "Unsupported URL prefix in authority flag %s", flag); + return -1; + } else if (eos) { + addr_len = eos - addr; + } else { + addr_len = strlen(addr); + } + + // Finally, parse the addr:port part. + char *addr_string = tor_strndup(addr, addr_len); + tor_addr_port_t dirport; + memset(&dirport, 0, sizeof(dirport)); + int rv = tor_addr_port_parse(LOG_WARN, addr_string, + &dirport.addr, &dirport.port, -1); + if (ds != NULL && rv == 0) { + trusted_dir_server_add_dirport(ds, usage, &dirport); + } else if (rv == -1) { + log_warn(LD_CONFIG, "Unable to parse address in authority flag %s",flag); + } + + tor_free(addr_string); + return rv; +} + /** Read the contents of a DirAuthority line from <b>line</b>. If * <b>validate_only</b> is 0, and the line is well-formed, and it * shares any bits with <b>required_type</b> or <b>required_type</b> @@ -5486,6 +5559,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char v3_digest[DIGEST_LEN]; dirinfo_type_t type = 0; double weight = 1.0; + smartlist_t *extra_dirports = smartlist_new(); memset(v3_digest, 0, sizeof(v3_digest)); @@ -5554,6 +5628,12 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } ipv6_addrport_ptr = &ipv6_addrport; } + } else if (!strcasecmpstart(flag, "upload=") || + !strcasecmpstart(flag, "download=") || + !strcasecmpstart(flag, "vote=")) { + // We'll handle these after creating the authority object. + smartlist_add(extra_dirports, flag); + flag = NULL; // prevent double-free. } else { log_warn(LD_CONFIG, "Unrecognized flag '%s' on DirAuthority line", flag); @@ -5597,6 +5677,13 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, goto err; } + if (validate_only) { + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(NULL, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); + } + if (!validate_only && (!required_type || required_type & type)) { dir_server_t *ds; if (required_type) @@ -5608,16 +5695,23 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, ipv6_addrport_ptr, digest, v3_digest, type, weight))) goto err; + + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(ds, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); dir_server_add(ds); } r = 0; goto done; - err: + err: r = -1; - done: + done: + SMARTLIST_FOREACH(extra_dirports, char*, s, tor_free(s)); + smartlist_free(extra_dirports); SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 510ece42a3..151b77c457 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -1060,6 +1060,10 @@ struct or_options_t { /** List of policy allowed to query the Metrics port. */ struct config_line_t *MetricsPortPolicy; + /** How far must we be into the current bandwidth-measurement period to + * report bandwidth observations from this period? */ + int TestingMinTimeToReportBandwidth; + /** * Configuration objects for individual modules. * diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc index 00b307782b..039454a0d0 100644 --- a/src/app/config/testnet.inc +++ b/src/app/config/testnet.inc @@ -19,6 +19,7 @@ { "TestingV3AuthInitialDistDelay", "20 seconds" }, { "TestingAuthDirTimeToLearnReachability", "0 minutes" }, { "MinUptimeHidServDirectoryV2", "0 minutes" }, +{ "TestingMinTimeToReportBandwidth", "0 seconds" }, { "TestingServerDownloadInitialDelay", "0" }, { "TestingClientDownloadInitialDelay", "0" }, { "TestingServerConsensusDownloadInitialDelay", "0" }, diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index b17d7bf2bd..92755914ec 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -1261,7 +1261,7 @@ socket_failed_from_resource_exhaustion(void) */ if (get_max_sockets() > 65535) { /* TCP port exhaustion */ - rep_hist_note_overload(OVERLOAD_GENERAL); + rep_hist_note_tcp_exhaustion(); } else { /* File descriptor exhaustion */ rep_hist_note_overload(OVERLOAD_FD_EXHAUSTED); diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 2bcc642a97..f2f3bb5b34 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -2019,7 +2019,7 @@ cpath_build_state_to_crn_ipv6_extend_flag(const cpath_build_state_t *state, } /** Decide a suitable length for circ's cpath, and pick an exit - * router (or use <b>exit</b> if provided). Store these in the + * router (or use <b>exit_ei</b> if provided). Store these in the * cpath. * * If <b>is_hs_v3_rp_circuit</b> is set, then this exit should be suitable to @@ -2072,7 +2072,7 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, return 0; } -/** Give <b>circ</b> a new exit destination to <b>exit</b>, and add a +/** Give <b>circ</b> a new exit destination to <b>exit_ei</b>, and add a * hop to the cpath reflecting this. Don't send the next extend cell -- * the caller will do this if it wants to. */ @@ -2114,8 +2114,6 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) return -1; } - // XXX: Should cannibalized circuits be dirty or not? Not easy to say.. - return 0; } diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 4f62284e29..46be358dec 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -2586,8 +2586,10 @@ conns_compare_by_buffer_age_(const void **a_, const void **b_) /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ -void + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. + * + * Return the number of bytes removed. */ +size_t circuits_handle_oom(size_t current_allocation) { smartlist_t *circlist; @@ -2613,12 +2615,11 @@ circuits_handle_oom(size_t current_allocation) tor_zstd_get_total_allocation(), tor_lzma_get_total_allocation(), hs_cache_get_total_allocation()); - { size_t mem_target = (size_t)(get_options()->MaxMemInQueues * FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) - return; + return 0; mem_to_recover = current_allocation - mem_target; } @@ -2697,7 +2698,6 @@ circuits_handle_oom(size_t current_allocation) } SMARTLIST_FOREACH_END(circ); done_recovering_mem: - log_notice(LD_GENERAL, "Removed %"TOR_PRIuSZ" bytes by killing %d circuits; " "%d circuits remain alive. Also killed %d non-linked directory " "connections.", @@ -2705,6 +2705,8 @@ circuits_handle_oom(size_t current_allocation) n_circuits_killed, smartlist_len(circlist) - n_circuits_killed, n_dirconns_killed); + + return mem_recovered; } /** Verify that circuit <b>c</b> has all of its invariants diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index f5791d7c12..147e2cb2f8 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -232,7 +232,7 @@ int circuit_count_pending_on_channel(channel_t *chan); MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c)); void circuit_free_all(void); -void circuits_handle_oom(size_t current_allocation); +size_t circuits_handle_oom(size_t current_allocation); void circuit_clear_testing_cell_stats(circuit_t *circ); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index f5a9e73856..7462388f2c 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -2722,11 +2722,18 @@ cell_queues_get_total_allocation(void) /** The time at which we were last low on memory. */ static time_t last_time_under_memory_pressure = 0; +/** Statistics on how many bytes were removed by the OOM per type. */ +uint64_t oom_stats_n_bytes_removed_dns = 0; +uint64_t oom_stats_n_bytes_removed_cell = 0; +uint64_t oom_stats_n_bytes_removed_geoip = 0; +uint64_t oom_stats_n_bytes_removed_hsdir = 0; + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ STATIC int cell_queues_check_size(void) { + size_t removed = 0; time_t now = time(NULL); size_t alloc = cell_queues_get_total_allocation(); alloc += half_streams_get_total_allocation(); @@ -2751,20 +2758,27 @@ cell_queues_check_size(void) if (hs_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = hs_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= hs_cache_handle_oom(now, bytes_to_remove); + removed = hs_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_hsdir += removed; + alloc -= removed; } if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = geoip_client_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= geoip_client_cache_handle_oom(now, bytes_to_remove); + removed = geoip_client_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_geoip += removed; + alloc -= removed; } if (dns_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = dns_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= dns_cache_handle_oom(now, bytes_to_remove); + removed = dns_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_dns += removed; + alloc -= removed; } - circuits_handle_oom(alloc); + removed = circuits_handle_oom(alloc); + oom_stats_n_bytes_removed_cell += removed; return 1; } } diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 2f337d5d16..eac920f491 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -49,6 +49,11 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; +extern uint64_t oom_stats_n_bytes_removed_dns; +extern uint64_t oom_stats_n_bytes_removed_cell; +extern uint64_t oom_stats_n_bytes_removed_geoip; +extern uint64_t oom_stats_n_bytes_removed_hsdir; + void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index bd0d41d29e..2950a1c0cc 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -1075,7 +1075,7 @@ static const control_cmd_syntax_t redirectstream_syntax = { .max_args = UINT_MAX, // XXX should be 3. }; -/** Called when we receive a REDIRECTSTERAM command. Try to change the target +/** Called when we receive a REDIRECTSTREAM command. Try to change the target * address of the named AP stream, and report success or failure. */ static int handle_control_redirectstream(control_connection_t *conn, diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h index ed6b00647e..ac45f3787b 100644 --- a/src/feature/dirclient/dir_server_st.h +++ b/src/feature/dirclient/dir_server_st.h @@ -16,6 +16,8 @@ #include "core/or/or.h" #include "feature/nodelist/routerstatus_st.h" +struct smartlist_t; + /** Represents information about a single trusted or fallback directory * server. */ struct dir_server_t { @@ -48,6 +50,10 @@ struct dir_server_t { time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ + /** Authority only. Can be null. If present, a list of auth_dirport_t + * representing HTTP dirports for this authority. */ + struct smartlist_t *auth_dirports; + routerstatus_t fake_status; /**< Used when we need to pass this trusted * dir_server_t to * directory_request_set_routerstatus. diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index fd677d33fe..a2b20621a0 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -1119,6 +1119,7 @@ directory_request_set_routerstatus(directory_request_t *req, { req->routerstatus = status; } + /** * Helper: update the addresses, ports, and identities in <b>req</b> * from the routerstatus object in <b>req</b>. Return 0 on success. @@ -1161,7 +1162,7 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } - /* At this point, if we are a client making a direct connection to a + /* At this point, if we are a client making a direct connection to a * directory server, we have selected a server that has at least one address * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if @@ -1176,6 +1177,37 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } + /* One last thing: If we're talking to an authority, we might want to use + * a special HTTP port for it based on our purpose. + */ + if (req->indirection == DIRIND_DIRECT_CONN && status->is_authority) { + const dir_server_t *ds = router_get_trusteddirserver_by_digest( + status->identity_digest); + if (ds) { + const tor_addr_port_t *v4 = NULL; + if (authdir_mode_v3(get_options())) { + // An authority connecting to another authority should always + // prefer the VOTING usage, if one is specifically configured. + v4 = trusted_dir_server_get_dirport_exact( + ds, AUTH_USAGE_VOTING, AF_INET); + } + if (! v4) { + // Everybody else should prefer a usage dependent on their + // the dir_purpose. + auth_dirport_usage_t usage = + auth_dirport_usage_for_purpose(req->dir_purpose); + v4 = trusted_dir_server_get_dirport(ds, usage, AF_INET); + } + tor_assert_nonfatal(v4); + if (v4) { + // XXXX We could, if we wanted, also select a v6 address. But a v4 + // address must exist here, and we as a relay are required to support + // ipv4. So we just that. + tor_addr_port_copy(&use_dir_ap, v4); + } + } + } + directory_request_set_or_addr_port(req, &use_or_ap); directory_request_set_dir_addr_port(req, &use_dir_ap); directory_request_set_directory_id_digest(req, status->identity_digest); @@ -1194,7 +1226,7 @@ directory_initiate_request,(directory_request_t *request)) tor_assert_nonfatal( ! directory_request_dir_contact_info_specified(request)); if (directory_request_set_dir_from_routerstatus(request) < 0) { - return; + return; // or here XXXX } } @@ -1309,6 +1341,8 @@ directory_initiate_request,(directory_request_t *request)) entry_guard_cancel(&guard_state); } + // XXXX This is the case where we replace. + switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: diff --git a/src/feature/hs/hs_metrics.c b/src/feature/hs/hs_metrics.c index e023eab90c..0f1824c51c 100644 --- a/src/feature/hs/hs_metrics.c +++ b/src/feature/hs/hs_metrics.c @@ -29,18 +29,6 @@ port_to_str(const uint16_t port) return buf; } -/** Return a static buffer pointer that contains a formatted label on the form - * of key=value. - * - * Subsequent call to this function invalidates the previous buffer. */ -static const char * -format_label(const char *key, const char *value) -{ - static char buf[128]; - tor_snprintf(buf, sizeof(buf), "%s=%s", key, value); - return buf; -} - /** Initialize a metrics store for the given service. * * Essentially, this goes over the base_metrics array and adds them all to the @@ -61,12 +49,12 @@ init_store(hs_service_t *service) /* Add labels to the entry. */ metrics_store_entry_add_label(entry, - format_label("onion", service->onion_address)); + metrics_format_label("onion", service->onion_address)); if (base_metrics[i].port_as_label && service->config.ports) { SMARTLIST_FOREACH_BEGIN(service->config.ports, const hs_port_config_t *, p) { metrics_store_entry_add_label(entry, - format_label("port", port_to_str(p->virtual_port))); + metrics_format_label("port", port_to_str(p->virtual_port))); } SMARTLIST_FOREACH_END(p); } } @@ -96,7 +84,7 @@ hs_metrics_update_by_service(const hs_metrics_key_t key, SMARTLIST_FOREACH_BEGIN(entries, metrics_store_entry_t *, entry) { if (port == 0 || metrics_store_entry_has_label(entry, - format_label("port", port_to_str(port)))) { + metrics_format_label("port", port_to_str(port)))) { metrics_store_entry_update(entry, n); break; } diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index 1f18bd71a2..1f1ac4d106 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -43,6 +43,14 @@ #include "feature/dirclient/dir_server_st.h" #include "feature/nodelist/node_st.h" +/** Information about an (HTTP) dirport for a directory authority. */ +struct auth_dirport_t { + /** What is the intended usage for this dirport? One of AUTH_USAGE_* */ + auth_dirport_usage_t usage; + /** What is the correct address/port ? */ + tor_addr_port_t dirport; +}; + /** Global list of a dir_server_t object for each directory * authority. */ static smartlist_t *trusted_dir_servers = NULL; @@ -66,6 +74,11 @@ add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir) /* IPv6 DirPort is not a thing yet for authorities. */ nodelist_add_addr_to_address_set(&dir->ipv6_addr, dir->ipv6_orport, 0); } + if (dir->auth_dirports) { + SMARTLIST_FOREACH_BEGIN(dir->auth_dirports, const auth_dirport_t *, p) { + nodelist_add_addr_to_address_set(&p->dirport.addr, 0, p->dirport.port); + } SMARTLIST_FOREACH_END(p); + } } /** Go over the trusted directory server list and add their address(es) to the @@ -256,7 +269,10 @@ MOCK_IMPL(int, router_digest_is_trusted_dir_type, /** Return true iff the given address matches a trusted directory that matches * at least one bit of type. * - * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. */ + * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. + * + * Only ORPorts' addresses are considered. + */ bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) { @@ -281,6 +297,39 @@ router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) return false; } +/** Return an appropriate usage value describing which authdir port to use + * for a given directory connection purpose. + */ +auth_dirport_usage_t +auth_dirport_usage_for_purpose(int purpose) +{ + switch (purpose) { + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + case DIR_PURPOSE_FETCH_CONSENSUS: + case DIR_PURPOSE_FETCH_CERTIFICATE: + case DIR_PURPOSE_FETCH_MICRODESC: + return AUTH_USAGE_DOWNLOAD; + + case DIR_PURPOSE_UPLOAD_DIR: + return AUTH_USAGE_UPLOAD; + + case DIR_PURPOSE_UPLOAD_VOTE: + case DIR_PURPOSE_UPLOAD_SIGNATURES: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_STATUS_VOTE: + return AUTH_USAGE_VOTING; + + case DIR_PURPOSE_SERVER: + case DIR_PURPOSE_UPLOAD_HSDESC: + case DIR_PURPOSE_FETCH_HSDESC: + case DIR_PURPOSE_HAS_FETCHED_HSDESC: + default: + tor_assert_nonfatal_unreached(); + return AUTH_USAGE_LEGACY; + } +} + /** Create a directory server at <b>address</b>:<b>port</b>, with OR identity * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, * add ourself. If <b>is_authority</b>, this is a directory authority. Return @@ -357,6 +406,7 @@ dir_server_new(int is_authority, ent->fake_status.ipv4_dirport = ent->ipv4_dirport; ent->fake_status.ipv4_orport = ent->ipv4_orport; ent->fake_status.ipv6_orport = ent->ipv6_orport; + ent->fake_status.is_authority = !! is_authority; return ent; } @@ -404,10 +454,98 @@ trusted_dir_server_new(const char *nickname, const char *address, ipv6_addrport, digest, v3_auth_digest, type, weight); + + if (ipv4_dirport) { + tor_addr_port_t p; + memset(&p, 0, sizeof(p)); + tor_addr_copy(&p.addr, &ipv4_addr); + p.port = ipv4_dirport; + trusted_dir_server_add_dirport(result, AUTH_USAGE_LEGACY, &p); + } tor_free(hostname); return result; } +/** + * Add @a dirport as an HTTP DirPort contact point for the directory authority + * @a ds, for use when contacting that authority for the given @a usage. + * + * Multiple ports of the same usage are allowed; if present, then only + * the first one of each address family is currently used. + */ +void +trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport) +{ + tor_assert(ds); + tor_assert(dirport); + + if (BUG(! ds->is_authority)) { + return; + } + + if (ds->auth_dirports == NULL) { + ds->auth_dirports = smartlist_new(); + } + + auth_dirport_t *port = tor_malloc_zero(sizeof(auth_dirport_t)); + port->usage = usage; + tor_addr_port_copy(&port->dirport, dirport); + smartlist_add(ds->auth_dirports, port); +} + +/** + * Helper for trusted_dir_server_get_dirport: only return the exact requested + * usage type. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport_exact(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + tor_assert(ds); + tor_assert_nonfatal(addr_family == AF_INET || addr_family == AF_INET6); + if (ds->auth_dirports == NULL) + return NULL; + + SMARTLIST_FOREACH_BEGIN(ds->auth_dirports, const auth_dirport_t *, port) { + if (port->usage == usage && + tor_addr_family(&port->dirport.addr) == addr_family) { + return &port->dirport; + } + } SMARTLIST_FOREACH_END(port); + + return NULL; +} + +/** + * Return the DirPort of the authority @a ds for with the usage type + * @a usage and address family @a addr_family. If none is found, try + * again with an AUTH_USAGE_LEGACY dirport, if there is one. Return NULL + * if no port can be found. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + const tor_addr_port_t *port; + + while (1) { + port = trusted_dir_server_get_dirport_exact(ds, usage, addr_family); + if (port) + return port; + + // If we tried LEGACY, there is no fallback from this point. + if (usage == AUTH_USAGE_LEGACY) + return NULL; + + // Try again with LEGACY. + usage = AUTH_USAGE_LEGACY; + } +} + /** Return a new dir_server_t for a fallback directory server at * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest * <b>id_digest</b> */ @@ -447,6 +585,10 @@ dir_server_free_(dir_server_t *ds) if (!ds) return; + if (ds->auth_dirports) { + SMARTLIST_FOREACH(ds->auth_dirports, auth_dirport_t *, p, tor_free(p)); + smartlist_free(ds->auth_dirports); + } tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h index f744fecf92..3b4faf07af 100644 --- a/src/feature/nodelist/dirlist.h +++ b/src/feature/nodelist/dirlist.h @@ -11,6 +11,28 @@ #ifndef TOR_DIRLIST_H #define TOR_DIRLIST_H +typedef struct auth_dirport_t auth_dirport_t; +/** + * Different usages for an authority's HTTP directory port. + * + * Historically, only legacy ports existed; proposal 330 added multiple types + * of dirport to better enable authorities to offload work and resist DoS + * attacks. + **/ +typedef enum auth_dirport_usage_t { + /** Flag for an authority's dirport that is intended for misc/legacy + * usage. May be used when no other dirport is available. */ + AUTH_USAGE_LEGACY, + /** Flag for an authority's dirport that is intended for descriptor uploads + * only. */ + AUTH_USAGE_UPLOAD, + /** Flag for an authority's dirport that is intended for voting only */ + AUTH_USAGE_VOTING, + /** Flag for an authority's dirport that is intended for relay downloads + * only. */ + AUTH_USAGE_DOWNLOAD, +} auth_dirport_usage_t; + int get_n_authorities(dirinfo_type_t type); const smartlist_t *router_get_trusted_dir_servers(void); const smartlist_t *router_get_fallback_dir_servers(void); @@ -18,6 +40,8 @@ smartlist_t *router_get_trusted_dir_servers_mutable(void); smartlist_t *router_get_fallback_dir_servers_mutable(void); void mark_all_dirservers_up(smartlist_t *server_list); +auth_dirport_usage_t auth_dirport_usage_for_purpose(int purpose); + dir_server_t *router_get_trusteddirserver_by_digest(const char *d); dir_server_t *router_get_fallback_dirserver_by_digest( const char *digest); @@ -28,6 +52,14 @@ MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, MOCK_DECL(int, router_digest_is_trusted_dir_type, (const char *digest, dirinfo_type_t type)); +const tor_addr_port_t *trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); +const tor_addr_port_t *trusted_dir_server_get_dirport_exact( + const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); + bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type); #define router_addr_is_trusted_dir(d) \ @@ -41,6 +73,9 @@ dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, const tor_addr_port_t *addrport_ipv6, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight); +void trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport); dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, uint16_t dir_port, uint16_t or_port, const tor_addr_port_t *addrport_ipv6, diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 22f929808e..c6e0439338 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -212,20 +212,11 @@ evdns_log_cb(int warn, const char *msg) tor_log(severity, LD_EXIT, "eventdns: %s", msg); } -/** Helper: passed to eventdns.c as a callback so it can generate random - * numbers for transaction IDs and 0x20-hack coding. */ -static void -dns_randfn_(char *b, size_t n) -{ - crypto_rand(b,n); -} - /** Initialize the DNS subsystem; called by the OR process. */ int dns_init(void) { init_cache_map(); - evdns_set_random_bytes_fn(dns_randfn_); if (server_mode(get_options())) { int r = configure_nameservers(1); return r; @@ -1659,6 +1650,10 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses, dns_found_answer(string_address, orig_query_type, result, &addr, hostname, ttl); + /* The result can be changed within this function thus why we note the result + * at the end. */ + rep_hist_note_dns_error(type, result); + tor_free(arg_); } @@ -1677,6 +1672,9 @@ launch_one_resolve(const char *address, uint8_t query_type, addr[0] = (char) query_type; memcpy(addr+1, address, addr_len + 1); + /* Note the query for our statistics. */ + rep_hist_note_dns_request(query_type); + switch (query_type) { case DNS_IPv4_A: req = evdns_base_resolve_ipv4(the_evdns_base, diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am index 84bb1ff35e..8a121cef01 100644 --- a/src/feature/relay/include.am +++ b/src/feature/relay/include.am @@ -15,6 +15,7 @@ MODULE_RELAY_SOURCES = \ src/feature/relay/routermode.c \ src/feature/relay/relay_config.c \ src/feature/relay/relay_handshake.c \ + src/feature/relay/relay_metrics.c \ src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ src/feature/relay/routerkeys.c \ @@ -30,6 +31,7 @@ noinst_HEADERS += \ src/feature/relay/onion_queue.h \ src/feature/relay/relay_config.h \ src/feature/relay/relay_handshake.h \ + src/feature/relay/relay_metrics.h \ src/feature/relay/relay_periodic.h \ src/feature/relay/relay_sys.h \ src/feature/relay/relay_find_addr.h \ diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 85ec0dc74a..c09f4d5b9b 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -164,6 +164,7 @@ onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin) #define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + rep_hist_note_circuit_handshake_dropped(onionskin->handshake_type); if (onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR) { char *m; /* Note this ntor onionskin drop as an overload */ diff --git a/src/feature/relay/relay_metrics.c b/src/feature/relay/relay_metrics.c new file mode 100644 index 0000000000..1c5d1fd7cc --- /dev/null +++ b/src/feature/relay/relay_metrics.c @@ -0,0 +1,378 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.c + * @brief Relay metrics exposed through the MetricsPort + **/ + +#define RELAY_METRICS_ENTRY_PRIVATE + +#include "orconfig.h" + +#include "core/or/or.h" +#include "core/or/relay.h" + +#include "lib/malloc/malloc.h" +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_store.h" +#include "lib/log/util_bug.h" + +#include "feature/relay/relay_metrics.h" +#include "feature/stats/rephist.h" + +#include <event2/dns.h> + +/** Declarations of each fill function for metrics defined in base_metrics. */ +static void fill_dns_error_values(void); +static void fill_dns_query_values(void); +static void fill_global_bw_limit_values(void); +static void fill_socket_values(void); +static void fill_onionskins_values(void); +static void fill_oom_values(void); +static void fill_tcp_exhaustion_values(void); + +/** The base metrics that is a static array of metrics added to the metrics + * store. + * + * The key member MUST be also the index of the entry in the array. */ +static const relay_metrics_entry_t base_metrics[] = +{ + { + .key = RELAY_METRICS_NUM_OOM_BYTES, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_oom_bytes_total), + .help = "Total number of bytes the OOM has freed by subsystem", + .fill_fn = fill_oom_values, + }, + { + .key = RELAY_METRICS_NUM_ONIONSKINS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_onionskins_total), + .help = "Total number of onionskins handled", + .fill_fn = fill_onionskins_values, + }, + { + .key = RELAY_METRICS_NUM_SOCKETS, + .type = METRICS_TYPE_GAUGE, + .name = METRICS_NAME(relay_load_socket_total), + .help = "Total number of sockets", + .fill_fn = fill_socket_values, + }, + { + .key = RELAY_METRICS_NUM_GLOBAL_RW_LIMIT, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_global_rate_limit_reached_total), + .help = "Total number of global connection bucket limit reached", + .fill_fn = fill_global_bw_limit_values, + }, + { + .key = RELAY_METRICS_NUM_DNS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_query_total), + .help = "Total number of DNS queries done by this relay", + .fill_fn = fill_dns_query_values, + }, + { + .key = RELAY_METRICS_NUM_DNS_ERRORS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_error_total), + .help = "Total number of DNS errors encountered by this relay", + .fill_fn = fill_dns_error_values, + }, + { + .key = RELAY_METRICS_NUM_TCP_EXHAUSTION, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_tcp_exhaustion_total), + .help = "Total number of times we ran out of TCP ports", + .fill_fn = fill_tcp_exhaustion_values, + }, +}; +static const size_t num_base_metrics = ARRAY_LENGTH(base_metrics); + +/** The only and single store of all the relay metrics. */ +static metrics_store_t *the_store; + +/** Helper function to convert an handshake type into a string. */ +static inline const char * +handshake_type_to_str(const uint16_t type) +{ + switch (type) { + case ONION_HANDSHAKE_TYPE_TAP: + return "tap"; + case ONION_HANDSHAKE_TYPE_FAST: + return "fast"; + case ONION_HANDSHAKE_TYPE_NTOR: + return "ntor"; + default: + // LCOV_EXCL_START + tor_assert_unreached(); + // LCOV_EXCL_STOP + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_tcp_exhaustion_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_TCP_EXHAUSTION]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, rep_hist_get_n_tcp_exhaustion()); +} + +/** Helper array containing mapping for the name of the different DNS records + * and their corresponding libevent values. */ +static struct dns_type { + const char *name; + uint8_t type; +} dns_types[] = { + { .name = "A", .type = DNS_IPv4_A }, + { .name = "PTR", .type = DNS_PTR }, + { .name = "AAAA", .type = DNS_IPv6_AAAA }, +}; +static const size_t num_dns_types = ARRAY_LENGTH(dns_types); + +/** Fill function for the RELAY_METRICS_NUM_DNS_ERRORS metrics. */ +static void +fill_dns_error_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS_ERRORS]; + + /* Helper array to map libeven DNS errors to their names and so we can + * iterate over this array to add all metrics. */ + static struct dns_error { + const char *name; + uint8_t key; + } errors[] = { + { .name = "success", .key = DNS_ERR_NONE }, + { .name = "format", .key = DNS_ERR_FORMAT }, + { .name = "serverfailed", .key = DNS_ERR_SERVERFAILED }, + { .name = "notexist", .key = DNS_ERR_NOTEXIST }, + { .name = "notimpl", .key = DNS_ERR_NOTIMPL }, + { .name = "refused", .key = DNS_ERR_REFUSED }, + { .name = "truncated", .key = DNS_ERR_TRUNCATED }, + { .name = "unknown", .key = DNS_ERR_UNKNOWN }, + { .name = "timeout", .key = DNS_ERR_TIMEOUT }, + { .name = "shutdown", .key = DNS_ERR_SHUTDOWN }, + { .name = "cancel", .key = DNS_ERR_CANCEL }, + { .name = "nodata", .key = DNS_ERR_NODATA }, + }; + static const size_t num_errors = ARRAY_LENGTH(errors); + + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + + for (size_t j = 0; j < num_errors; j++) { + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("reason", errors[j].name)); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_error(dns_types[i].type, errors[j].key)); + } + tor_free(record_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_dns_query_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS]; + + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_request(dns_types[i].type)); + tor_free(record_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_GLOBAL_RW_LIMIT metrics. */ +static void +fill_global_bw_limit_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_GLOBAL_RW_LIMIT]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "read")); + metrics_store_entry_update(sentry, rep_hist_get_n_read_limit_reached()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "write")); + metrics_store_entry_update(sentry, rep_hist_get_n_write_limit_reached()); +} + +/** Fill function for the RELAY_METRICS_NUM_SOCKETS metrics. */ +static void +fill_socket_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_SOCKETS]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("state", "opened")); + metrics_store_entry_update(sentry, get_n_open_sockets()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, get_max_sockets()); +} + +/** Fill function for the RELAY_METRICS_NUM_ONIONSKINS metrics. */ +static void +fill_onionskins_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_ONIONSKINS]; + + for (uint16_t t = 0; t <= MAX_ONION_HANDSHAKE_TYPE; t++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *type_label = + tor_strdup(metrics_format_label("type", handshake_type_to_str(t))); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "processed")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_handshake_assigned(t)); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "dropped")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_handshake_dropped(t)); + tor_free(type_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_OOM_BYTES metrics. */ +static void +fill_oom_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_OOM_BYTES]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "cell")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_cell); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "dns")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_dns); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "geoip")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_geoip); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "hsdir")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_hsdir); +} + +/** Reset the global store and fill it with all the metrics from base_metrics + * and their associated values. + * + * To pull this off, every metrics has a "fill" function that is called and in + * charge of adding the metrics to the store, appropriate labels and finally + * updating the value to report. */ +static void +fill_store(void) +{ + /* Reset the current store, we are about to fill it with all the things. */ + metrics_store_reset(the_store); + + /* Call the fill function for each metrics. */ + for (size_t i = 0; i < num_base_metrics; i++) { + if (BUG(!base_metrics[i].fill_fn)) { + continue; + } + base_metrics[i].fill_fn(); + } +} + +/** Return a list of all the relay metrics stores. This is the + * function attached to the .get_metrics() member of the subsys_t. */ +const smartlist_t * +relay_metrics_get_stores(void) +{ + /* We can't have the caller to free the returned list so keep it static, + * simply update it. */ + static smartlist_t *stores_list = NULL; + + /* We dynamically fill the store with all the metrics upon a request. The + * reason for this is because the exposed metrics of a relay are often + * internal counters in the fast path and thus we fetch the value when a + * metrics port request arrives instead of keeping a local metrics store of + * those values. */ + fill_store(); + + if (!stores_list) { + stores_list = smartlist_new(); + smartlist_add(stores_list, the_store); + } + + return stores_list; +} + +/** Initialize the relay metrics. */ +void +relay_metrics_init(void) +{ + if (BUG(the_store)) { + return; + } + the_store = metrics_store_new(); +} + +/** Free the relay metrics. */ +void +relay_metrics_free(void) +{ + if (!the_store) { + return; + } + /* NULL is set with this call. */ + metrics_store_free(the_store); +} diff --git a/src/feature/relay/relay_metrics.h b/src/feature/relay/relay_metrics.h new file mode 100644 index 0000000000..00dfeaa624 --- /dev/null +++ b/src/feature/relay/relay_metrics.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.h + * @brief Header for feature/relay/relay_metrics.c + **/ + +#ifndef TOR_FEATURE_RELAY_RELAY_METRICS_H +#define TOR_FEATURE_RELAY_RELAY_METRICS_H + +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_common.h" + +/** Metrics key for each reported metrics. This key is also used as an index in + * the base_metrics array. */ +typedef enum { + /** Number of OOM invocation. */ + RELAY_METRICS_NUM_OOM_BYTES = 0, + /** Number of onionskines handled. */ + RELAY_METRICS_NUM_ONIONSKINS = 1, + /** Number of sockets. */ + RELAY_METRICS_NUM_SOCKETS = 2, + /** Number of global connection rate limit. */ + RELAY_METRICS_NUM_GLOBAL_RW_LIMIT = 3, + /** Number of DNS queries. */ + RELAY_METRICS_NUM_DNS = 4, + /** Number of DNS query errors. */ + RELAY_METRICS_NUM_DNS_ERRORS = 5, + /** Number of TCP exhaustion reached. */ + RELAY_METRICS_NUM_TCP_EXHAUSTION = 6, +} relay_metrics_key_t; + +/** The metadata of a relay metric. */ +typedef struct relay_metrics_entry_t { + /* Metric key used as a static array index. */ + relay_metrics_key_t key; + /* Metric type. */ + metrics_type_t type; + /* Metrics output name. */ + const char *name; + /* Metrics output help comment. */ + const char *help; + /* Update value function. */ + void (*fill_fn)(void); +} relay_metrics_entry_t; + +/* Init. */ +void relay_metrics_init(void); +void relay_metrics_free(void); + +/* Accessors. */ +const smartlist_t *relay_metrics_get_stores(void); + +#endif /* !defined(TOR_FEATURE_RELAY_RELAY_METRICS_H) */ diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c index 25fc0bbd32..9c43734b84 100644 --- a/src/feature/relay/relay_sys.c +++ b/src/feature/relay/relay_sys.c @@ -14,6 +14,7 @@ #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" +#include "feature/relay/relay_metrics.h" #include "feature/relay/onion_queue.h" #include "feature/relay/relay_periodic.h" #include "feature/relay/relay_sys.h" @@ -25,6 +26,7 @@ static int subsys_relay_initialize(void) { + relay_metrics_init(); relay_register_periodic_events(); return 0; } @@ -37,6 +39,7 @@ subsys_relay_shutdown(void) clear_pending_onions(); routerkeys_free_all(); router_free_all(); + relay_metrics_free(); } const struct subsys_fns_t sys_relay = { @@ -46,4 +49,6 @@ const struct subsys_fns_t sys_relay = { .level = RELAY_SUBSYS_LEVEL, .initialize = subsys_relay_initialize, .shutdown = subsys_relay_shutdown, + + .get_metrics = relay_metrics_get_stores, }; diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 67d3e3ee75..a2ca472307 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -2622,7 +2622,10 @@ check_descriptor_bandwidth_changed(time_t now) if ((prev != cur && (!prev || !cur)) || cur > (prev * BANDWIDTH_CHANGE_FACTOR) || cur < (prev / BANDWIDTH_CHANGE_FACTOR) ) { - if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) { + const bool change_recent_enough = + last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now; + const bool testing_network = get_options()->TestingTorNetwork; + if (change_recent_enough || testing_network || !prev) { log_info(LD_GENERAL, "Measured bandwidth has changed; rebuilding descriptor."); mark_my_descriptor_dirty("bandwidth has changed"); diff --git a/src/feature/stats/bwhist.c b/src/feature/stats/bwhist.c index d5a9b73605..552dc7ad74 100644 --- a/src/feature/stats/bwhist.c +++ b/src/feature/stats/bwhist.c @@ -206,16 +206,24 @@ bwhist_note_dir_bytes_read(uint64_t num_bytes, time_t when) add_obs(dir_read_array, when, num_bytes); } -/** Helper: Return the largest value in b->maxima. (This is equal to the +/** + * Helper: Return the largest value in b->maxima. (This is equal to the * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last * NUM_SECS_BW_SUM_IS_VALID seconds.) + * + * Also include the current period if we have been observing it for + * at least min_observation_time seconds. */ STATIC uint64_t -find_largest_max(bw_array_t *b) +find_largest_max(bw_array_t *b, int min_observation_time) { int i; uint64_t max; - max=0; + time_t period_start = b->next_period - NUM_SECS_BW_SUM_INTERVAL; + if (b->cur_obs_time > period_start + min_observation_time) + max = b->max_total; + else + max = 0; for (i=0; i<NUM_TOTALS; ++i) { if (b->maxima[i]>max) max = b->maxima[i]; @@ -233,8 +241,9 @@ MOCK_IMPL(int, bwhist_bandwidth_assess,(void)) { uint64_t w,r; - r = find_largest_max(read_array); - w = find_largest_max(write_array); + int min_obs_time = get_options()->TestingMinTimeToReportBandwidth; + r = find_largest_max(read_array, min_obs_time); + w = find_largest_max(write_array, min_obs_time); if (r>w) return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE); else diff --git a/src/feature/stats/bwhist.h b/src/feature/stats/bwhist.h index e7fc60fdee..d61c442e5d 100644 --- a/src/feature/stats/bwhist.h +++ b/src/feature/stats/bwhist.h @@ -28,7 +28,7 @@ int bwhist_load_state(struct or_state_t *state, char **err); #ifdef BWHIST_PRIVATE typedef struct bw_array_t bw_array_t; -STATIC uint64_t find_largest_max(bw_array_t *b); +STATIC uint64_t find_largest_max(bw_array_t *b, int min_observation_time); STATIC void commit_max(bw_array_t *b); STATIC void advance_obs(bw_array_t *b); STATIC bw_array_t *bw_array_new(void); diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index b4b107c3f7..a0fe8597c1 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -1206,11 +1206,11 @@ format_bridge_stats_controller(time_t now) char * format_client_stats_heartbeat(time_t now) { - const int n_hours = 6; + const int n_seconds = get_options()->HeartbeatPeriod; char *out = NULL; int n_clients = 0; clientmap_entry_t **ent; - unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + unsigned cutoff = (unsigned)( (now-n_seconds)/60 ); if (!start_of_bridge_stats_interval) return NULL; /* Not initialized. */ @@ -1226,8 +1226,7 @@ format_client_stats_heartbeat(time_t now) } tor_asprintf(&out, "Heartbeat: " - "In the last %d hours, I have seen %d unique clients.", - n_hours, + "Since last heartbeat message, I have seen %d unique clients.", n_clients); return out; diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index e25c01331d..59d5313cc8 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -84,6 +84,8 @@ #include "feature/nodelist/networkstatus_st.h" #include "core/or/or_circuit_st.h" +#include <event2/dns.h> + #ifdef HAVE_FCNTL_H #include <fcntl.h> #endif @@ -207,6 +209,183 @@ typedef struct { /** Current state of overload stats */ static overload_stats_t overload_stats; +/** Counters to count the number of times we've reached an overload for the + * global connection read/write limit. Reported on the MetricsPort. */ +static uint64_t stats_n_read_limit_reached = 0; +static uint64_t stats_n_write_limit_reached = 0; + +/** Total number of times we've reached TCP port exhaustion. */ +static uint64_t stats_n_tcp_exhaustion = 0; + +/***** DNS statistics *****/ + +/** Represents the statistics of DNS queries seen if it is an Exit. */ +typedef struct { + /* Total number of DNS errors found in RFC 1035 (from 0 to 5 code). */ + uint64_t stats_n_error_none; /* 0 */ + uint64_t stats_n_error_format; /* 1 */ + uint64_t stats_n_error_serverfailed; /* 2 */ + uint64_t stats_n_error_notexist; /* 3 */ + uint64_t stats_n_error_notimpl; /* 4 */ + uint64_t stats_n_error_refused; /* 5 */ + + /* Total number of DNS errors specific to libevent. */ + uint64_t stats_n_error_truncated; /* 65 */ + uint64_t stats_n_error_unknown; /* 66 */ + uint64_t stats_n_error_timeout; /* 67 */ + uint64_t stats_n_error_shutdown; /* 68 */ + uint64_t stats_n_error_cancel; /* 69 */ + uint64_t stats_n_error_nodata; /* 70 */ + + /* Total number of DNS request seen at an Exit. They might not all end + * successfully or might even be lost by tor. This counter is incremented + * right before the DNS request is initiated. */ + uint64_t stats_n_request; +} dns_stats_t; + +/** DNS statistics store for each DNS record type for which tor supports only + * three at the moment: A, PTR and AAAA. */ +static dns_stats_t dns_A_stats; +static dns_stats_t dns_PTR_stats; +static dns_stats_t dns_AAAA_stats; + +/** From a libevent record type, return a pointer to the corresponding DNS + * statistics store. NULL is returned if the type is unhandled. */ +static inline dns_stats_t * +get_dns_stats_by_type(const int type) +{ + switch (type) { + case DNS_IPv4_A: + return &dns_A_stats; + case DNS_PTR: + return &dns_PTR_stats; + case DNS_IPv6_AAAA: + return &dns_AAAA_stats; + default: + return NULL; + } +} + +/** Return the DNS error count for the given libevent DNS type and error code. + * The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_error(int type, uint8_t error) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; + } + + switch (error) { + case DNS_ERR_NONE: + return dns_stats->stats_n_error_none; + case DNS_ERR_FORMAT: + return dns_stats->stats_n_error_format; + case DNS_ERR_SERVERFAILED: + return dns_stats->stats_n_error_serverfailed; + case DNS_ERR_NOTEXIST: + return dns_stats->stats_n_error_notexist; + case DNS_ERR_NOTIMPL: + return dns_stats->stats_n_error_notimpl; + case DNS_ERR_REFUSED: + return dns_stats->stats_n_error_refused; + case DNS_ERR_TRUNCATED: + return dns_stats->stats_n_error_truncated; + case DNS_ERR_UNKNOWN: + return dns_stats->stats_n_error_unknown; + case DNS_ERR_TIMEOUT: + return dns_stats->stats_n_error_timeout; + case DNS_ERR_SHUTDOWN: + return dns_stats->stats_n_error_shutdown; + case DNS_ERR_CANCEL: + return dns_stats->stats_n_error_cancel; + case DNS_ERR_NODATA: + return dns_stats->stats_n_error_nodata; + default: + /* Unhandled code sent back by libevent. */ + return 0; + } +} + +/** Return the total number of DNS request seen for the given libevent DNS + * record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_request(int type) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; + } + return dns_stats->stats_n_request; +} + +/** Note a DNS error for the given given libevent DNS record type and error + * code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +void +rep_hist_note_dns_error(int type, uint8_t error) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + /* Unsupported DNS query type. */ + if (!dns_stats) { + return; + } + + switch (error) { + case DNS_ERR_NONE: + dns_stats->stats_n_error_none++; + break; + case DNS_ERR_FORMAT: + dns_stats->stats_n_error_format++; + break; + case DNS_ERR_SERVERFAILED: + dns_stats->stats_n_error_serverfailed++; + break; + case DNS_ERR_NOTEXIST: + dns_stats->stats_n_error_notexist++; + break; + case DNS_ERR_NOTIMPL: + dns_stats->stats_n_error_notimpl++; + break; + case DNS_ERR_REFUSED: + dns_stats->stats_n_error_refused++; + break; + case DNS_ERR_TRUNCATED: + dns_stats->stats_n_error_truncated++; + break; + case DNS_ERR_UNKNOWN: + dns_stats->stats_n_error_unknown++; + break; + case DNS_ERR_TIMEOUT: + dns_stats->stats_n_error_timeout++; + break; + case DNS_ERR_SHUTDOWN: + dns_stats->stats_n_error_shutdown++; + break; + case DNS_ERR_CANCEL: + dns_stats->stats_n_error_cancel++; + break; + case DNS_ERR_NODATA: + dns_stats->stats_n_error_nodata++; + break; + default: + /* Unhandled code sent back by libevent. */ + break; + } +} + +/** Note a DNS request for the given given libevent DNS record type. */ +void +rep_hist_note_dns_request(int type) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return; + } + dns_stats->stats_n_request++; +} + +/***** END of DNS statistics *****/ + /** Return true if this overload happened within the last `n_hours`. */ static bool overload_happened_recently(time_t overload_time, int n_hours) @@ -221,6 +400,20 @@ overload_happened_recently(time_t overload_time, int n_hours) /* The current version of the overload stats version */ #define OVERLOAD_STATS_VERSION 1 +/** Return the stats_n_read_limit_reached counter. */ +uint64_t +rep_hist_get_n_read_limit_reached(void) +{ + return stats_n_read_limit_reached; +} + +/** Return the stats_n_write_limit_reached counter. */ +uint64_t +rep_hist_get_n_write_limit_reached(void) +{ + return stats_n_write_limit_reached; +} + /** Returns an allocated string for server descriptor for publising information * on whether we are overloaded or not. */ char * @@ -299,6 +492,7 @@ rep_hist_note_overload(overload_type_t overload) SET_TO_START_OF_HOUR(overload_stats.overload_general_time); break; case OVERLOAD_READ: { + stats_n_read_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_read_counted + 60) { /* Count once a minute */ overload_stats.overload_read_count++; @@ -307,6 +501,7 @@ rep_hist_note_overload(overload_type_t overload) break; } case OVERLOAD_WRITE: { + stats_n_write_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_write_counted + 60) { /* Count once a minute */ overload_stats.overload_write_count++; @@ -321,6 +516,22 @@ rep_hist_note_overload(overload_type_t overload) } } +/** Note down that we've reached a TCP port exhaustion. This triggers an + * overload general event. */ +void +rep_hist_note_tcp_exhaustion(void) +{ + stats_n_tcp_exhaustion++; + rep_hist_note_overload(OVERLOAD_GENERAL); +} + +/** Return the total number of TCP exhaustion times we've reached. */ +uint64_t +rep_hist_get_n_tcp_exhaustion(void) +{ + return stats_n_tcp_exhaustion; +} + /** Return the or_history_t for the OR with identity digest <b>id</b>, * creating it if necessary. */ static or_history_t * @@ -520,7 +731,7 @@ rep_hist_downrate_old_runs(time_t now) return stability_last_downrated + STABILITY_INTERVAL; /* Okay, we should downrate the data. By how much? */ - while (stability_last_downrated + STABILITY_INTERVAL < now) { + while (stability_last_downrated + STABILITY_INTERVAL <= now) { stability_last_downrated += STABILITY_INTERVAL; alpha *= STABILITY_ALPHA; } @@ -1790,6 +2001,7 @@ rep_hist_note_desc_served(const char * desc) * @{ */ STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC uint64_t onion_handshakes_dropped[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ /** A new onionskin (using the <b>type</b> handshake) has arrived. */ @@ -1809,6 +2021,15 @@ rep_hist_note_circuit_handshake_assigned(uint16_t type) onion_handshakes_assigned[type]++; } +/** We've just drop an onionskin (using the <b>type</b> handshake) due to being + * overloaded. */ +void +rep_hist_note_circuit_handshake_dropped(uint16_t type) +{ + if (type <= MAX_ONION_HANDSHAKE_TYPE) + onion_handshakes_dropped[type]++; +} + /** Get the circuit handshake value that is requested. */ MOCK_IMPL(int, rep_hist_get_circuit_handshake_requested, (uint16_t type)) @@ -1829,6 +2050,16 @@ rep_hist_get_circuit_handshake_assigned, (uint16_t type)) return onion_handshakes_assigned[type]; } +/** Get the circuit handshake value that is dropped. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_handshake_dropped, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return onion_handshakes_dropped[type]; +} + /** Log our onionskin statistics since the last time we were called. */ void rep_hist_log_circuit_handshake_stats(time_t now) diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index d4a2f301cf..ca305dfd8d 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -58,10 +58,12 @@ time_t rep_hist_desc_stats_write(time_t now); void rep_hist_note_circuit_handshake_requested(uint16_t type); void rep_hist_note_circuit_handshake_assigned(uint16_t type); +void rep_hist_note_circuit_handshake_dropped(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); MOCK_DECL(int, rep_hist_get_circuit_handshake_requested, (uint16_t type)); MOCK_DECL(int, rep_hist_get_circuit_handshake_assigned, (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_handshake_dropped, (uint16_t type)); void rep_hist_hs_stats_init(time_t now); void rep_hist_hs_stats_term(void); @@ -78,11 +80,17 @@ void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here); void rep_hist_log_link_protocol_counts(void); +uint64_t rep_hist_get_n_dns_error(int type, uint8_t error); +uint64_t rep_hist_get_n_dns_request(int type); +void rep_hist_note_dns_request(int type); +void rep_hist_note_dns_error(int type, uint8_t error); + extern uint64_t rephist_total_alloc; extern uint32_t rephist_total_num; #ifdef TOR_UNIT_TESTS extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; +extern uint64_t onion_handshakes_dropped[MAX_ONION_HANDSHAKE_TYPE+1]; #endif #ifdef REPHIST_PRIVATE @@ -159,6 +167,12 @@ void rep_hist_note_overload(overload_type_t overload); char *rep_hist_get_overload_general_line(void); char *rep_hist_get_overload_stats_lines(void); +void rep_hist_note_tcp_exhaustion(void); +uint64_t rep_hist_get_n_tcp_exhaustion(void); + +uint64_t rep_hist_get_n_read_limit_reached(void); +uint64_t rep_hist_get_n_write_limit_reached(void); + #ifdef TOR_UNIT_TESTS struct hs_v2_stats_t; const struct hs_v2_stats_t *rep_hist_get_hs_v2_stats(void); diff --git a/src/lib/metrics/metrics_common.c b/src/lib/metrics/metrics_common.c index 5941a4d892..f3f7e22d88 100644 --- a/src/lib/metrics/metrics_common.c +++ b/src/lib/metrics/metrics_common.c @@ -11,6 +11,7 @@ #include "orconfig.h" #include "lib/log/util_bug.h" +#include "lib/string/printf.h" #include "lib/metrics/metrics_common.h" @@ -27,3 +28,15 @@ metrics_type_to_str(const metrics_type_t type) tor_assert_unreached(); } } + +/** Return a static buffer pointer that contains a formatted label on the form + * of key=value. + * + * Subsequent call to this function invalidates the previous buffer. */ +const char * +metrics_format_label(const char *key, const char *value) +{ + static char buf[128]; + tor_snprintf(buf, sizeof(buf), "%s=\"%s\"", key, value); + return buf; +} diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h index 59aa9c0e90..3644ad3d50 100644 --- a/src/lib/metrics/metrics_common.h +++ b/src/lib/metrics/metrics_common.h @@ -42,4 +42,7 @@ typedef struct metrics_gauge_t { const char *metrics_type_to_str(const metrics_type_t type); +/* Helpers. */ +const char *metrics_format_label(const char *key, const char *value); + #endif /* !defined(TOR_LIB_METRICS_METRICS_COMMON_H) */ diff --git a/src/lib/metrics/metrics_store.c b/src/lib/metrics/metrics_store.c index 4cab5245f3..33b1780438 100644 --- a/src/lib/metrics/metrics_store.c +++ b/src/lib/metrics/metrics_store.c @@ -34,7 +34,8 @@ struct metrics_store_t { }; /** Function pointer to the format function of a specific driver. */ -typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *); +typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *, + bool no_comment); /** Helper: Free a single entry in a metrics_store_t taking a void pointer * parameter. */ @@ -47,6 +48,8 @@ metrics_store_free_void(void *p) smartlist_free(list); } +#include <stdio.h> + /** Put the given store output in the buffer data and use the format function * given in fmt to get it for each entry. */ static void @@ -57,8 +60,11 @@ get_output(const metrics_store_t *store, buf_t *data, fmt_driver_fn_t fmt) tor_assert(fmt); STRMAP_FOREACH(store->entries, key, const smartlist_t *, entries) { + /* Indicate that we've formatted the coment already for the entries. */ + bool comment_formatted = false; SMARTLIST_FOREACH_BEGIN(entries, const metrics_store_entry_t *, entry) { - fmt(entry, data); + fmt(entry, data, comment_formatted); + comment_formatted = true; } SMARTLIST_FOREACH_END(entry); } STRMAP_FOREACH_END; } @@ -138,3 +144,14 @@ metrics_store_get_output(const metrics_format_t fmt, // LCOV_EXCL_STOP } } + +/** Reset a store as in free its content. */ +void +metrics_store_reset(metrics_store_t *store) +{ + if (store == NULL) { + return; + } + strmap_free(store->entries, metrics_store_free_void); + store->entries = strmap_new(); +} diff --git a/src/lib/metrics/metrics_store.h b/src/lib/metrics/metrics_store.h index 42bc56e8fd..d85f484bd6 100644 --- a/src/lib/metrics/metrics_store.h +++ b/src/lib/metrics/metrics_store.h @@ -28,6 +28,7 @@ metrics_store_t *metrics_store_new(void); metrics_store_entry_t *metrics_store_add(metrics_store_t *store, metrics_type_t type, const char *name, const char *help); +void metrics_store_reset(metrics_store_t *store); /* Accessors. */ smartlist_t *metrics_store_get_all(const metrics_store_t *store, diff --git a/src/lib/metrics/prometheus.c b/src/lib/metrics/prometheus.c index 65241ed6c1..aac23ac92e 100644 --- a/src/lib/metrics/prometheus.c +++ b/src/lib/metrics/prometheus.c @@ -42,14 +42,17 @@ format_labels(smartlist_t *labels) /** Format the given entry in to the buffer data. */ void -prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data) +prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data, + bool no_comment) { tor_assert(entry); tor_assert(data); - buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); - buf_add_printf(data, "# TYPE %s %s\n", entry->name, - metrics_type_to_str(entry->type)); + if (!no_comment) { + buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); + buf_add_printf(data, "# TYPE %s %s\n", entry->name, + metrics_type_to_str(entry->type)); + } buf_add_printf(data, "%s%s %" PRIi64 "\n", entry->name, format_labels(entry->labels), metrics_store_entry_get_value(entry)); diff --git a/src/lib/metrics/prometheus.h b/src/lib/metrics/prometheus.h index 19770e7911..faa7681daa 100644 --- a/src/lib/metrics/prometheus.h +++ b/src/lib/metrics/prometheus.h @@ -13,6 +13,6 @@ #include "lib/metrics/metrics_store_entry.h" void prometheus_format_store_entry(const metrics_store_entry_t *entry, - buf_t *data); + buf_t *data, bool no_comment); #endif /* !defined(TOR_LIB_METRICS_PROMETHEUS_H) */ diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 9bdced9e6f..de3ea5e74a 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -23,11 +23,10 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) diff --git a/src/test/include.am b/src/test/include.am index d5dcebfaea..b5e121d1ad 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -170,6 +170,7 @@ src_test_test_SOURCES += \ src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ + src/test/test_dirauth_ports.c \ src/test/test_dirvote.c \ src/test/test_dir_common.c \ src/test/test_dir_handle_get.c \ diff --git a/src/test/test.c b/src/test/test.c index 40c053a660..9543a24376 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1,5 +1,4 @@ /* Copyright (c) 2001-2004, Roger Dingledine. -->a * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2021, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -673,6 +672,7 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, + { "dir/auth/ports/", dirauth_port_tests }, { "dir/auth/process_descs/", process_descs_tests }, { "dir/md/", microdesc_tests }, { "dirauth/dirvote/", dirvote_tests}, diff --git a/src/test/test.h b/src/test/test.h index f88bc98498..700aa70a4b 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -120,6 +120,7 @@ extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; extern struct testcase_t crypto_rng_tests[]; extern struct testcase_t crypto_tests[]; +extern struct testcase_t dirauth_port_tests[]; extern struct testcase_t dir_handle_get_tests[]; extern struct testcase_t dir_tests[]; extern struct testcase_t dirvote_tests[]; diff --git a/src/test/test_connection.c b/src/test/test_connection.c index 9c726c07f8..87940f71e6 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -992,12 +992,12 @@ test_conn_describe(void *arg) #define STR(x) #x /* where arg is an expression (constant, variable, compound expression) */ -#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg) \ - { #name "_" STR(x), \ +#define CONNECTION_TESTCASE_ARG(name, extra, fork, setup, arg) \ + { STR(name)"/"extra, \ test_conn_##name, \ - fork, \ - &setup, \ - (void *)arg } + (fork), \ + &(setup), \ + (void *)(arg) } #endif /* !defined(COCCI) */ static const unsigned int PROXY_CONNECT_ARG = PROXY_CONNECT; @@ -1007,14 +1007,14 @@ struct testcase_t connection_tests[] = { CONNECTION_TESTCASE(get_basic, TT_FORK, test_conn_get_basic_st), CONNECTION_TESTCASE(get_rsrc, TT_FORK, test_conn_get_rsrc_st), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "microdesc", TT_FORK, test_conn_download_status_st, "microdesc"), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "ns", TT_FORK, test_conn_download_status_st, "ns"), - CONNECTION_TESTCASE_ARG(https_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(https_proxy_connect, "https", TT_FORK, test_conn_proxy_connect_st, &PROXY_CONNECT_ARG), - CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, "haproxy", TT_FORK, test_conn_proxy_connect_st, &PROXY_HAPROXY_ARG), //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair), diff --git a/src/test/test_dirauth_ports.c b/src/test/test_dirauth_ports.c new file mode 100644 index 0000000000..5dc0b0b631 --- /dev/null +++ b/src/test/test_dirauth_ports.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONFIG_PRIVATE + +#include "core/or/or.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/dirlist.h" +#include "app/config/config.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +static void +test_dirauth_port_parsing(void *arg) +{ + (void)arg; + + // This one is okay. + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/ " + "download=http://128.31.0.39:9131 " + "vote=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + + // These have bad syntax. + setup_capture_of_logs(LOG_WARN); + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "uploadx=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + expect_log_msg_containing("Unrecognized flag"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=https://128.31.0.39:9131/ " // https is not recognized + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL scheme"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/tor " // suffix is not supported + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL prefix"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.256:9131/ " // "256" is not ipv4. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://xyz.example.com/ " // hostnames not supported. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + done: + teardown_capture_of_logs(); +} + +static void +test_dirauth_port_lookup(void *arg) +{ + (void)arg; + + clear_dir_servers(); + + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.40:9132/ " + "download=http://128.31.0.41:9133 " + "vote=http://128.31.0.42:9134/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + rv = parse_dir_authority_line( + "morgoth orport=9101 " + "v3ident=D586D18309DED4CDFFFFFFFFDB97EFA96D330566 " + "upload=http://128.31.0.43:9140/ " + "128.31.0.44:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + const smartlist_t *servers = router_get_trusted_dir_servers(); + tt_assert(servers); + tt_int_op(smartlist_len(servers), OP_EQ, 2); + const dir_server_t *moria = smartlist_get(servers, 0); + const dir_server_t *morgoth = smartlist_get(servers, 1); + tt_str_op(moria->nickname, OP_EQ, "moria1"); + tt_str_op(morgoth->nickname, OP_EQ, "morgoth"); + + const tor_addr_port_t *dirport; + + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9132); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9133); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9134); + + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9140); + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + + done: + ; +} + +#define T(name) \ + { #name, test_dirauth_port_ ## name, TT_FORK, NULL, NULL } + +struct testcase_t dirauth_port_tests[] = { + T(parsing), + T(lookup), + END_OF_TESTCASES +}; diff --git a/src/test/test_relay.c b/src/test/test_relay.c index 7338340e25..dbedc021e4 100644 --- a/src/test/test_relay.c +++ b/src/test/test_relay.c @@ -98,7 +98,7 @@ test_relay_close_circuit(void *arg) tt_int_op(new_count, OP_EQ, old_count + 1); /* Ensure our write totals are 0 */ - tt_u64_op(find_largest_max(write_array), OP_EQ, 0); + tt_u64_op(find_largest_max(write_array, 86400), OP_EQ, 0); /* Mark the circuit for close */ circuit_mark_for_close(TO_CIRCUIT(orcirc), 0); @@ -107,7 +107,7 @@ test_relay_close_circuit(void *arg) advance_obs(write_array); commit_max(write_array); /* Check for two cells plus overhead */ - tt_u64_op(find_largest_max(write_array), OP_EQ, + tt_u64_op(find_largest_max(write_array, 86400), OP_EQ, 2*(get_cell_network_size(nchan->wide_circ_ids) +TLS_PER_CELL_OVERHEAD)); diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 09ff8df4ab..ad52fdaa22 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -253,7 +253,7 @@ build_socks_resolve_request(uint8_t **out, } static void -onion_warning(const char *hostname) +onion_hs_warning(const char *hostname) { log_warn(LD_NET, "%s is a hidden service; those don't have IP addresses. " @@ -264,6 +264,15 @@ onion_warning(const char *hostname) hostname); } +static void +onion_exit_warning(const char *hostname) +{ + log_warn(LD_NET, + "%s is a link pointing to an exit node; however, .exit domains" + "have been long defunct and are not valid anymore.", + hostname); +} + /** Given a <b>len</b>-byte SOCKS4a response in <b>response</b>, set * *<b>addr_out</b> to the address it contains (in host order). * Return 0 on success, -1 on error. @@ -306,9 +315,15 @@ parse_socks4a_resolve_response(const char *hostname, if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); result = -1; goto cleanup; } + + if (!strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); + result = -1; goto cleanup; + } + result = -1; goto cleanup; } @@ -493,7 +508,11 @@ do_resolve(const char *hostname, (unsigned)reply_field, socks5_reason_to_string(reply_field)); if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); + } + + if (reply_field == 4 && !strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); } socks5_server_reply_free(reply); diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 3929b91763..841d227a43 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -217,7 +217,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.6.5-dev" +#define VERSION "0.4.7.0-alpha-dev" #define HAVE_STRUCT_SOCKADDR_IN6 #define HAVE_STRUCT_IN6_ADDR |