diff options
285 files changed, 11267 insertions, 7131 deletions
diff --git a/.gitignore b/.gitignore index 19a82fb4b6..d3ac3bbc52 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,6 @@ core.* /autom4te.cache /build-stamp /compile -/config.rust /configure /Doxyfile /orconfig.h @@ -63,7 +62,6 @@ core.* /config.guess /config.sub /conftest* -/link_rust.sh /micro-revision.* /patch-stamp /stamp-h @@ -161,12 +159,6 @@ core.* /src/app/tor-cov /src/app/tor-cov.exe -# /src/rust -/src/rust/.cargo/config -/src/rust/.cargo/registry -/src/rust/target -/src/rust/registry - # /src/test /src/test/Makefile /src/test/Makefile.in diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 8f80405650..0000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "src/ext/rust"] - path = src/ext/rust - url = https://git.torproject.org/tor-rust-dependencies diff --git a/.travis.yml b/.travis.yml index aaca3a7368..ceec8169c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,6 @@ language: c cache: ccache: true - ## cargo: true - directories: - - $HOME/.cargo - ## caching CARGO_TARGET_DIR actually slows down the build over time, - ## because old build products are never deleted. - ## where we point CARGO_TARGET_DIR in all our cargo invocations - #- $TRAVIS_BUILD_DIR/src/rust/target compiler: - gcc @@ -29,8 +22,6 @@ env: - HARDENING_OPTIONS="--enable-all-bugs-are-fatal --enable-expensive-hardening" ## We turn off asciidoc by default, because it's slow - ASCIIDOC_OPTIONS="--disable-asciidoc" - ## Our default rust version is the minimum supported version - - RUST_VERSION="1.31.0" ## Turn off tor's sandbox in chutney, until we fix sandbox errors that are ## triggered by Ubuntu Xenial and Bionic. See #32722. - CHUTNEY_TOR_SANDBOX="0" @@ -68,10 +59,6 @@ matrix: ## We check disable module dirauth - env: MODULES_OPTIONS="--disable-module-dirauth" HARDENING_OPTIONS="--enable-expensive-hardening" - ## We run rust on Linux, because it's faster than rust on macOS - ## We check rust offline - - env: RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true - ## We check NSS ## Use -std=gnu99 to turn off some newer features, and maybe turn on some ## extra gcc warnings? @@ -86,14 +73,6 @@ matrix: ## We run `make doxygen` without `make check`. - env: SKIP_MAKE_CHECK="yes" DOXYGEN="yes" - ## macOS builds are very slow, and we have a limited number of - ## concurrent macOS jobs. We're not actively developing Rust, so it is - ## the lowest priority. - ## We run rust on macOS, because we have seen macOS rust failures before - #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" - # compiler: clang - # os: osx - ## Allow the build to report success (with non-required sub-builds ## continuing to run) if all required sub-builds have succeeded. fast_finish: true @@ -101,16 +80,6 @@ matrix: ## Careful! We use global envs, which makes it hard to allow failures by env: ## https://docs.travis-ci.com/user/customizing-the-build#matching-jobs-with-allow_failures allow_failures: - ## macOS rust and chutney are very slow, so we let the build finish before - ## they are done. We'd like to fast finish, but still eventually show - ## any failures in the build status. But Travis doesn't have that ability. - - ## Since this job is disabled, there's not much point having an exception - ## for it - #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" - # compiler: clang - # os: osx - ## Since we're actively developing IPv6, we want to require the IPv6 ## chutney tests #- env: CHUTNEY_MAKE="test-network-ipv6" CHUTNEY="yes" CHUTNEY_ALLOW_FAILURES="2" SKIP_MAKE_CHECK="yes" @@ -187,8 +156,6 @@ osx_image: xcode11.2 before_install: ## Set pipefail: we use pipes - set -o pipefail || echo "pipefail failed" - ## Create empty rust directories for non-Rust builds, so caching succeeds - - if [[ "$RUST_OPTIONS" == "" ]]; then mkdir -p $HOME/.cargo $TRAVIS_BUILD_DIR/src/rust/target; fi install: ## If we're on OSX, configure ccache (ccache is automatically installed and configured on Linux) @@ -200,13 +167,6 @@ install: - if [[ "$COVERAGE_OPTIONS" != "" ]]; then pip install --user cpp-coveralls; fi ## If we're on OSX, and using asciidoc, configure asciidoc - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export XML_CATALOG_FILES="/usr/local/etc/xml/catalog"; fi - ## If we're using Rust, download rustup - - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi - ## Install the stable channels of rustc and cargo and setup our toolchain environment - - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain $RUST_VERSION; fi - - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi - ## If we're testing rust builds in offline-mode, then set up our vendored dependencies - - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi ## If we're running chutney, install it. - if [[ "$CHUTNEY" != "" ]]; then git clone --depth 1 https://github.com/torproject/chutney.git ; export CHUTNEY_PATH="$(pwd)/chutney"; fi ## If we're running stem, install it. @@ -215,13 +175,6 @@ install: ## Finally, list installed package versions - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then dpkg-query --show; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew list --versions; fi - ## Get some info about rustup, rustc and cargo - - if [[ "$RUST_OPTIONS" != "" ]]; then which rustup; fi - - if [[ "$RUST_OPTIONS" != "" ]]; then which rustc; fi - - if [[ "$RUST_OPTIONS" != "" ]]; then which cargo; fi - - if [[ "$RUST_OPTIONS" != "" ]]; then rustup --version; fi - - if [[ "$RUST_OPTIONS" != "" ]]; then rustc --version; fi - - if [[ "$RUST_OPTIONS" != "" ]]; then cargo --version; fi ## Get python version - python --version ## If we're running chutney, show the chutney commit @@ -240,7 +193,7 @@ script: # Skip test_rebind and test_include on macOS - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export TOR_SKIP_TEST_REBIND=true; export TOR_SKIP_TEST_INCLUDE=true; fi - ./autogen.sh - - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $NSS_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules" + - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $NSS_OPTIONS $OPENSSL_OPTIONS --enable-fatal-warnings --disable-silent-rules" - echo "Configure flags are $CONFIGURE_FLAGS CC=\"$CC $C_DIALECT_OPTIONS\"" - ./configure $CONFIGURE_FLAGS CC="$CC $C_DIALECT_OPTIONS"; ## We run `make check` because that's what https://jenkins.torproject.org does. @@ -270,9 +223,6 @@ after_failure: before_cache: ## Delete all gcov files. - if [[ "$COVERAGE_OPTIONS" != "" ]]; then make reset-gcov; fi - ## Delete the cargo registry before caching .cargo, because it's cheaper to - ## download the registry and throw it away, rather than caching it - - rm -rf $HOME/.cargo/registry notifications: irc: @@ -1,3 +1,1124 @@ +Changes in version 0.4.7.3-alpha - 2021-12-15 + This third alpha release of the 0.4.7.x series fixes several bugs including + two major ones affecting Bridges and Relays (see below). If you are running + an earlier 0.4.7.x version, you should upgrade to this version. + + o Major bugfixes (bridges): + - Make Tor work reliably again when you have multiple bridges + configured and one or more of them are unreachable. The problem + came because we require that we have bridge descriptors for both + of our first two bridges (else we refuse to try to connect), but + in some cases we would wait three hours before trying to fetch + these missing descriptors, and/or never recover when we do try to + fetch them. Fixes bugs 40396 and 40495; bugfix on 0.3.0.5-rc + and 0.3.2.1-alpha. + + o Major bugfixes (relay, overload): + - Change the MetricsPort DNS "timeout" label to be "tor_timeout" in + order to indicate that this was a DNS timeout from tor perspective + and not the DNS server itself. + - Deprecate overload_dns_timeout_period_secs and + overload_dns_timeout_scale_percent consensus parameters as well. + They were used to assess the overload state which is no more now. + - Don't make Tor DNS timeout trigger an overload general state. + These timeouts are different from DNS server timeout. They have to + be seen as timeout related to UX and not because of a network + problem. Fixes bug 40527; bugfix on 0.4.6.1-alpha. + + o Minor feature (reproducible build): + - The repository can now build reproducible tarballs which adds the + build command "make dist-reprod" for that purpose. Closes + ticket 26299. + + o Minor features (compilation): + - Give an error message if trying to build with a version of + LibreSSL known not to work with Tor. (There's an incompatibility + with LibreSSL versions 3.2.1 through 3.4.0 inclusive because of + their incompatibility with OpenSSL 1.1.1's TLSv1.3 APIs.) Closes + ticket 40511. + + o Minor features (fallbackdir): + - Regenerate fallback directories generated on December 15, 2021. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/12/15. + + o Minor features (portability): + - Try to prevent a compiler warning about printf arguments that + could sometimes occur on MSYS2 depending on the configuration. + Closes ticket 40355. + + o Minor bugfix (pluggable transport): + - Do not kill a managed proxy if one of its transport configurations + emits a method error. Instead log a warning and continue processing + method arguments. Fixes bug 7362; bugfix on 0.2.3.6-alpha. + + o Minor bugfixes (bridges): + - When we don't yet have a descriptor for one of our bridges, + disable the entry guard retry schedule on that bridge. The entry + guard retry schedule and the bridge descriptor retry schedule can + conflict, e.g. where we mark a bridge as "maybe up" yet we don't + try to fetch its descriptor yet, leading Tor to wait (refusing to + do anything) until it becomes time to fetch the descriptor. Fixes + bug 40497; bugfix on 0.3.0.3-alpha. + + o Minor bugfixes (compilation): + - Fix our configuration logic to detect whether we had OpenSSL 3: + previously, our logic was reversed. This has no other effect than + to change whether we suppress deprecated API warnings. Fixes bug + 40429; bugfix on 0.3.5.13. + + o Minor bugfixes (controller, path bias): + - When a circuit's path is specified, in full or in part, from the + controller API, do not count that circuit towards our path-bias + calculations. (Doing so was incorrect, since we cannot tell + whether the controller is selecting relays randomly.) Resolves a + "Bug" warning. Fixes bug 40515; bugfix on 0.2.4.10-alpha. + + o Minor bugfixes (logging): + - When we no longer have enough directory information to use the + network, we would log a notice-level message -- but we would not + reliably log a message when we recovered and resumed using the + network. Now make sure there is always a corresponding message + about recovering. Fixes bug 40496; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (performance, DoS): + - Fix one case of a not-especially viable denial-of-service attack + found by OSS-Fuzz in our consensus-diff parsing code. This attack + causes a lot small of memory allocations and then immediately + frees them: this is only slow when running with all the sanitizers + enabled. Fixes one case of bug 40472; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (relay): + - Reject IPv6-only DirPorts. Our reachability self-test forces + DirPorts to be IPv4, but our configuration parser allowed them to + be IPv6-only, which led to an assertion failure. Fixes bug 40494; + bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (sandbox): + - Fix the sandbox on i386 by modifying it to allow the + "clock_gettime64" and "statx" system calls and to filter the + "chown32" and "stat64" system calls in place of "chown" and + "stat", respectively. Fixes bug 40505; bugfix on 0.2.5.4-alpha. + + o Documentation (man, relay): + - Missing "OverloadStatistics" in tor.1 manpage. Fixes bug 40504; + bugfix on 0.4.6.1-alpha. + +Changes in version 0.4.6.9 - 2021-12-15 + This version fixes several bugs from earlier versions of Tor. One important + piece is the removal of DNS timeout metric from the overload general signal. + See below for more details. + + o Major bugfixes (relay, overload): + - Don't make Tor DNS timeout trigger an overload general state. + These timeouts are different from DNS server timeout. They have to + be seen as timeout related to UX and not because of a network + problem. Fixes bug 40527; bugfix on 0.4.6.1-alpha. + + o Minor feature (reproducible build): + - The repository can now build reproducible tarballs which adds the + build command "make dist-reprod" for that purpose. Closes + ticket 26299. + + o Minor features (compilation): + - Give an error message if trying to build with a version of + LibreSSL known not to work with Tor. (There's an incompatibility + with LibreSSL versions 3.2.1 through 3.4.0 inclusive because of + their incompatibility with OpenSSL 1.1.1's TLSv1.3 APIs.) Closes + ticket 40511. + + o Minor features (fallbackdir): + - Regenerate fallback directories generated on December 15, 2021. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/12/15. + + o Minor bugfixes (compilation): + - Fix our configuration logic to detect whether we had OpenSSL 3: + previously, our logic was reversed. This has no other effect than + to change whether we suppress deprecated API warnings. Fixes bug + 40429; bugfix on 0.3.5.13. + + o Minor bugfixes (relay): + - Reject IPv6-only DirPorts. Our reachability self-test forces + DirPorts to be IPv4, but our configuration parser allowed them to + be IPv6-only, which led to an assertion failure. Fixes bug 40494; + bugfix on 0.4.5.1-alpha. + + o Documentation (man, relay): + - Missing "OverloadStatistics" in tor.1 manpage. Fixes bug 40504; + bugfix on 0.4.6.1-alpha. + + +Changes in version 0.4.7.2-alpha - 2021-10-26 + This second alpha release of the 0.4.7.x series adds two major + features: congestion control (prop324) for network performance, and + the MiddleOnly flag (prop335) voted by the authorities to pin relays + to the middle position for various network health reasons. This + release also fixes numerous bugs. + + The congestion control feature, detailed in proposal 324, still needs + more work before we can enable it by default. It is currently in its + testing and tuning phase which means that you should expect more + 0.4.7.x alphas as congestion control gets stabilized and tuned for + optimal performance. And so, at this release, it can not be used + without a custom patch. + + o Major features (congestion control): + - Implement support for flow control over congestion controlled + circuits. This work comes from proposal 324. Closes ticket 40450. + + o Major features (directory authority): + - Add a new consensus method to handle MiddleOnly specially. When + enough authorities are using this method, then any relay tagged + with the MiddleOnly flag will have its Exit, Guard, HSDir, and + V2Dir flags automatically cleared, and will have its BadExit flag + automatically set. Implements part of proposal 335. + - Authorities can now be configured to label relays as "MiddleOnly". + When voting for this flag, authorities automatically vote against + Exit, Guard, HSDir, and V2Dir; and in favor of BadExit. Implements + part of proposal 335. Based on a patch from Neel Chauhan. + + o Major bugfix (relay, metrics): + - On the MetricsPort, the DNS error statistics are not reported by + record type ("record=...") anymore due to a libevent bug + (https://github.com/libevent/libevent/issues/1219). Fixes bug + 40490; bugfix on 0.4.7.1-alpha. + + o Major bugfixes (relay, overload state): + - Relays report the general overload state for DNS timeout errors + only if X% of all DNS queries over Y seconds are errors. Before + that, it only took 1 timeout to report the overload state which + was just too low of a threshold. The X and Y values are 1% and 10 + minutes respectively but they are also controlled by consensus + parameters. Fixes bug 40491; bugfix on 0.4.6.1-alpha. + + o Minor feature (authority, relay): + - Reject End-Of-Life relays running version 0.4.2.x, 0.4.3.x, + 0.4.4.x and 0.4.5 alphas/rc. Closes ticket 40480. + + o Minor feature (onion service v2): + - Onion service v2 addresses are now not recognized anymore by tor + meaning a bad hostname is returned when attempting to pass it on a + SOCKS connection. No more deprecation log is emitted client side. + Closes ticket 40476. + - See https://blog.torproject.org/v2-deprecation-timeline for + details on how to transition from v2 to v3. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor features (logging, heartbeat): + - When a relay receives a cell that isn't encrypted properly for it, + but the relay is the last hop on the circuit, the relay now counts + how many cells of this kind it receives, on how many circuits, and + reports this information in the log. Previously, we'd log each + cell at PROTOCOL_WARN level, which is far too verbose to be + useful. Fixes part of ticket 40400. + + o Minor features (testing): + - We now have separate fuzzers for the inner layers of v3 onion + service descriptors, to prevent future bugs like 40392. Closes + ticket 40488. + + o Minor bugfixes (compilation): + - Fix compilation error when __NR_time is not defined. Fixes bug + 40465; bugfix on 0.2.5.5-alpha. Patch by Daniel Pinto. + + o Minor bugfixes (dirauth, bandwidth scanner): + - Add the AuthDirDontVoteOnDirAuthBandwidth dirauth config parameter + to avoid voting on bandwidth scanner weights to v3 directory + authorities. Fixes bug 40471; bugfix on 0.2.2.1-alpha. Patch by + Neel Chauhan. + + o Minor bugfixes (fragile-hardening, sandbox): + - When building with --enable-fragile-hardening, add or relax Linux + seccomp rules to allow AddressSanitizer to execute normally if the + process terminates with the sandbox active. This has the side + effect of disabling the filtering of file- and directory-open + requests on most systems and dilutes the effectiveness of the + sandbox overall, as a wider range of system calls must be + permitted. Fixes bug 11477; bugfix on 0.2.5.4-alpha. + + o Minor bugfixes (logging): + - If a channel has never received or transmitted a cell, or seen a + client, do not calculate time diffs against 1/1/1970 but log a + better prettier message. Fixes bug 40182; bugfix on 0.2.4.4. + + o Minor bugfixes (onion service): + - Fix a warning BUG that would occur often on heavily loaded onion + service leading to filling the logs with useless warnings. Fixes + bug 34083; bugfix on 0.3.2.1-alpha. + + o Minor bugfix (CI, onion service): + - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, config): + - Fix a memory leak for a small config line string that could occur + if the onion service failed to be configured from file properly. + Fixes bug 40484; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, TROVE-2021-008): + - Only log v2 access attempts once total, in order to not pollute + the logs with warnings and to avoid recording the times on disk + when v2 access was attempted. Note that the onion address was + _never_ logged. This counts as a Low-severity security issue. + Fixes bug 40474; bugfix on 0.4.5.8. + - Note that due to #40476 which removes v2 support entirely, this + log line is not emitted anymore. We still mention this in the + changelog because it is a Low-severity TROVE. + + o Minor bugfixes (usability): + - Do not log "RENDEZVOUS1 cell with unrecognized rendezvous cookie" + at LOG_PROTOCOL_WARN; instead log it at DEBUG. This warning can + happen naturally if a client gives up on a rendezvous circuit + after sending INTRODUCE1. Fixes part of bug 40400; bugfix + on 0.1.1.13-alpha. + - Do not log "circuit_receive_relay_cell failed" at + LOG_PROTOCOL_WARN; instead log it at DEBUG. In every case where we + would want to log this as a protocol warning, we are already + logging another warning from inside circuit_receive_relay_cell. + Fixes part of bug 40400; bugfix on 0.1.1.9-alpha. + + o Code simplification and refactoring: + - Lower the official maximum for "guard-extreme-restriction-percent" + to 100. This has no effect on when the guard code will generate a + warning, but it makes the intent of the option clearer. Fixes bug + 40486; bugfix on 0.3.0.1-alpha. + + o Testing: + - Add unit tests for the Linux seccomp sandbox. Resolves + issue 16803. + + o Code simplification and refactoring (rust): + - Remove Rust support and its associated code. It is unsupported and + Rust focus should be shifted to arti. Closes ticket 40469. + + o Testing (CI, chutney): + - Bump the data size that chutney transmits to 5MBytes in order to + trigger the flow control and congestion window code. Closes + ticket 40485. + + +Changes in version 0.4.6.8 - 2021-10-26 + This version fixes several bugs from earlier versions of Tor. One + highlight is a fix on how we track DNS timeouts to report general + relay overload. + + o Major bugfixes (relay, overload state): + - Relays report the general overload state for DNS timeout errors + only if X% of all DNS queries over Y seconds are errors. Before + that, it only took 1 timeout to report the overload state which + was just too low of a threshold. The X and Y values are 1% and 10 + minutes respectively but they are also controlled by consensus + parameters. Fixes bug 40491; bugfix on 0.4.6.1-alpha. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor features (testing): + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change the smallest + amount of time over which they're willing to report their observed + maximum bandwidth. Previously, this was fixed at 1 day. For + safety, values under 2 hours are only supported on testing + networks. Part of a fix for ticket 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a + fix for ticket 40337. + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited until they had + been running for a full day. Closes ticket 40337. + + o Minor bugfix (onion service): + - Do not flag an HSDir as non-running in case the descriptor upload + or fetch fails. An onion service closes pending directory + connections before uploading a new descriptor which can thus lead + to wrongly flagging many relays and thus affecting circuit building + path selection. Fixes bug 40434; bugfix on 0.2.0.13-alpha. + - Improve logging when a bad HS version is given. Fixes bug 40476; + bugfix on 0.4.6.1-alpha. + + o Minor bugfix (CI, onion service): + - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (compatibility): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + o Minor bugfixes (onion service, TROVE-2021-008): + - Only log v2 access attempts once total, in order to not pollute + the logs with warnings and to avoid recording the times on disk + when v2 access was attempted. Note that the onion address was + _never_ logged. This counts as a Low-severity security issue. + Fixes bug 40474; bugfix on 0.4.5.8. + + +Changes in version 0.4.5.11 - 2021-10-26 + The major change in this version is that v2 onion services are now + disabled at the client, service, and relay: any Tor nodes running this + version and onward will stop supporting v2 onion services. This is the + last step in the long deprecation process of v2 onion services. + Everyone running an earlier version, whether as a client, a relay, or + an onion service, should upgrade to Tor 0.3.5.17, 0.4.5.11, + or 0.4.6.8. + + o Major feature (onion service v2): + - See https://blog.torproject.org/v2-deprecation-timeline for + details on how to transition from v2 to v3. + - The control port commands HSFETCH and HSPOST no longer allow + version 2, and it is no longer possible to create a v2 service + with ADD_ONION. + - Tor no longer allows creating v2 services, or connecting as a + client to a v2 service. Relays will decline to be a v2 HSDir or + introduction point. This effectively disables onion service + version 2 Tor-wide. Closes ticket 40476. + + o Minor features (bridge, backport from 0.4.6.8): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor features (logging, diagnostic, backport from 0.4.6.5): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + o Minor features (testing, backport from 0.4.6.8): + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change the smallest + amount of time over which they're willing to report their observed + maximum bandwidth. Previously, this was fixed at 1 day. For + safety, values under 2 hours are only supported on testing + networks. Part of a fix for ticket 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a + fix for ticket 40337. + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited until they had + been running for a full day. Closes ticket 40337. + + o Minor bugfix (CI, onion service): + - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; + bugfix on 0.3.2.1-alpha. + + o Minor bugfix (onion service, backport from 0.4.6.8): + - Do not flag an HSDir as non-running in case the descriptor upload + or fetch fails. An onion service closes pending directory + connections before uploading a new descriptor which can thus lead + to wrongly flagging many relays and thus affecting circuit building + path selection. Fixes bug 40434; bugfix on 0.2.0.13-alpha. + + o Minor bugfixes (compatibility, backport from 0.4.6.8): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + o Minor bugfixes (consensus handling, backport from 0.4.6.4-rc): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (onion service, TROVE-2021-008, backport from 0.4.6.8): + - Only log v2 access attempts once total, in order to not pollute + the logs with warnings and to avoid recording the times on disk + when v2 access was attempted. Note that the onion address was + _never_ logged. This counts as a Low-severity security issue. + Fixes bug 40474; bugfix on 0.4.5.8. + + +Changes in version 0.3.5.17 - 2021-10-26 + The major change in this version is that v2 onion services are now + disabled at the client, service, and relay: any Tor nodes running this + version and onward will stop supporting v2 onion services. This is the + last step in the long deprecation process of v2 onion services. + Everyone running an earlier version, whether as a client, a relay, or + an onion service, should upgrade to Tor 0.3.5.17, 0.4.5.11, + or 0.4.6.8. + + o Major feature (onion service v2, backport from 0.4.5.11): + - See https://blog.torproject.org/v2-deprecation-timeline for + details on how to transition from v2 to v3. + - The control port commands HSFETCH and HSPOST no longer allow + version 2, and it is no longer possible to create a v2 service + with ADD_ONION. + - Tor no longer allows creating v2 services, or connecting as a + client to a v2 service. Relays will decline to be a v2 HSDir or + introduction point. This effectively disables onion service + version 2 Tor-wide. Closes ticket 40476. + + o Minor features (bridge, backport from 0.4.6.8): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor bugfixes (compatibility, backport from 0.4.6.8): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + +Changes in version 0.4.7.1-alpha - 2021-09-17 + This version is the first alpha release of the 0.4.7.x series. One + major feature is Vanguards Lite, from proposal 333, to help mitigate + guard discovery attacks against onion services. It also includes + numerous bugfixes. + + o Major features (Proposal 332, onion services, guard selection algorithm): + - Clients and onion services now choose four long-lived "layer 2" + guard relays for use as the middle hop in all onion circuits. + These relays are kept in place for a randomized duration averaging + 1 week. This mitigates guard discovery attacks against clients and + short-lived onion services such as OnionShare. Long-lived onion + services that need high security should still use the Vanguards + addon (https://github.com/mikeperry-tor/vanguards). Closes ticket + 40363; implements proposal 333. + + o Minor features (bridge testing support): + - Let external bridge reachability testing tools discard cached + bridge descriptors when setting new bridges, so they can be sure + to get a clean reachability test. Implements ticket 40209. + + o Minor features (fuzzing): + - When building with --enable-libfuzzer, use a set of compiler flags + that works with more recent versions of the library. Previously we + were using a set of flags from 2017. Closes ticket 40407. + + o Minor features (testing configuration): + - When TestingTorNetwork is enabled, skip the permissions check on + hidden service directories. Closes ticket 40338. + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change the smallest + amount of time over which they're willing to report their observed + maximum bandwidth. Previously, this was fixed at 1 day. For + safety, values under 2 hours are only supported on testing + networks. Part of a fix for ticket 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a + fix for ticket 40337. + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited until they had + been running for a full day. Closes ticket 40337. + + o Minor bugfixes (circuit padding): + - Don't send STOP circuit padding cells when the other side has + already shut down the corresponding padding machine. Fixes bug + 40435; bugfix on 0.4.0.1-alpha. + + o Minor bugfixes (compatibility): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + o Minor bugfixes (control, sandbox): + - Allows the control command SAVECONF to succeed when the seccomp + sandbox is enabled. Makes SAVECONF keep only one backup file, to + simplify implementation. Fixes bug 40317; bugfix on 0.2.5.4-alpha. + Patch by Daniel Pinto. + + o Minor bugfixes (heartbeat): + - Adjust the heartbeat log message about distinct clients to + consider the HeartbeatPeriod rather than a flat 6-hour delay. + Fixes bug 40330; bugfix on 0.2.6.3-alpha. + + o Minor bugfixes (logging, relay): + - Add spaces between the "and" when logging the "Your server has not + managed to confirm reachability for its" on dual-stack relays. + Fixes bug 40453; bugfix on 0.4.5.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (onion service): + - Do not flag an HSDir as non-running in case the descriptor upload + or fetch fails. An onion service closes pending directory + connections before uploading a new descriptor which leads to + wrongly flagging many relays and thus affecting circuit path + selection. Fixes bug 40434; bugfix on 0.2.0.13-alpha. + + o Minor bugfixes (statistics): + - Fix a fencepost issue when we check stability_last_downrated where + we called rep_hist_downrate_old_runs() twice. Fixes bug 40394; + bugfix on 0.2.0.5-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (tests): + - Fix a bug that prevented some tests from running with the correct + names. Fixes bug 40365; bugfix on 0.4.3.1-alpha. + + o Documentation: + - Add links to original tor design paper and anonbib to + docs/HACKING/README.1st.md. Closes ticket 33742. Patch from + Emily Bones. + - Describe the "fingerprint-ed25519" file in the tor.1 man page. + Fixes bug 40467; bugfix on 0.4.3.1-alpha. Patch by Neel Chauhan. + + +Changes in version 0.4.6.7 - 2021-08-16 + This version fixes several bugs from earlier versions of Tor, + including one that could lead to a denial-of-service attack. Everyone + running an earlier version, whether as a client, a relay, or an onion + service, should upgrade to Tor 0.3.5.16, 0.4.5.10, or 0.4.6.7. + + o Major bugfixes (cryptography, security): + - Resolve an assertion failure caused by a behavior mismatch between + our batch-signature verification code and our single-signature + verification code. This assertion failure could be triggered + remotely, leading to a denial of service attack. We fix this issue + by disabling batch verification. Fixes bug 40078; bugfix on + 0.2.6.1-alpha. This issue is also tracked as TROVE-2021-007 and + CVE-2021-38385. Found by Henry de Valence. + + o Minor feature (fallbackdir): + - Regenerate fallback directories list. Close ticket 40447. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/08/12. + + o Minor bugfix (crypto): + - Disable the unused batch verification feature of ed25519-donna. + Fixes bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry + de Valence. + + o Minor bugfixes (onion service): + - Send back the extended SOCKS error 0xF6 (Onion Service Invalid + Address) for a v2 onion address. Fixes bug 40421; bugfix + on 0.4.6.2-alpha. + + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW + in order to reduce CPU load on the directory relays. Fixes bug + 40301; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (timekeeping): + - Calculate the time of day correctly on systems where the time_t + type includes leap seconds. (This is not the case on most + operating systems, but on those where it occurs, our tor_timegm + function did not correctly invert the system's gmtime function, + which could result in assertion failures when calculating voting + schedules.) Fixes bug 40383; bugfix on 0.2.0.3-alpha. + + +Changes in version 0.4.5.10 - 2021-08-16 + This version fixes several bugs from earlier versions of Tor, + including one that could lead to a denial-of-service attack. Everyone + running an earlier version, whether as a client, a relay, or an onion + service, should upgrade to Tor 0.3.5.16, 0.4.5.10, or 0.4.6.7. + + o Major bugfixes (cryptography, security): + - Resolve an assertion failure caused by a behavior mismatch between + our batch-signature verification code and our single-signature + verification code. This assertion failure could be triggered + remotely, leading to a denial of service attack. We fix this issue + by disabling batch verification. Fixes bug 40078; bugfix on + 0.2.6.1-alpha. This issue is also tracked as TROVE-2021-007 and + CVE-2021-38385. Found by Henry de Valence. + + o Minor feature (fallbackdir): + - Regenerate fallback directories list. Close ticket 40447. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/08/12. + + o Minor features (testing): + - Enable the deterministic RNG for unit tests that covers the + address set bloomfilter-based API's. Fixes bug 40419; bugfix + on 0.3.3.2-alpha. + + o Minor bugfix (crypto, backport from 0.4.6.7): + - Disable the unused batch verification feature of ed25519-donna. + Fixes bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry + de Valence. + + o Minor bugfixes (relay, backport from 0.4.6.7): + - Reduce the compression level for data streaming from HIGH to LOW. + Fixes bug 40301; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (timekeeping, backport from 0.4.6.7): + - Calculate the time of day correctly on systems where the time_t + type includes leap seconds. (This is not the case on most + operating systems, but on those where it occurs, our tor_timegm + function did not correctly invert the system's gmtime function, + which could result in assertion failures when calculating voting + schedules.) Fixes bug 40383; bugfix on 0.2.0.3-alpha. + + o Minor bugfixes (warnings, portability, backport from 0.4.6.6): + - Suppress a strict-prototype warning when building with some + versions of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. + + +Changes in version 0.3.5.16 - 2021-08-16 + This version fixes several bugs from earlier versions of Tor, + including one that could lead to a denial-of-service attack. Everyone + running an earlier version, whether as a client, a relay, or an onion + service, should upgrade to Tor 0.3.5.16, 0.4.5.10, or 0.4.6.7. + + o Major bugfixes (cryptography, security): + - Resolve an assertion failure caused by a behavior mismatch between + our batch-signature verification code and our single-signature + verification code. This assertion failure could be triggered + remotely, leading to a denial of service attack. We fix this issue + by disabling batch verification. Fixes bug 40078; bugfix on + 0.2.6.1-alpha. This issue is also tracked as TROVE-2021-007 and + CVE-2021-38385. Found by Henry de Valence. + + o Minor feature (fallbackdir): + - Regenerate fallback directories list. Close ticket 40447. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/08/12. + + o Minor bugfix (crypto, backport from 0.4.6.7): + - Disable the unused batch verification feature of ed25519-donna. + Fixes bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry + de Valence. + + o Minor bugfixes (relay, backport from 0.4.6.7): + - Reduce the compression level for data streaming from HIGH to LOW. + Fixes bug 40301; bugfix on 0.3.5.1-alpha. + + +Changes in version 0.4.6.6 - 2021-06-30 + Tor 0.4.6.6 makes several small fixes on 0.4.6.5, including one that + allows Tor to build correctly on older versions of GCC. You should + upgrade to this version if you were having trouble building Tor + 0.4.6.5; otherwise, there is probably no need. + + o Minor bugfixes (compilation): + - Fix a compilation error when trying to build Tor with a compiler + that does not support const variables in static initializers. + Fixes bug 40410; bugfix on 0.4.6.5. + - Suppress a strict-prototype warning when building with some + versions of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Enable the deterministic RNG for unit tests that covers the + address set bloomfilter-based API's. Fixes bug 40419; bugfix + on 0.3.3.2-alpha. + + +Changes in version 0.4.5.9 - 2021-06-14 + Tor 0.4.5.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (control, sandbox, backport from 0.4.6.4-rc): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port, backport from 0.4.6.4-rc): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + +Changes in version 0.4.4.9 - 2021-06-14 + Tor 0.4.4.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Note that the scheduled end-of-life date for the Tor 0.4.4.x series is + June 15. This is therefore the last release in its series. Everybody + still running 0.4.4.x should plan to upgrade to 0.4.5.x or later. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.15 - 2021-06-14 + Tor 0.3.5.15 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + +Changes in version 0.4.6.5 - 2021-06-14 + Tor 0.4.6.5 is the first stable release in its series. The 0.4.6.x + series includes numerous features and bugfixes, including a significant + improvement to our circuit timeout algorithm that should improve + observed client performance, and a way for relays to report when they are + overloaded. + + This release also includes security fixes for several security issues, + including a denial-of-service attack against onion service clients, + and another denial-of-service attack against relays. Everybody should + upgrade to one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Below are the changes since 0.4.6.4-rc. For a complete list of changes + since 0.4.5.8, see the ReleaseNotes file. + + o Major bugfixes (security): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor features (logging, diagnostic): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + +Changes in version 0.4.6.4-rc - 2021-05-28 + Tor 0.4.6.4-rc fixes a few bugs from previous releases. This, we hope, + the final release candidate in its series: unless major new issues are + found, the next release will be stable. + + o Minor features (compatibility): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor bugfixes (consensus handling): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (control, sandbox): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + o Removed features: + - Remove unneeded code for parsing private keys in directory + documents. This code was only used for client authentication in v2 + onion services, which are now unsupported. Closes ticket 40374. + + +Changes in version 0.4.5.8 - 2021-05-10 + Tor 0.4.5.8 fixes several bugs in earlier version, backporting fixes + from the 0.4.6.x series. + + o Minor features (compatibility, Linux seccomp sandbox, backport from 0.4.6.3-rc): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation, backport from 0.4.6.3-rc): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add warning message when connecting to now deprecated v2 onion + services. As announced, Tor 0.4.5.x is the last series that will + support v2 onions. Closes ticket 40373. + + o Minor bugfixes (bridge, pluggable transport, backport from 0.4.6.2-alpha): + - Fix a regression that made it impossible start Tor using a bridge + line with a transport name and no fingerprint. Fixes bug 40360; + bugfix on 0.4.5.4-rc. + + o Minor bugfixes (build, cross-compilation, backport from 0.4.6.3-rc): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (controller, backport from 0.4.6.1-alpha): + - Fix a "BUG" warning that would appear when a controller chooses + the first hop for a circuit, and that circuit completes. Fixes bug + 40285; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, client, memory leak, backport from 0.4.6.3-rc): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD, backport from 0.4.6.2-alpha): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + +Changes in version 0.4.6.3-rc - 2021-05-10 + Tor 0.4.6.3-rc is the first release candidate in its series. It fixes + a few small bugs from previous versions, and adds a better error + message when trying to use (no longer supported) v2 onion services. + + Though we anticipate that we'll be doing a bit more clean-up between + now and the stable release, we expect that our remaining changes will + be fairly simple. There will likely be at least one more release + candidate before 0.4.6.x is stable. + + o Major bugfixes (onion service, control port): + - Make the ADD_ONION command properly configure client authorization. + Before this fix, the created onion failed to add the client(s). + Fixes bug 40378; bugfix on 0.4.6.1-alpha. + + o Minor features (compatibility, Linux seccomp sandbox): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add a warning message when trying to connect to (no longer + supported) v2 onion services. Closes ticket 40373. + + o Minor bugfixes (build, cross-compilation): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (compiler warnings): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (logging, relay): + - Emit a warning if an Address is found to be internal and tor can't + use it. Fixes bug 40290; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (onion service, client, memory leak): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + Changes in version 0.4.6.2-alpha - 2021-04-15 Tor 0.4.6.2-alpha is the second alpha in its series. It fixes several small bugs in previous releases, and solves other issues that had diff --git a/Doxyfile.in b/Doxyfile.in index 4374e54858..d06c344528 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -864,10 +864,8 @@ RECURSIVE = YES # run. EXCLUDE = ./src/ext/ed25519 \ - ./src/ext/rust \ ./src/trunnel \ - ./src/test \ - ./src/rust/registry + ./src/test # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/Makefile.am b/Makefile.am index c889945d5a..b059496688 100644 --- a/Makefile.am +++ b/Makefile.am @@ -36,12 +36,6 @@ else TESTING_TOR_BINARY=$(top_builddir)/src/app/tor$(EXEEXT) endif -if USE_RUST -rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) -else -rust_ldadd= -endif - # "Common" libraries used to link tor's utility code. TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ @@ -253,7 +247,7 @@ endif TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings if LIBFUZZER_ENABLED -TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div +TEST_CFLAGS += -fsanitize=fuzzer-no-link # not "edge" endif @@ -601,51 +595,6 @@ check-typos: echo "You can install the latest version of misspell here: https://github.com/client9/misspell#install"; \ fi -.PHONY: rustfmt -rustfmt: -if USE_RUST - @if test -x "`which cargo-fmt 2>&1;true`"; then \ - echo "Formatting Rust code ..."; \ - (cd "$(top_srcdir)/src/rust" && cargo fmt --all --); \ - else \ - echo "Tor uses rustfmt (via cargo-fmt) to format Rust code."; \ - echo "However, it seems that you don't have rustfmt installed."; \ - printf "You can install rustfmt by following the directions here:"; \ - echo " https://github.com/rust-lang-nursery/rustfmt"; \ - fi -endif - -.PHONY: check-rustfmt -check-rustfmt: -if USE_RUST - @if test -x "`which cargo-fmt 2>&1;true`"; then \ - printf "Running rustfmt..."; \ - (cd "$(top_srcdir)/src/rust" && cargo fmt --all -- --check && echo "done.") || \ - (echo "**************** check-rustfmt failed. ****************"; \ - echo " Run \`make rustfmt\` to apply the above changes."; \ - exit 1); \ - else \ - echo "Tor uses rustfmt (via cargo-fmt) to format Rust code."; \ - echo "However, it seems that you don't have rustfmt installed."; \ - printf "You can install rustfmt by following the directions here:"; \ - echo " https://github.com/rust-lang-nursery/rustfmt"; \ - fi -endif - -.PHONY: clippy -clippy: -if USE_RUST - @if test -x "`which cargo-clippy 2>&1;true`"; then \ - echo "Running cargo clippy ..."; \ - echo "Prepare yourself for the onslaught of suggestions ..."; \ - (cd "$(top_srcdir)/src/rust" && cargo clippy); \ - else \ - echo "Tor can use clippy to lint Rust code."; \ - echo "However, it seems that you don't have clippy installed."; \ - echo "You can install the latest version of clippy by following the directions here: https://github.com/rust-lang-nursery/rust-clippy"; \ - fi -endif - .PHONY: check-changes check-changes: if USEPYTHON @@ -686,7 +635,7 @@ update-copyright: $(PERL) $(top_srcdir)/scripts/maint/updateCopyright.pl $(OWNED_TOR_C_FILES) .PHONY: autostyle -autostyle: update-versions rustfmt autostyle-ifdefs rectify-includes +autostyle: update-versions autostyle-ifdefs rectify-includes mostlyclean-local: rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no} @@ -694,14 +643,6 @@ mostlyclean-local: rm -rf $(top_builddir)/doc/doxygen rm -rf $(TEST_NETWORK_ALL_LOG_DIR) -clean-local: - rm -rf $(top_builddir)/src/rust/target - rm -rf $(top_builddir)/src/rust/.cargo/registry - -if USE_RUST -distclean-local: distclean-rust -endif - # This relies on some internal details of how automake implements # distcheck. We check two directories because automake-1.15 changed # from $(distdir)/_build to $(distdir)/_build/sub. @@ -1,3 +1,5 @@ +[](https://gitlab.torproject.org/tpo/core/tor/-/commits/main) + Tor protects your privacy on the internet by hiding the connection between your Internet address and the services you use. We believe Tor is reasonably secure, but please ensure you read the instructions and diff --git a/ReleaseNotes b/ReleaseNotes index 42017292c5..8757475080 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -2,6 +2,935 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.4.6.9 - 2021-12-15 + This version fixes several bugs from earlier versions of Tor. One important + piece is the removal of DNS timeout metric from the overload general signal. + See below for more details. + + o Major bugfixes (relay, overload): + - Don't make Tor DNS timeout trigger an overload general state. + These timeouts are different from DNS server timeout. They have to + be seen as timeout related to UX and not because of a network + problem. Fixes bug 40527; bugfix on 0.4.6.1-alpha. + + o Minor feature (reproducible build): + - The repository can now build reproducible tarballs which adds the + build command "make dist-reprod" for that purpose. Closes + ticket 26299. + + o Minor features (compilation): + - Give an error message if trying to build with a version of + LibreSSL known not to work with Tor. (There's an incompatibility + with LibreSSL versions 3.2.1 through 3.4.0 inclusive because of + their incompatibility with OpenSSL 1.1.1's TLSv1.3 APIs.) Closes + ticket 40511. + + o Minor features (fallbackdir): + - Regenerate fallback directories generated on December 15, 2021. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/12/15. + + o Minor bugfixes (compilation): + - Fix our configuration logic to detect whether we had OpenSSL 3: + previously, our logic was reversed. This has no other effect than + to change whether we suppress deprecated API warnings. Fixes bug + 40429; bugfix on 0.3.5.13. + + o Minor bugfixes (relay): + - Reject IPv6-only DirPorts. Our reachability self-test forces + DirPorts to be IPv4, but our configuration parser allowed them to + be IPv6-only, which led to an assertion failure. Fixes bug 40494; + bugfix on 0.4.5.1-alpha. + + o Documentation (man, relay): + - Missing "OverloadStatistics" in tor.1 manpage. Fixes bug 40504; + bugfix on 0.4.6.1-alpha. + + +Changes in version 0.4.6.8 - 2021-10-26 + This version fixes several bugs from earlier versions of Tor. One + highlight is a fix on how we track DNS timeouts to report general + relay overload. + + o Major bugfixes (relay, overload state): + - Relays report the general overload state for DNS timeout errors + only if X% of all DNS queries over Y seconds are errors. Before + that, it only took 1 timeout to report the overload state which + was just too low of a threshold. The X and Y values are 1% and 10 + minutes respectively but they are also controlled by consensus + parameters. Fixes bug 40491; bugfix on 0.4.6.1-alpha. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor features (testing): + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change the smallest + amount of time over which they're willing to report their observed + maximum bandwidth. Previously, this was fixed at 1 day. For + safety, values under 2 hours are only supported on testing + networks. Part of a fix for ticket 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a + fix for ticket 40337. + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited until they had + been running for a full day. Closes ticket 40337. + + o Minor bugfix (onion service): + - Do not flag an HSDir as non-running in case the descriptor upload + or fetch fails. An onion service closes pending directory + connections before uploading a new descriptor which can thus lead + to wrongly flagging many relays and thus affecting circuit building + path selection. Fixes bug 40434; bugfix on 0.2.0.13-alpha. + - Improve logging when a bad HS version is given. Fixes bug 40476; + bugfix on 0.4.6.1-alpha. + + o Minor bugfix (CI, onion service): + - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (compatibility): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + o Minor bugfixes (onion service, TROVE-2021-008): + - Only log v2 access attempts once total, in order to not pollute + the logs with warnings and to avoid recording the times on disk + when v2 access was attempted. Note that the onion address was + _never_ logged. This counts as a Low-severity security issue. + Fixes bug 40474; bugfix on 0.4.5.8. + + +Changes in version 0.4.5.11 - 2021-10-26 + The major change in this version is that v2 onion services are now + disabled at the client, service, and relay: any Tor nodes running this + version and onward will stop supporting v2 onion services. This is the + last step in the long deprecation process of v2 onion services. + Everyone running an earlier version, whether as a client, a relay, or + an onion service, should upgrade to Tor 0.3.5.17, 0.4.5.11, + or 0.4.6.8. + + o Major feature (onion service v2): + - See https://blog.torproject.org/v2-deprecation-timeline for + details on how to transition from v2 to v3. + - The control port commands HSFETCH and HSPOST no longer allow + version 2, and it is no longer possible to create a v2 service + with ADD_ONION. + - Tor no longer allows creating v2 services, or connecting as a + client to a v2 service. Relays will decline to be a v2 HSDir or + introduction point. This effectively disables onion service + version 2 Tor-wide. Closes ticket 40476. + + o Minor features (bridge, backport from 0.4.6.8): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor features (logging, diagnostic, backport from 0.4.6.5): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + o Minor features (testing, backport from 0.4.6.8): + - On a testing network, relays can now use the + TestingMinTimeToReportBandwidth option to change the smallest + amount of time over which they're willing to report their observed + maximum bandwidth. Previously, this was fixed at 1 day. For + safety, values under 2 hours are only supported on testing + networks. Part of a fix for ticket 40337. + - Relays on testing networks no longer rate-limit how frequently + they are willing to report new bandwidth measurements. Part of a + fix for ticket 40337. + - Relays on testing networks now report their observed bandwidths + immediately from startup. Previously, they waited until they had + been running for a full day. Closes ticket 40337. + + o Minor bugfix (CI, onion service): + - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; + bugfix on 0.3.2.1-alpha. + + o Minor bugfix (onion service, backport from 0.4.6.8): + - Do not flag an HSDir as non-running in case the descriptor upload + or fetch fails. An onion service closes pending directory + connections before uploading a new descriptor which can thus lead + to wrongly flagging many relays and thus affecting circuit building + path selection. Fixes bug 40434; bugfix on 0.2.0.13-alpha. + + o Minor bugfixes (compatibility, backport from 0.4.6.8): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + o Minor bugfixes (consensus handling, backport from 0.4.6.4-rc): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (onion service, TROVE-2021-008, backport from 0.4.6.8): + - Only log v2 access attempts once total, in order to not pollute + the logs with warnings and to avoid recording the times on disk + when v2 access was attempted. Note that the onion address was + _never_ logged. This counts as a Low-severity security issue. + Fixes bug 40474; bugfix on 0.4.5.8. + + +Changes in version 0.3.5.17 - 2021-10-26 + The major change in this version is that v2 onion services are now + disabled at the client, service, and relay: any Tor nodes running this + version and onward will stop supporting v2 onion services. This is the + last step in the long deprecation process of v2 onion services. + Everyone running an earlier version, whether as a client, a relay, or + an onion service, should upgrade to Tor 0.3.5.17, 0.4.5.11, + or 0.4.6.8. + + o Major feature (onion service v2, backport from 0.4.5.11): + - See https://blog.torproject.org/v2-deprecation-timeline for + details on how to transition from v2 to v3. + - The control port commands HSFETCH and HSPOST no longer allow + version 2, and it is no longer possible to create a v2 service + with ADD_ONION. + - Tor no longer allows creating v2 services, or connecting as a + client to a v2 service. Relays will decline to be a v2 HSDir or + introduction point. This effectively disables onion service + version 2 Tor-wide. Closes ticket 40476. + + o Minor features (bridge, backport from 0.4.6.8): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (fallbackdir): + - Regenerate fallback directories for October 2021. Closes + ticket 40493. + + o Minor bugfixes (compatibility, backport from 0.4.6.8): + - Fix compatibility with the most recent Libevent versions, which no + longer have an evdns_set_random_bytes() function. Because this + function has been a no-op since Libevent 2.0.4-alpha, it is safe + for us to just stop calling it. Fixes bug 40371; bugfix + on 0.2.1.7-alpha. + + +Changes in version 0.4.6.7 - 2021-08-16 + This version fixes several bugs from earlier versions of Tor, including one + that could lead to a denial-of-service attack. Everyone running an earlier + version, whether as a client, a relay, or an onion service, should upgrade + to Tor 0.3.5.16, 0.4.5.10, or 0.4.6.7. + + o Major bugfixes (cryptography, security): + - Resolve an assertion failure caused by a behavior mismatch between our + batch-signature verification code and our single-signature verification + code. This assertion failure could be triggered remotely, leading to a + denial of service attack. We fix this issue by disabling batch + verification. Fixes bug 40078; bugfix on 0.2.6.1-alpha. This issue is + also tracked as TROVE-2021-007 and CVE-2021-38385. Found by Henry de + Valence. + + o Minor feature (fallbackdir): + - Regenerate fallback directories list. Close ticket 40447. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, + as retrieved on 2021/08/12. + + o Minor bugfix (crypto): + - Disable the unused batch verification feature of ed25519-donna. Fixes + bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry de Valence. + + o Minor bugfixes (onion service): + - Send back the extended SOCKS error 0xF6 (Onion Service Invalid Address) + for a v2 onion address. Fixes bug 40421; bugfix on 0.4.6.2-alpha. + + o Minor bugfix (CI, onion service): + - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW in + order to reduce CPU load on the directory relays. Fixes bug 40301; + bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (timekeeping): + - Calculate the time of day correctly on systems where the time_t + type includes leap seconds. (This is not the case on most + operating systems, but on those where it occurs, our tor_timegm + function did not correctly invert the system's gmtime function, + which could result in assertion failures when calculating + voting schedules.) Fixes bug 40383; bugfix on 0.2.0.3-alpha. + + +Changes in version 0.4.5.10 - 2021-08-16 + This version fixes several bugs from earlier versions of Tor, including one + that could lead to a denial-of-service attack. Everyone running an earlier + version, whether as a client, a relay, or an onion service, should upgrade + to Tor 0.3.5.16, 0.4.5.10, or 0.4.6.7. + + o Major bugfixes (cryptography, security): + - Resolve an assertion failure caused by a behavior mismatch between our + batch-signature verification code and our single-signature verification + code. This assertion failure could be triggered remotely, leading to a + denial of service attack. We fix this issue by disabling batch + verification. Fixes bug 40078; bugfix on 0.2.6.1-alpha. This issue is + also tracked as TROVE-2021-007 and CVE-2021-38385. Found by Henry de + Valence. + + o Minor feature (fallbackdir): + - Regenerate fallback directories list. Close ticket 40447. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, + as retrieved on 2021/08/12. + + o Minor features (testing): + - Enable the deterministic RNG for unit tests that covers the address set + bloomfilter-based API's. Fixes bug 40419; bugfix on 0.3.3.2-alpha. + + o Minor bugfix (crypto): + - Disable the unused batch verification feature of ed25519-donna. Fixes + bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry de Valence. + + o Minor bugfixes (relay, backport from 0.4.6.x): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (timekeeping, backport from 0.4.6.x): + - Calculate the time of day correctly on systems where the time_t + type includes leap seconds. (This is not the case on most + operating systems, but on those where it occurs, our tor_timegm + function did not correctly invert the system's gmtime function, + which could result in assertion failures when calculating + voting schedules.) Fixes bug 40383; bugfix on 0.2.0.3-alpha. + + o Minor bugfixes (warnings, portability, backport from 0.4.6.x): + - Suppress a strict-prototype warning when building with some versions + of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. + + +Changes in version 0.3.5.16 - 2021-08-16 + This version fixes several bugs from earlier versions of Tor, including one + that could lead to a denial-of-service attack. Everyone running an earlier + version, whether as a client, a relay, or an onion service, should upgrade + to Tor 0.3.5.16, 0.4.5.10, or 0.4.6.7. + + o Major bugfixes (cryptography, security): + - Resolve an assertion failure caused by a behavior mismatch between our + batch-signature verification code and our single-signature verification + code. This assertion failure could be triggered remotely, leading to a + denial of service attack. We fix this issue by disabling batch + verification. Fixes bug 40078; bugfix on 0.2.6.1-alpha. This issue is + also tracked as TROVE-2021-007 and CVE-2021-38385. Found by Henry de + Valence. + + o Minor feature (fallbackdir): + - Regenerate fallback directories list. Close ticket 40447. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, + as retrieved on 2021/08/12. + + o Minor bugfix (crypto): + - Disable the unused batch verification feature of ed25519-donna. Fixes + bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry de Valence. + + o Minor bugfixes (relay, backport from 0.4.6.x): + - Reduce the compression level for data streaming from HIGH to LOW. Fixes + bug 40301; bugfix on 0.3.5.1-alpha. + + +Changes in version 0.4.6.6 - 2021-06-30 + Tor 0.4.6.6 makes several small fixes on 0.4.6.5, including one that + allows Tor to build correctly on older versions of GCC. You should + upgrade to this version if you were having trouble building Tor + 0.4.6.5; otherwise, there is probably no need. + + o Minor bugfixes (compilation): + - Fix a compilation error when trying to build Tor with a compiler + that does not support const variables in static initializers. + Fixes bug 40410; bugfix on 0.4.6.5. + - Suppress a strict-prototype warning when building with some + versions of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing): + - Enable the deterministic RNG for unit tests that covers the + address set bloomfilter-based API's. Fixes bug 40419; bugfix + on 0.3.3.2-alpha. + + +Changes in version 0.4.6.5 - 2021-06-14 + Tor 0.4.6.5 is the first stable release in its series. The 0.4.6.x + series includes numerous features and bugfixes, including a significant + improvement to our circuit timeout algorithm that should improve + observed client performance, and a way for relays to report when they are + overloaded. + + This release also includes security fixes for several security issues, + including a denial-of-service attack against onion service clients, + and another denial-of-service attack against relays. Everybody should + upgrade to one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Below are the changes since 0.4.5.8. For a list of changes since + 0.4.6.4-rc, see the ChangeLog file. + + o Major bugfixes (security): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Major features (control port, onion services): + - Add controller support for creating version 3 onion services with + client authorization. Previously, only v2 onion services could be + created with client authorization. Closes ticket 40084. Patch by + Neel Chauhan. + + o Major features (directory authority): + - When voting on a relay with a Sybil-like appearance, add the Sybil + flag when clearing out the other flags. This lets a relay operator + know why their relay hasn't been included in the consensus. Closes + ticket 40255. Patch by Neel Chauhan. + + o Major features (metrics): + - Relays now report how overloaded they are in their extrainfo + documents. This information is controlled with the + OverloadStatistics torrc option, and it will be used to improve + decisions about the network's load balancing. Implements proposal + 328; closes ticket 40222. + + o Major features (relay, denial of service): + - Add a new DoS subsystem feature to control the rate of client + connections for relays. Closes ticket 40253. + + o Major features (statistics): + - Relays now publish statistics about the number of v3 onion + services and volume of v3 onion service traffic, in the same + manner they already do for v2 onions. Closes ticket 23126. + + o Major bugfixes (circuit build timeout): + - Improve the accuracy of our circuit build timeout calculation for + 60%, 70%, and 80% build rates for various guard choices. We now + use a maximum likelihood estimator for Pareto parameters of the + circuit build time distribution, instead of a "right-censored + estimator". This causes clients to ignore circuits that never + finish building in their timeout calculations. Previously, clients + were counting such unfinished circuits as having the highest + possible build time value, when in reality these circuits most + likely just contain relays that are offline. We also now wait a + bit longer to let circuits complete for measurement purposes, + lower the minimum possible effective timeout from 1.5 seconds to + 10ms, and increase the resolution of the circuit build time + histogram from 50ms bin widths to 10ms bin widths. Additionally, + we alter our estimate Xm by taking the maximum of the top 10 most + common build time values of the 10ms histogram, and compute Xm as + the average of these. Fixes bug 40168; bugfix on 0.2.2.14-alpha. + - Remove max_time calculation and associated warning from circuit + build timeout 'alpha' parameter estimation, as this is no longer + needed by our new estimator from 40168. Fixes bug 34088; bugfix + on 0.2.2.9-alpha. + + o Major bugfixes (signing key): + - In the tor-gencert utility, give an informative error message if + the passphrase given in `--create-identity-key` is too short. + Fixes bug 40189; bugfix on 0.2.0.1-alpha. Patch by Neel Chauhan. + + o Minor features (bridge): + - We now announce the URL to Tor's new bridge status at + https://bridges.torproject.org/ when Tor is configured to run as a + bridge relay. Closes ticket 30477. + + o Minor features (build system): + - New "make lsp" command to auto generate the compile_commands.json + file used by the ccls server. The "bear" program is needed for + this. Closes ticket 40227. + + o Minor features (client): + - Clients now check whether their streams are attempting to re-enter + the Tor network (i.e. to send Tor traffic over Tor), and close + them preemptively if they think exit relays will refuse them for + this reason. See ticket 2667 for details. Closes ticket 40271. + + o Minor features (command line): + - Add long format name "--torrc-file" equivalent to the existing + command-line option "-f". Closes ticket 40324. Patch by + Daniel Pinto. + + o Minor features (command-line interface): + - Add build informations to `tor --version` in order to ease + reproducible builds. Closes ticket 32102. + - When parsing command-line flags that take an optional argument, + treat the argument as absent if it would start with a '-' + character. Arguments in that form are not intelligible for any of + our optional-argument flags. Closes ticket 40223. + - Allow a relay operator to list the ed25519 keys on the command + line by adding the `rsa` and `ed25519` arguments to the + --list-fingerprint flag to show the respective RSA and ed25519 + relay fingerprint. Closes ticket 33632. Patch by Neel Chauhan. + + o Minor features (compatibility): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (control port, stream handling): + - Add the stream ID to the event line in the ADDRMAP control event. + Closes ticket 40249. Patch by Neel Chauhan. + + o Minor features (dormant mode): + - Add a new 'DormantTimeoutEnabled' option to allow coarse-grained + control over whether the client ever becomes dormant from + inactivity. Most people won't need this. Closes ticket 40228. + - Add a new 'DormantTimeoutEnabled' option for coarse-grained + control over whether the client can become dormant from + inactivity. Most people won't need this. Closes ticket 40228. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor features (logging): + - Edit heartbeat log messages so that more of them begin with the + string "Heartbeat: ". Closes ticket 40322; patch + from 'cypherpunks'. + - Change the DoS subsystem heartbeat line format to be more clear on + what has been detected/rejected, and which option is disabled (if + any). Closes ticket 40308. + - In src/core/mainloop/mainloop.c and src/core/mainloop/connection.c, + put brackets around IPv6 addresses in log messages. Closes ticket + 40232. Patch by Neel Chauhan. + + o Minor features (logging, diagnostic): + - Log decompression failures at a higher severity level, since they + can help provide missing context for other warning messages. We + rate-limit these messages, to avoid flooding the logs if they + begin to occur frequently. Closes ticket 40175. + + o Minor features (onion services): + - Add a warning message when trying to connect to (no longer + supported) v2 onion services. Closes ticket 40373. + + o Minor features (performance, windows): + - Use SRWLocks to implement locking on Windows. Replaces the + "critical section" locking implementation with the faster + SRWLocks, available since Windows Vista. Closes ticket 17927. + Patch by Daniel Pinto. + + o Minor features (protocol, proxy support, defense in depth): + - Close HAProxy connections if they somehow manage to send us data + before we start reading. Closes another case of ticket 40017. + + o Minor features (tests, portability): + - Port the hs_build_address.py test script to work with recent + versions of python. Closes ticket 40213. Patch from + Samanta Navarro. + + o Minor features (vote document): + - Add a "stats" line to directory authority votes, to report various + statistics that authorities compute about the relays. This will + help us diagnose the network better. Closes ticket 40314. + + o Minor bugfixes (build): + - The configure script now shows whether or not lzma and zstd have + been used, not just if the enable flag was passed in. Fixes bug + 40236; bugfix on 0.4.3.1-alpha. + + o Minor bugfixes (compatibility): + - Fix a failure in the test cases when running on the "hppa" + architecture, along with a related test that might fail on other + architectures in the future. Fixes bug 40274; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (compilation): + - Fix a compilation warning about unused functions when building + with a libc that lacks the GLOB_ALTDIRFUNC constant. Fixes bug + 40354; bugfix on 0.4.5.1-alpha. Patch by Daniel Pinto. + + o Minor bugfixes (consensus handling): + - Avoid a set of bugs that could be caused by inconsistently + preferring an out-of-date consensus stored in a stale directory + cache over a more recent one stored on disk as the latest + consensus. Fixes bug 40375; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (control, sandbox): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (directory authorities, voting): + - Add a new consensus method (31) to support any future changes that + authorities decide to make to the value of bwweightscale or + maxunmeasuredbw. Previously, there was a bug that prevented the + authorities from parsing these consensus parameters correctly under + most circumstances. Fixes bug 19011; bugfix on 0.2.2.10-alpha. + + o Minor bugfixes (ipv6): + - Allow non-SOCKSPorts to disable IPv4, IPv6, and PreferIPv4. Some + rare configurations might break, but in this case you can disable + NoIPv4Traffic and NoIPv6Traffic as needed. Fixes bug 33607; bugfix + on 0.4.1.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (key generation): + - Do not require a valid torrc when using the `--keygen` argument to + generate a signing key. This allows us to generate keys on systems + or users which may not run Tor. Fixes bug 40235; bugfix on + 0.2.7.2-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (logging, relay): + - Emit a warning if an Address is found to be internal and tor can't + use it. Fixes bug 40290; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (metrics port): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + o Minor bugfixes (onion services, logging): + - Downgrade the severity of a few rendezvous circuit-related + warnings from warning to info. Fixes bug 40207; bugfix on + 0.3.2.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (relay): + - Reduce the compression level for data streaming from HIGH to LOW. + This should reduce the CPU and memory burden for directory caches. + Fixes bug 40301; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + o Code simplification and refactoring: + - Remove the orconn_ext_or_id_map structure and related functions. + (Nothing outside of unit tests used them.) Closes ticket 33383. + Patch by Neel Chauhan. + + o Removed features: + - Remove unneeded code for parsing private keys in directory + documents. This code was only used for client authentication in v2 + onion services, which are now unsupported. Closes ticket 40374. + - As of this release, Tor no longer supports the old v2 onion + services. They were deprecated last July for security, and support + will be removed entirely later this year. We strongly encourage + everybody to migrate to v3 onion services. For more information, + see https://blog.torproject.org/v2-deprecation-timeline . Closes + ticket 40266. (NOTE: We accidentally released an earlier version + of the 0.4.6.1-alpha changelog without this entry. Sorry for + the confusion!) + + o Code simplification and refactoring (metrics, DoS): + - Move the DoS subsystem into the subsys manager, including its + configuration options. Closes ticket 40261. + + o Documentation (manual): + - Move the ServerTransport* options to the "SERVER OPTIONS" section. + Closes issue 40331. + - Indicate that the HiddenServiceStatistics option also applies to + bridges. Closes ticket 40346. + - Move the description of BridgeRecordUsageByCountry to the section + "STATISTICS OPTIONS". Closes ticket 40323. + + o Removed features (relay): + - Because DirPorts are only used on authorities, relays no longer + advertise them. Similarly, self-testing for DirPorts has been + disabled, since an unreachable DirPort is no reason for a relay + not to advertise itself. (Configuring a DirPort will still work, + for now.) Closes ticket 40282. + + +Changes in version 0.4.5.9 - 2021-06-14 + Tor 0.4.5.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (control, sandbox, backport from 0.4.6.4-rc): + - Allow the control command SAVECONF to succeed when the seccomp + sandbox is enabled, and make SAVECONF keep only one backup file to + simplify implementation. Previously SAVECONF allowed a large + number of backup files, which made it incompatible with the + sandbox. Fixes bug 40317; bugfix on 0.2.5.4-alpha. Patch by + Daniel Pinto. + + o Minor bugfixes (metrics port, backport from 0.4.6.4-rc): + - Fix a bug that made tor try to re-bind() on an already open + MetricsPort every 60 seconds. Fixes bug 40370; bugfix + on 0.4.5.1-alpha. + + +Changes in version 0.4.4.9 - 2021-06-14 + Tor 0.4.4.9 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + Note that the scheduled end-of-life date for the Tor 0.4.4.x series is + June 15. This is therefore the last release in its series. Everybody + still running 0.4.4.x should plan to upgrade to 0.4.5.x or later. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + +Changes in version 0.3.5.15 - 2021-06-14 + Tor 0.3.5.15 fixes several security issues, including a + denial-of-service attack against onion service clients, and another + denial-of-service attack against relays. Everybody should upgrade to + one of 0.3.5.15, 0.4.4.9, 0.4.5.9, or 0.4.6.5. + + o Major bugfixes (security, backport from 0.4.6.5): + - Don't allow relays to spoof RELAY_END or RELAY_RESOLVED cell on + half-closed streams. Previously, clients failed to validate which + hop sent these cells: this would allow a relay on a circuit to end + a stream that wasn't actually built with it. Fixes bug 40389; + bugfix on 0.3.5.1-alpha. This issue is also tracked as TROVE-2021- + 003 and CVE-2021-34548. + + o Major bugfixes (security, defense-in-depth, backport from 0.4.6.5): + - Detect more failure conditions from the OpenSSL RNG code. + Previously, we would detect errors from a missing RNG + implementation, but not failures from the RNG code itself. + Fortunately, it appears those failures do not happen in practice + when Tor is using OpenSSL's default RNG implementation. Fixes bug + 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as + TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. + + o Major bugfixes (security, denial of service, backport from 0.4.6.5): + - Resist a hashtable-based CPU denial-of-service attack against + relays. Previously we used a naive unkeyed hash function to look + up circuits in a circuitmux object. An attacker could exploit this + to construct circuits with chosen circuit IDs, to create + collisions and make the hash table inefficient. Now we use a + SipHash construction here instead. Fixes bug 40391; bugfix on + 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005 and + CVE-2021-34549. Reported by Jann Horn from Google's Project Zero. + - Fix an out-of-bounds memory access in v3 onion service descriptor + parsing. An attacker could exploit this bug by crafting an onion + service descriptor that would crash any client that tried to visit + it. Fixes bug 40392; bugfix on 0.3.0.1-alpha. This issue is also + tracked as TROVE-2021-006 and CVE-2021-34550. Reported by Sergei + Glazunov from Google's Project Zero. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor features (compatibility, backport from 0.4.6.4-rc): + - Remove an assertion function related to TLS renegotiation. It was + used nowhere outside the unit tests, and it was breaking + compilation with recent alpha releases of OpenSSL 3.0.0. Closes + ticket 40399. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/06/10. + + +Changes in version 0.4.5.8 - 2021-05-10 + Tor 0.4.5.8 fixes several bugs in earlier version, backporting fixes + from the 0.4.6.x series. + + o Minor features (compatibility, Linux seccomp sandbox, backport from 0.4.6.3-rc): + - Add a workaround to enable the Linux sandbox to work correctly + with Glibc 2.33. This version of Glibc has started using the + fstatat() system call, which previously our sandbox did not allow. + Closes ticket 40382; see the ticket for a discussion of trade-offs. + + o Minor features (compilation, backport from 0.4.6.3-rc): + - Make the autoconf script build correctly with autoconf versions + 2.70 and later. Closes part of ticket 40335. + + o Minor features (fallback directory list, backport from 0.4.6.2-alpha): + - Regenerate the list of fallback directories to contain a new set + of 200 relays. Closes ticket 40265. + + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, as + retrieved on 2021/05/07. + + o Minor features (onion services): + - Add warning message when connecting to now deprecated v2 onion + services. As announced, Tor 0.4.5.x is the last series that will + support v2 onions. Closes ticket 40373. + + o Minor bugfixes (bridge, pluggable transport, backport from 0.4.6.2-alpha): + - Fix a regression that made it impossible start Tor using a bridge + line with a transport name and no fingerprint. Fixes bug 40360; + bugfix on 0.4.5.4-rc. + + o Minor bugfixes (build, cross-compilation, backport from 0.4.6.3-rc): + - Allow a custom "ar" for cross-compilation. Our previous build + script had used the $AR environment variable in most places, but + it missed one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. + + o Minor bugfixes (channel, DoS, backport from 0.4.6.2-alpha): + - Fix a non-fatal BUG() message due to a too-early free of a string, + when listing a client connection from the DoS defenses subsystem. + Fixes bug 40345; bugfix on 0.4.3.4-rc. + + o Minor bugfixes (compiler warnings, backport from 0.4.6.3-rc): + - Fix an indentation problem that led to a warning from GCC 11.1.1. + Fixes bug 40380; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (controller, backport from 0.4.6.1-alpha): + - Fix a "BUG" warning that would appear when a controller chooses + the first hop for a circuit, and that circuit completes. Fixes bug + 40285; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, client, memory leak, backport from 0.4.6.3-rc): + - Fix a bug where an expired cached descriptor could get overwritten + with a new one without freeing it, leading to a memory leak. Fixes + bug 40356; bugfix on 0.3.5.1-alpha. + + o Minor bugfixes (testing, BSD, backport from 0.4.6.2-alpha): + - Fix pattern-matching errors when patterns expand to invalid paths + on BSD systems. Fixes bug 40318; bugfix on 0.4.5.1-alpha. Patch by + Daniel Pinto. + + Changes in version 0.3.5.14 - 2021-03-16 Tor 0.3.5.14 backports fixes for two important denial-of-service bugs in earlier versions of Tor. diff --git a/changes/autoconf-2.70 b/changes/autoconf-2.70 deleted file mode 100644 index 27a9f243b1..0000000000 --- a/changes/autoconf-2.70 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (compilation): - - Make the autoconf script build correctly with autoconf versions 2.70 - and later. Closes part of ticket 40335. diff --git a/changes/bug40078 b/changes/bug40078 deleted file mode 100644 index 717309e076..0000000000 --- a/changes/bug40078 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfix (crypto): - - Disable the unused batch verification feature of ed25519-donna. Fixes - bug 40078; bugfix on 0.2.6.1-alpha. Found by Henry de Valence.
\ No newline at end of file diff --git a/changes/bug40175 b/changes/bug40175 deleted file mode 100644 index aa2ce9566f..0000000000 --- a/changes/bug40175 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (logging, diagnostic): - - Log decompression failures at a higher severity level, since they - can help provide missing context for other warning messages. - We rate-limit these messages, to avoid flooding the logs if they - begin to occur frequently. Closes ticket 40175. diff --git a/changes/bug40317 b/changes/bug40317 deleted file mode 100644 index 18ec499a51..0000000000 --- a/changes/bug40317 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (control, sandbox): - - Allows the control command SAVECONF to succeed when the seccomp - sandbox is enabled. Makes SAVECONF keep only one backup file to - simplify implementation. Fixes bug 40317; bugfix on 0.2.5.4-alpha. - Patch by Daniel Pinto. diff --git a/changes/bug40371 b/changes/bug40371 deleted file mode 100644 index 8cc7117f9f..0000000000 --- a/changes/bug40371 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (compatibility): - - Fix compatibility with the most recent Libevent versions, which - no longer have an evdns_set_random_bytes() function. Because - this function has been a no-op since Libevent 2.0.4-alpha, - it is safe for us to just stop calling it. Fixes bug 40371; - bugfix on 0.2.1.7-alpha. diff --git a/changes/bug40375 b/changes/bug40375 deleted file mode 100644 index 7ac32bc628..0000000000 --- a/changes/bug40375 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (consensus handling): - - Avoid a set of bugs that could be caused by inconsistently preferring - an out-of-date consensus stored in a stale directory cache over - a more recent one stored on disk as the latest consensus. - Fixes bug 40375; bugfix on 0.3.1.1-alpha. diff --git a/changes/bug40380 b/changes/bug40380 deleted file mode 100644 index 24d2876b7d..0000000000 --- a/changes/bug40380 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compiler warnings): - - Fix an indentation problem that led to a warning from GCC 11.1.1. - Fixes bug 40380; bugfix on 0.3.0.1-alpha. diff --git a/changes/bug40383 b/changes/bug40383 deleted file mode 100644 index c4ca46fac7..0000000000 --- a/changes/bug40383 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (timekeeping): - - Calculate the time of day correctly on systems where the time_t - type includes leap seconds. (This is not the case on most - operating systems, but on those where it occurs, our tor_timegm - function did not correctly invert the system's gmtime function, - which could result in assertion failures when calculating - voting schedules.) Fixes bug 40383; bugfix on 0.2.0.3-alpha. diff --git a/changes/bug40391 b/changes/bug40391 deleted file mode 100644 index e3c186275f..0000000000 --- a/changes/bug40391 +++ /dev/null @@ -1,9 +0,0 @@ - o Major bugfixes (security): - - Resist a hashtable-based CPU denial-of-service attack against - relays. Previously we used a naive unkeyed hash function to look up - circuits in a circuitmux object. An attacker could exploit this to - construct circuits with chosen circuit IDs in order to try to create - collisions and make the hash table inefficient. Now we use a SipHash - construction for this hash table instead. Fixes bug 40391; bugfix on - 0.2.4.4-alpha. This issue is also tracked as TROVE-2021-005. - Reported by Jann Horn from Google's Project Zero. diff --git a/changes/bug40392 b/changes/bug40392 deleted file mode 100644 index 4dffa50bb2..0000000000 --- a/changes/bug40392 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (security, denial of service, onion services): - - Fix an out-of-bounds memory access in v3 descriptor parsing. Fixes bug - 40392; bugfix on 0.3.0.1-alpha. This issue is also tracked as - TROVE-2021-006. Reported by Sergei Glazunov from Google's Project Zero.
\ No newline at end of file diff --git a/changes/bug40399 b/changes/bug40399 deleted file mode 100644 index 7954b85eaf..0000000000 --- a/changes/bug40399 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor features (compatibility): - - Remove an assertion function related to TLS renegotiation. - It was used nowhere outside the unit tests, and it was breaking - compilation with recent alpha releases of OpenSSL 3.0.0. - Closes ticket 40399. diff --git a/changes/bug40409 b/changes/bug40409 deleted file mode 100644 index b8d061be78..0000000000 --- a/changes/bug40409 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (warnings, portability): - - Suppress a strict-prototype warning when building with some versions - of NSS. Fixes bug 40409; bugfix on 0.3.5.1-alpha. diff --git a/changes/bug40429 b/changes/bug40429 deleted file mode 100644 index 9bf3b63818..0000000000 --- a/changes/bug40429 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (compilation): - - Fix our configuration logic to detect whether we had OpenSSL 3: - previously, our logic was reversed. This has no other effect than to - change whether we suppress deprecated API warnings. Fixes - bug 40429; bugfix on 0.3.5.13. diff --git a/changes/fallbackdirs-2021-12-15 b/changes/fallbackdirs-2021-12-15 deleted file mode 100644 index ce08cef5cb..0000000000 --- a/changes/fallbackdirs-2021-12-15 +++ /dev/null @@ -1,2 +0,0 @@ - o Minor features (fallbackdir): - - Regenerate fallback directories generated on December 15, 2021. diff --git a/changes/geoip-2021-05-07 b/changes/geoip-2021-05-07 deleted file mode 100644 index 07bf12c4d8..0000000000 --- a/changes/geoip-2021-05-07 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (geoip data): - - Update the geoip files to match the IPFire Location Database, - as retrieved on 2021/05/07. diff --git a/changes/geoip-2021-06-10 b/changes/geoip-2021-06-10 deleted file mode 100644 index 2b798012c8..0000000000 --- a/changes/geoip-2021-06-10 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (geoip data): - - Update the geoip files to match the IPFire Location Database, - as retrieved on 2021/06/10. diff --git a/changes/geoip-2021-08-12 b/changes/geoip-2021-08-12 deleted file mode 100644 index 59afcc5bb7..0000000000 --- a/changes/geoip-2021-08-12 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (geoip data): - - Update the geoip files to match the IPFire Location Database, - as retrieved on 2021/08/12. diff --git a/changes/geoip-2021-12-15 b/changes/geoip-2021-12-15 deleted file mode 100644 index 5123557138..0000000000 --- a/changes/geoip-2021-12-15 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (geoip data): - - Update the geoip files to match the IPFire Location Database, - as retrieved on 2021/12/15. diff --git a/changes/ticket26299 b/changes/ticket26299 deleted file mode 100644 index 6b08adf53c..0000000000 --- a/changes/ticket26299 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor feature (reproducible build): - - The repository can now build reproducible tarballs which adds the build - command "make dist-reprod" for that purpose. Closes ticket 26299. diff --git a/changes/ticket30477 b/changes/ticket30477 deleted file mode 100644 index 379fc4e7eb..0000000000 --- a/changes/ticket30477 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (bridge): - - We now announce the URL to Tor's new bridge status at - https://bridges.torproject.org/ when Tor is configured to run as a bridge - relay. Closes ticket 30477. diff --git a/changes/ticket40290 b/changes/ticket40290 deleted file mode 100644 index 3d3a64be93..0000000000 --- a/changes/ticket40290 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfix (logging, relay): - - Emit a warning if an Address is found to be internal and tor can't use it. - Fixes bug 40290; bugfix on 0.4.5.1-alpha. diff --git a/changes/ticket40301 b/changes/ticket40301 deleted file mode 100644 index c1fd821e3f..0000000000 --- a/changes/ticket40301 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (relay): - - Reduce the compression level for data streaming from HIGH to LOW. Fixes - bug 40301; bugfix on 0.3.5.1-alpha. - diff --git a/changes/ticket40337 b/changes/ticket40337 deleted file mode 100644 index 1c86fc4c99..0000000000 --- a/changes/ticket40337 +++ /dev/null @@ -1,16 +0,0 @@ - o Minor features (testing): - - On a testing network, relays can now use the - TestingMinTimeToReportBandwidth option to change - the smallest amount of time over which they're willing to report - their observed maximum bandwidth. Previously, this was fixed - at 1 day. For safety, values under 2 hours are only supported on - testing networks. Part of a fix for ticket 40337. - - o Minor features (testing): - - Relays on testing networks now report their observed bandwidths - immediately from startup. Previously, they waited - until they had been running for a full day. Closes ticket - 40337. - - Relays on testing networks no longer rate-limit how frequently - they are willing to report new bandwidth measurements. Part of a fix - for ticket 40337. diff --git a/changes/ticket40356 b/changes/ticket40356 deleted file mode 100644 index 59c32ce0cc..0000000000 --- a/changes/ticket40356 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfix (onion service, client, memory leak): - - An expired cached descriptor could have been overwritten with a new one - leading to a memory leak. Fixes bug 40356; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket40369 b/changes/ticket40369 deleted file mode 100644 index abb59a7125..0000000000 --- a/changes/ticket40369 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (build, cross-compilation): - - Allow a custom "ar" for cross-compilation. Our previous build script - had used the $AR environment variable in most places, but it missed - one. Fixes bug 40369; bugfix on 0.4.5.1-alpha. diff --git a/changes/ticket40370 b/changes/ticket40370 deleted file mode 100644 index fcdb0eb173..0000000000 --- a/changes/ticket40370 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfix (metrics port): - - Fix a bug that made tor try to re-bind() every 60 seconds on an already - open MetricsPort. Fixes bug 40370; bugfix on 0.4.5.1-alpha. - diff --git a/changes/ticket40373 b/changes/ticket40373 deleted file mode 100644 index 3b2edd0652..0000000000 --- a/changes/ticket40373 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (onion services): - - Add warning message when connecting to deprecated v2 onions. - Closes ticket 40373.
\ No newline at end of file diff --git a/changes/ticket40374 b/changes/ticket40374 deleted file mode 100644 index 8b6e99b8af..0000000000 --- a/changes/ticket40374 +++ /dev/null @@ -1,4 +0,0 @@ - o Removed features: - - Remove unneeded code for parsing private keys in directory documents. - This code was only used for client authentication in v2 onion - services, which are now unsupported. Closes ticket 40374. diff --git a/changes/ticket40378 b/changes/ticket40378 deleted file mode 100644 index 35b2fd7bd4..0000000000 --- a/changes/ticket40378 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (onion service, control port): - - Make the ADD_ONION command properly configure client authorization. Before - this fix, the created onion failed to add the client(s). Fixes bug 40378; - bugfix on 0.4.6.1-alpha. diff --git a/changes/ticket40382 b/changes/ticket40382 deleted file mode 100644 index 5ac1b771b9..0000000000 --- a/changes/ticket40382 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (compatibility, Linux seccomp sandbox): - - Add a workaround to enable the Linux sandbox to work correctly - on systems running Glibc 2.33. These versions have started - using the fstatat() system call, which previously our sandbox did not - allow. - Closes ticket 40382; see the ticket for a discussion of tradeoffs. diff --git a/changes/ticket40389 b/changes/ticket40389 deleted file mode 100644 index 7dcf65b32e..0000000000 --- a/changes/ticket40389 +++ /dev/null @@ -1,3 +0,0 @@ - o Major bugfixes (relay, TROVE): - - Don't allow entry or middle relays to spoof RELAY_END or RELAY_RESOLVED - cell on half-closed streams. Fixes bug 40389; bugfix on 0.3.5.1-alpha. diff --git a/changes/ticket40390 b/changes/ticket40390 deleted file mode 100644 index b56fa4d9da..0000000000 --- a/changes/ticket40390 +++ /dev/null @@ -1,8 +0,0 @@ - o Major bugfixes (security, defense-in-depth): - - Detect a wider variety of failure conditions from the OpenSSL RNG - code. Previously, we would detect errors from a missing RNG - implementation, but not failures from the RNG code itself. - Fortunately, it appears those failures do not happen in practice - when Tor is using OpenSSL's default RNG implementation. - Fixes bug 40390; bugfix on 0.2.8.1-alpha. This issue is also tracked as - TROVE-2021-004. Reported by Jann Horn at Google's Project Zero. diff --git a/changes/ticket40410 b/changes/ticket40410 deleted file mode 100644 index 658e0683c7..0000000000 --- a/changes/ticket40410 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation): - - Fix a compilation error when trying to build Tor with a compiler that - does not support expanding statitically initialized const values in - macro's. Fixes bug 40410; bugfix on 0.4.5.1-alpha diff --git a/changes/ticket40419 b/changes/ticket40419 deleted file mode 100644 index 0004329662..0000000000 --- a/changes/ticket40419 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (testing): - - Enable the deterministic RNG for unit tests that covers the address set - bloomfilter-based API's. Fixes bug 40419; bugfix on 0.3.3.2-alpha. diff --git a/changes/ticket40421 b/changes/ticket40421 deleted file mode 100644 index d2dcc5533d..0000000000 --- a/changes/ticket40421 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (onion service): - - Send back the extended SOCKS error 0xF6 for a v2 onion address. Fixes bug - 40421; bugfix on 0.4.6.2-alpha. diff --git a/changes/ticket40434 b/changes/ticket40434 deleted file mode 100644 index 988bb416be..0000000000 --- a/changes/ticket40434 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfix (onion service): - - Do not flag an HSDir as non-running in case the descriptor upload or - fetch fails. An onion service closes pending directory connections - before uploading a new descriptor which can thus lead to wrongly - flagging many relays and thus affecting circuit building path selection. - Fixes bug 40434; bugfix on 0.2.0.13-alpha. diff --git a/changes/ticket40447 b/changes/ticket40447 deleted file mode 100644 index d1be646a7d..0000000000 --- a/changes/ticket40447 +++ /dev/null @@ -1,2 +0,0 @@ - o Minor feature (fallbackdir): - - Regenerate fallback directories list. Close ticket 40447. diff --git a/changes/ticket40474 b/changes/ticket40474 deleted file mode 100644 index d2a7231106..0000000000 --- a/changes/ticket40474 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (onion service, TROVE-2021-008): - - Only log once any v2 access attempts in order to not pollute the logs - with warnings and avoid recording the times on disk when v2 access was - attempted. Important to note that the onion address was _never_ logged. - That is a Low security issue. Fixes bug 40474; bugfix on 0.4.5.8. diff --git a/changes/ticket40476 b/changes/ticket40476 deleted file mode 100644 index 86e4377a1c..0000000000 --- a/changes/ticket40476 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfix (onion service): - - Improve logging when a bad HS version is given. Fixes bug 40476; bugfix on - 0.4.6.1-alpha. diff --git a/changes/ticket40491 b/changes/ticket40491 deleted file mode 100644 index 01c6c7d748..0000000000 --- a/changes/ticket40491 +++ /dev/null @@ -1,7 +0,0 @@ - o Major bugfixes (relay, overload state): - - Report the general overload state for DNS timeout errors only if X% of all - DNS queries over Y seconds are errors. Before that, it only took 1 timeout - to report the overload state which was just too low of a threshold. The X - and Y values are 1% and 10 minutes respectively but they are also - controlled by consensus parameters. Fixes bug 40491; bugfix on - 0.4.6.1-alpha. diff --git a/changes/ticket40493 b/changes/ticket40493 deleted file mode 100644 index eb9baf916b..0000000000 --- a/changes/ticket40493 +++ /dev/null @@ -1,2 +0,0 @@ - o Minor features (fallbackdir): - - Regenerate fallback directories for October 2021. Close ticket 40493. diff --git a/changes/ticket40494 b/changes/ticket40494 deleted file mode 100644 index a0e6c38443..0000000000 --- a/changes/ticket40494 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (relay): - - Reject IPv6-only DirPorts. Our reachability self-test forces DirPorts to - be IPv4, but our configuration parser allowed them to be IPv6-only, - which led to an assertion failure. Fixes bug 40494; bugfix on - 0.4.5.1-alpha. diff --git a/changes/ticket40500 b/changes/ticket40500 deleted file mode 100644 index 2e5ff39761..0000000000 --- a/changes/ticket40500 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfix (CI, onion service): - - Exclude onion service version 2 Stem tests in our CI. Fixes bug 40500; - bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket40504 b/changes/ticket40504 deleted file mode 100644 index 9095591419..0000000000 --- a/changes/ticket40504 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation (man, relay): - - Missing "OverloadStatistics" in tor.1 manpage. Fixes bug 40504; bugfix on - 0.4.6.1-alpha. diff --git a/changes/ticket40511 b/changes/ticket40511 deleted file mode 100644 index 756edd874d..0000000000 --- a/changes/ticket40511 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor features (compilation): - - Give an error message if trying to build with a version of LibreSSL - known not to work with Tor. (There's an incompatibility with - LibreSSL versions 3.2.1 through 3.4.0 inclusive because of their - incompatibility with OpenSSL 1.1.1's TLSv1.3 APIs.) - Closes ticket 40511. diff --git a/changes/ticket40527 b/changes/ticket40527 deleted file mode 100644 index 631b3d4bb9..0000000000 --- a/changes/ticket40527 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (relay, overload): - - Don't make Tor DNS timeout trigger an overload general state. These - timeouts are different from DNS server timeout. They have to be seen as - timeout related to UX and not because of a network problem. Fixes bug - 40527; bugfix on 0.4.6.1-alpha. diff --git a/config.rust.in b/config.rust.in deleted file mode 100644 index 11b671b980..0000000000 --- a/config.rust.in +++ /dev/null @@ -1,24 +0,0 @@ -# Used by our cargo build.rs script to get variables from autoconf. -# -# The "configure" script will generate "config.rust" from "config.rust.in", -# and then build.rs will read "config.rust". - -BUILDDIR=@BUILDDIR@ -TOR_LDFLAGS_zlib=@TOR_LDFLAGS_zlib@ -TOR_LDFLAGS_nss=@TOR_LDFLAGS_nss@ -TOR_LDFLAGS_openssl=@TOR_LDFLAGS_openssl@ -TOR_LDFLAGS_libevent=@TOR_LDFLAGS_libevent@ -TOR_ZLIB_LIBS=@TOR_ZLIB_LIBS@ -TOR_LIB_MATH=@TOR_LIB_MATH@ -TOR_LIBEVENT_LIBS=@TOR_LIBEVENT_LIBS@ -TOR_OPENSSL_LIBS=@TOR_OPENSSL_LIBS@ -TOR_LIB_WS32=@TOR_LIB_WS32@ -TOR_LIB_GDI=@TOR_LIB_GDI@ -TOR_LIB_USERENV=@TOR_LIB_USERENV@ -CURVE25519_LIBS=@CURVE25519_LIBS@ -TOR_SYSTEMD_LIBS=@TOR_SYSTEMD_LIBS@ -TOR_LZMA_LIBS=@TOR_LZMA_LIBS@ -TOR_ZSTD_LIBS=@TOR_ZSTD_LIBS@ -LIBS=@LIBS@ -LDFLAGS=@LDFLAGS@ -NSS_LIBS=@NSS_LIBS@
\ No newline at end of file diff --git a/configure.ac b/configure.ac index 2cd55798f5..3619ab56a3 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.4.6.9-dev]) +AC_INIT([tor],[0.4.7.3-alpha-dev])) AC_CONFIG_SRCDIR([src/app/main/tor_main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -18,7 +18,7 @@ AC_DEFINE_UNQUOTED([CONFIG_FLAGS], ["$configure_flags"], [Flags passed to config # version number changes. Tor uses it to make sure that it # only shuts down for missing "required protocols" when those protocols # are listed as required by a consensus after this date. -AC_DEFINE(APPROX_RELEASE_DATE, ["2021-12-16"], # for 0.4.6.9-dev +AC_DEFINE(APPROX_RELEASE_DATE, ["2021-12-16"], # for 0.4.7.3-alpha-dev [Approximate date when this software was released. (Updated when the version changes.)]) # "foreign" means we don't follow GNU package layout standards @@ -75,10 +75,6 @@ AC_ARG_ENABLE(oss-fuzz, AS_HELP_STRING(--enable-oss-fuzz, [build extra fuzzers based on 'oss-fuzz' environment])) AC_ARG_ENABLE(memory-sentinels, AS_HELP_STRING(--disable-memory-sentinels, [disable code that tries to prevent some kinds of memory access bugs. For fuzzing only.])) -AC_ARG_ENABLE(rust, - AS_HELP_STRING(--enable-rust, [enable rust integration])) -AC_ARG_ENABLE(cargo-online-mode, - AS_HELP_STRING(--enable-cargo-online-mode, [Allow cargo to make network requests to fetch crates. For builds with rust only.])) AC_ARG_ENABLE(restart-debugging, AS_HELP_STRING(--enable-restart-debugging, [Build Tor with support for debugging in-process restart. Developers only.])) AC_ARG_ENABLE(zstd-advanced-apis, @@ -115,10 +111,14 @@ AM_CONDITIONAL(COVERAGE_ENABLED, test "x$enable_coverage" = "xyes") AM_CONDITIONAL(DISABLE_ASSERTS_IN_UNIT_TESTS, test "x$enable_asserts_in_tests" = "xno") AM_CONDITIONAL(LIBFUZZER_ENABLED, test "x$enable_libfuzzer" = "xyes") AM_CONDITIONAL(OSS_FUZZ_ENABLED, test "x$enable_oss_fuzz" = "xyes") -AM_CONDITIONAL(USE_RUST, test "x$enable_rust" = "xyes") AM_CONDITIONAL(USE_NSS, test "x$enable_nss" = "xyes") AM_CONDITIONAL(USE_OPENSSL, test "x$enable_nss" != "xyes") +if test "x$enable_coverage" = "xyes"; then + AC_DEFINE(ENABLE_COVERAGE, 1, + [Defined if coverage support is enabled for the unit tests]) +fi + if test "x$enable_nss" = "xyes"; then AC_DEFINE(ENABLE_NSS, 1, [Defined if we're building with NSS.]) @@ -233,6 +233,7 @@ AC_ARG_ENABLE(fragile-hardening, AS_HELP_STRING(--enable-fragile-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower])) if test "x$enable_expensive_hardening" = "xyes" || test "x$enable_fragile_hardening" = "xyes"; then fragile_hardening="yes" + AC_DEFINE(ENABLE_FRAGILE_HARDENING, 1, [Defined if we're building with additional, fragile and expensive compiler hardening]) AC_DEFINE(DEBUG_SMARTLIST, 1, [Enable smartlist debugging]) fi @@ -477,13 +478,6 @@ fi AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"]) -dnl List all external rust crates we depend on here. Include the version -rust_crates=" \ - digest-0.7.2 \ - libc-0.2.39 \ -" -AC_SUBST(rust_crates) - ifdef([AC_C_FLEXIBLE_ARRAY_MEMBER], [ AC_C_FLEXIBLE_ARRAY_MEMBER ], [ @@ -636,105 +630,6 @@ fi AC_C_BIGENDIAN -AC_ARG_VAR([TOR_RUST_TARGET], [Rust target, must be specified when cross-compiling (HOST != BUILD). example: i686-pc-windows-gnu]) - -if test "x$enable_rust" = "xyes"; then - AC_ARG_VAR([RUSTC], [path to the rustc binary]) - AC_CHECK_PROG([RUSTC], [rustc], [rustc],[no]) - if test "x$RUSTC" = "xno"; then - AC_MSG_ERROR([rustc unavailable but rust integration requested.]) - fi - - AC_ARG_VAR([CARGO], [path to the cargo binary]) - AC_CHECK_PROG([CARGO], [cargo], [cargo],[no]) - if test "x$CARGO" = "xno"; then - AC_MSG_ERROR([cargo unavailable but rust integration requested.]) - fi - - AC_DEFINE([HAVE_RUST], 1, [have Rust]) - if test "x$enable_fatal_warnings" = "xyes"; then - RUST_WARN= - else - RUST_WARN=# - fi - if test "x$enable_cargo_online_mode" = "xyes"; then - CARGO_ONLINE= - RUST_DL=# - else - CARGO_ONLINE=--frozen - RUST_DL= - - dnl When we're not allowed to touch the network, we need crate dependencies - dnl locally available. - AC_MSG_CHECKING([rust crate dependencies]) - AC_ARG_VAR([TOR_RUST_DEPENDENCIES], [path to directory with local crate mirror]) - if test "x$TOR_RUST_DEPENDENCIES" = "x"; then - TOR_RUST_DEPENDENCIES="${srcdir}/src/ext/rust/crates" - fi - dnl Check whether the path exists before we try to cd into it. - if test ! -d "$TOR_RUST_DEPENDENCIES"; then - AC_MSG_ERROR([Rust dependency directory $TOR_RUST_DEPENDENCIES does not exist. Specify a dependency directory using the TOR_RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.]) - ERRORED=1 - fi - dnl Make the path absolute, since we'll be using it from within a - dnl subdirectory. - TOR_RUST_DEPENDENCIES=$(cd "$TOR_RUST_DEPENDENCIES" ; pwd) - - for dep in $rust_crates; do - if test ! -d "$TOR_RUST_DEPENDENCIES"/"$dep"; then - AC_MSG_ERROR([Failure to find rust dependency $TOR_RUST_DEPENDENCIES/$dep. Specify a dependency directory using the TOR_RUST_DEPENDENCIES variable or allow cargo to fetch crates using --enable-cargo-online-mode.]) - ERRORED=1 - fi - done - if test "x$ERRORED" = "x"; then - AC_MSG_RESULT([yes]) - fi - fi - - dnl For now both MSVC and MinGW rust libraries will output static libs with - dnl the MSVC naming convention. - if test "$bwin32" = "true"; then - tor_rust_static_name=tor_rust.lib - else - tor_rust_static_name=libtor_rust.a - fi - - AC_CANONICAL_BUILD - - if test -n "$TOR_RUST_TARGET"; then - if test "$host" = "$build"; then - AC_MSG_ERROR([HOST = BUILD is invalid if TOR_RUST_TARGET is specified, see configure --help for more information.]) - fi - RUST_TARGET_PROP="target = '$TOR_RUST_TARGET'" - TOR_RUST_LIB_PATH="src/rust/target/$TOR_RUST_TARGET/release/$tor_rust_static_name" - else - if test "$host" != "$build"; then - AC_MSG_ERROR([TOR_RUST_TARGET must be specified when cross-compiling with Rust enabled.]) - fi - RUST_TARGET_PROP= - TOR_RUST_LIB_PATH="src/rust/target/release/$tor_rust_static_name" - fi - - AC_SUBST(RUST_TARGET_PROP) - AC_SUBST(TOR_RUST_LIB_PATH) - AC_SUBST(CARGO_ONLINE) - AC_SUBST(RUST_WARN) - AC_SUBST(RUST_DL) - - dnl Let's check the rustc version, too - AC_MSG_CHECKING([rust version]) - RUSTC_VERSION=`$RUSTC --version` - RUSTC_VERSION_MAJOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 1` - RUSTC_VERSION_MINOR=`$RUSTC --version | cut -d ' ' -f 2 | cut -d '.' -f 2` - if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then - AC_MSG_ERROR([rustc version couldn't be identified]) - fi - if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 31; then - AC_MSG_ERROR([rustc must be at least version 1.31.0]) - fi - AC_MSG_RESULT([$RUSTC_VERSION]) -fi - AC_SEARCH_LIBS(socket, [socket network]) AC_SEARCH_LIBS(gethostbyname, [nsl]) AC_SEARCH_LIBS(dlopen, [dl]) @@ -794,6 +689,7 @@ AC_CHECK_FUNCS( readpassphrase \ rint \ sigaction \ + snprintf \ socketpair \ statvfs \ strncasecmp \ @@ -811,7 +707,8 @@ AC_CHECK_FUNCS( uname \ usleep \ vasprintf \ - _vscprintf + _vscprintf \ + vsnprintf ) # Apple messed up when they added some functions: they @@ -910,8 +807,6 @@ if test "$bwin32" = "true"; then TOR_LIB_WS32=-lws2_32 TOR_LIB_IPHLPAPI=-liphlpapi TOR_LIB_SHLWAPI=-lshlwapi - # Some of the cargo-cults recommend -lwsock32 as well, but I don't - # think it's actually necessary. TOR_LIB_GDI=-lgdi32 TOR_LIB_USERENV=-luserenv TOR_LIB_BCRYPT=-lbcrypt @@ -1467,33 +1362,6 @@ if test "$fragile_hardening" = "yes"; then TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) fi -dnl Find the correct libraries to add in order to use the sanitizers. -dnl -dnl When building Rust, Cargo will run the linker with the -nodefaultlibs -dnl option, which will prevent the compiler from linking the sanitizer -dnl libraries it needs. We need to specify them manually. -dnl -dnl What's more, we need to specify them in a linker script rather than -dnl from build.rs: these options aren't allowed in the cargo:rustc-flags -dnl variable. -RUST_LINKER_OPTIONS="" -if test "x$have_clang" = "xyes"; then - if test "x$CFLAGS_ASAN" != "x"; then - RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=$CFLAGS_ASAN -Cdefault-linker-libraries" - fi - if test "x$CFLAGS_UBSAN" != "x"; then - RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=$CFLAGS_UBSAN -Cdefault-linker-libraries" - fi -else - if test "x$CFLAGS_ASAN" != "x"; then - RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=-fsanitize=address -Cdefault-linker-libraries" - fi - if test "x$CFLAGS_UBSAN" != "x"; then - RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -Clink-arg=-fsanitize=undefined -Cdefault-linker-libraries" - fi -fi -AC_SUBST(RUST_LINKER_OPTIONS) - CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN" CFLAGS_CONSTTIME="$CFLAGS_FWRAPV" @@ -2704,11 +2572,9 @@ CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_z AC_CONFIG_FILES([ Doxyfile Makefile - config.rust contrib/operator-tools/tor.logrotate src/config/torrc.sample src/config/torrc.minimal - src/rust/.cargo/config scripts/maint/checkOptionDocs.pl warning_flags ]) @@ -2784,12 +2650,6 @@ AS_ECHO test "x$enable_fatal_warnings" = "xyes" && value=1 || value=0 PPRINT_PROP_BOOL([Warnings are fatal (--enable-fatal-warnings)], $value) -test "x$enable_rust" = "xyes" && value=1 || value=0 -PPRINT_PROP_BOOL([Rust support (--enable-rust)], $value) - -test "x$enable_cargo_online_mode" = "xyes" && value=1 || value=0 -PPRINT_PROP_BOOL([Cargo Online Fetch (--enable-cargo-online-mode)], $value) - test "x$enable_android" = "xyes" && value=1 || value=0 PPRINT_PROP_BOOL([Android support (--enable-android)], $value) diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index 9ac9026223..60389afc5e 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.4.6.9-dev" +!define VERSION "0.4.7.3-alpha-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/CircuitPaddingDevelopment.md b/doc/HACKING/CircuitPaddingDevelopment.md index 95ffbae4dd..e4aa9ddd09 100644 --- a/doc/HACKING/CircuitPaddingDevelopment.md +++ b/doc/HACKING/CircuitPaddingDevelopment.md @@ -381,11 +381,10 @@ use case. #### 2.2.2. Detecting and Negotiating Machine Support When a new machine specification is added to Tor (or removed from Tor), you -should bump the Padding subprotocol version in `src/core/or/protover.c` and -`src/rust/protover/protover.rs`, add a field to `protover_summary_flags_t` in -`or.h`, and set this field in `memoize_protover_summary()` in versions.c. This -new field must then be checked in `circpad_node_supports_padding()` in -`circuitpadding.c`. +should bump the Padding subprotocol version in `src/core/or/protover.c`, add a +field to `protover_summary_flags_t` in `or.h`, and set this field in +`memoize_protover_summary()` in versions.c. This new field must then be +checked in `circpad_node_supports_padding()` in `circuitpadding.c`. Note that this protocol version update and associated support check is not necessary if your experiments will *only* be using your own relays that diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index cd3417d0b5..c5dd6c744f 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -59,7 +59,7 @@ Some compatible licenses include: Each main development series (like 0.2.1, 0.2.2, etc) has its main work applied to a single branch. At most one series can be the development series at a time; all other series are maintenance series that get bug-fixes only. -The development series is built in a git branch called "master"; the +The development series is built in a git branch called "main"; the maintenance series are built in branches called "maint-0.2.0", "maint-0.2.1", and so on. We regularly merge the active maint branches forward. @@ -75,7 +75,7 @@ If you're working on a bugfix for a bug that occurs in a particular version, base your bugfix branch on the "maint" branch for the first supported series that has that bug. (As of June 2013, we're supporting 0.2.3 and later.) -If you're working on a new feature, base it on the master branch. If you're +If you're working on a new feature, base it on the main branch. If you're working on a new feature and it will take a while to implement and/or you'd like to avoid the possibility of unrelated bugs in Tor while you're implementing your feature, consider branching off of the latest maint- branch. diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md deleted file mode 100644 index c821465173..0000000000 --- a/doc/HACKING/CodingStandardsRust.md +++ /dev/null @@ -1,553 +0,0 @@ -# Rust Coding Standards - -You MUST follow the standards laid out in `doc/HACKING/CodingStandards.md`, -where applicable. - -## Module/Crate Declarations - -Each Tor C module which is being rewritten MUST be in its own crate. -See the structure of `src/rust` for examples. - -In your crate, you MUST use `lib.rs` ONLY for pulling in external -crates (e.g. `extern crate libc;`) and exporting public objects from -other Rust modules (e.g. `pub use mymodule::foo;`). For example, if -you create a crate in `src/rust/yourcrate`, your Rust code should -live in `src/rust/yourcrate/yourcode.rs` and the public interface -to it should be exported in `src/rust/yourcrate/lib.rs`. - -If your code is to be called from Tor C code, you MUST define a safe -`ffi.rs`. See the "Safety" section further down for more details. - -For example, in a hypothetical `tor_addition` Rust module: - -In `src/rust/tor_addition/addition.rs`: - -```rust -pub fn get_sum(a: i32, b: i32) -> i32 { - a + b -} -``` - -In `src/rust/tor_addition/lib.rs`: - -```rust -pub use addition::*; -``` - -In `src/rust/tor_addition/ffi.rs`: - -```rust -#[no_mangle] -pub extern "C" fn tor_get_sum(a: c_int, b: c_int) -> c_int { - get_sum(a, b) -} -``` - -If your Rust code must call out to parts of Tor's C code, you must -declare the functions you are calling in the `external` crate, located -at `src/rust/external`. - -<!-- XXX get better examples of how to declare these externs, when/how they --> -<!-- XXX are unsafe, what they are expected to do —isis --> - -Modules should strive to be below 500 lines (tests excluded). Single -responsibility and limited dependencies should be a guiding standard. - -If you have any external modules as dependencies (e.g. `extern crate -libc;`), you MUST declare them in your crate's `lib.rs` and NOT in any -other module. - -## Dependencies and versions - -In general, we use modules from only the Rust standard library -whenever possible. We will review including external crates on a -case-by-case basis. - -If a crate only contains traits meant for compatibility between Rust -crates, such as [the digest crate](https://crates.io/crates/digest) or -[the failure crate](https://crates.io/crates/failure), it is very likely -permissible to add it as a dependency. However, a brief review should -be conducted as to the usefulness of implementing external traits -(i.e. how widespread is the usage, how many other crates either -implement the traits or have trait bounds based upon them), as well as -the stability of the traits (i.e. if the trait is going to change, we'll -potentially have to re-do all our implementations of it). - -For large external libraries, especially which implement features which -would be labour-intensive to reproduce/maintain ourselves, such as -cryptographic or mathematical/statistics libraries, only crates which -have stabilised to 1.0.0 should be considered, however, again, we may -make exceptions on a case-by-case basis. - -Currently, Tor requires that you use the latest stable Rust version. At -some point in the future, we will freeze on a given stable Rust version, -to ensure backward compatibility with stable distributions that ship it. - -## Updating/Adding Dependencies - -To add/remove/update dependencies, first add your dependencies, -exactly specifying their versions, into the appropriate *crate-level* -`Cargo.toml` in `src/rust/` (i.e. *not* `/src/rust/Cargo.toml`, but -instead the one for your crate). Also, investigate whether your -dependency has any optional dependencies which are unnecessary but are -enabled by default. If so, you'll likely be able to enable/disable -them via some feature, e.g.: - -```toml -[dependencies] -foo = { version = "1.0.0", default-features = false } -``` - -Next, run `/scripts/maint/updateRustDependencies.sh`. Then, go into -`src/ext/rust` and commit the changes to the `tor-rust-dependencies` -repo. - -## Documentation - -You MUST include `#![deny(missing_docs)]` in your crate. - -For function/method comments, you SHOULD include a one-sentence, "first person" -description of function behaviour (see requirements for documentation as -described in `src/HACKING/CodingStandards.md`), then an `# Inputs` section -for inputs or initialisation values, a `# Returns` section for return -values/types, a `# Warning` section containing warnings for unsafe behaviours or -panics that could happen. For publicly accessible -types/constants/objects/functions/methods, you SHOULD also include an -`# Examples` section with runnable doctests. - -You MUST document your module with _module docstring_ comments, -i.e. `//!` at the beginning of each line. - -## Style - -You SHOULD consider breaking up large literal numbers with `_` when it makes it -more human readable to do so, e.g. `let x: u64 = 100_000_000_000`. - -## Testing - -All code MUST be unittested and integration tested. - -Public functions/objects exported from a crate SHOULD include doctests -describing how the function/object is expected to be used. - -Integration tests SHOULD go into a `tests/` directory inside your -crate. Unittests SHOULD go into their own module inside the module -they are testing, e.g. in `src/rust/tor_addition/addition.rs` you -should put: - -```rust -#[cfg(test)] -mod test { - use super::*; - -#[test] - fn addition_with_zero() { - let sum: i32 = get_sum(5i32, 0i32); - assert_eq!(sum, 5); - } -} -``` - -## Benchmarking - -The external `test` crate can be used for most benchmarking. However, using -this crate requires nightly Rust. Since we may want to switch to a more -stable Rust compiler eventually, we shouldn't do things which will automatically -break builds for stable compilers. Therefore, you MUST feature-gate your -benchmarks in the following manner. - -If you wish to benchmark some of your Rust code, you MUST put the -following in the `[features]` section of your crate's `Cargo.toml`: - -```toml -[features] -bench = [] -``` - -Next, in your crate's `lib.rs` you MUST put: - -```rust -#[cfg(all(test, feature = "bench"))] -extern crate test; -``` - -This ensures that the external crate `test`, which contains utilities -for basic benchmarks, is only used when running benchmarks via `cargo -bench --features bench`. - -Finally, to write your benchmark code, in -`src/rust/tor_addition/addition.rs` you SHOULD put: - -```rust -#[cfg(all(test, features = "bench"))] -mod bench { - use test::Bencher; - use super::*; - -#[bench] - fn addition_small_integers(b: &mut Bencher) { - b.iter(| | get_sum(5i32, 0i32)); - } -} -``` - -## Fuzzing - -If you wish to fuzz parts of your code, please see the -[cargo fuzz](https://github.com/rust-fuzz/cargo-fuzz) crate, which uses -[libfuzzer-sys](https://github.com/rust-fuzz/libfuzzer-sys). - -## Whitespace & Formatting - -You MUST run `rustfmt` (https://github.com/rust-lang-nursery/rustfmt) -on your code before your code will be merged. You can install rustfmt -by doing `cargo install rustfmt-nightly` and then run it with `cargo -fmt`. - -## Safety - -You SHOULD read [the nomicon](https://doc.rust-lang.org/nomicon/) before writing -Rust FFI code. It is *highly advised* that you read and write normal Rust code -before attempting to write FFI or any other unsafe code. - -Here are some additional bits of advice and rules: - -0. Any behaviours which Rust considers to be undefined are forbidden - - From https://doc.rust-lang.org/reference/behavior-considered-undefined.html: - - > Behavior considered undefined - > - > The following is a list of behavior which is forbidden in all Rust code, - > including within unsafe blocks and unsafe functions. Type checking provides the - > guarantee that these issues are never caused by safe code. - > - > * Data races - > * Dereferencing a null/dangling raw pointer - > * Reads of [undef](https://llvm.org/docs/LangRef.html#undefined-values) - > (uninitialized) memory - > * Breaking the - > [pointer aliasing rules](https://llvm.org/docs/LangRef.html#pointer-aliasing-rules) - > with raw pointers (a subset of the rules used by C) - > * `&mut T` and `&T` follow LLVM’s scoped noalias model, except if the `&T` - > contains an `UnsafeCell<U>`. Unsafe code must not violate these aliasing - > guarantees. - > * Mutating non-mutable data (that is, data reached through a shared - > reference or data owned by a `let` binding), unless that data is - > contained within an `UnsafeCell<U>`. - > * Invoking undefined behavior via compiler intrinsics: - > - Indexing outside of the bounds of an object with - > `std::ptr::offset` (`offset` intrinsic), with the exception of - > one byte past the end which is permitted. - > - Using `std::ptr::copy_nonoverlapping_memory` (`memcpy32`/`memcpy64` - > intrinsics) on overlapping buffers - > * Invalid values in primitive types, even in private fields/locals: - > - Dangling/null references or boxes - > - A value other than `false` (0) or `true` (1) in a `bool` - > - A discriminant in an `enum` not included in the type definition - > - A value in a `char` which is a surrogate or above `char::MAX` - > - Non-UTF-8 byte sequences in a `str` - > * Unwinding into Rust from foreign code or unwinding from Rust into foreign - > code. Rust's failure system is not compatible with exception handling in other - > languages. Unwinding must be caught and handled at FFI boundaries. - -1. `unwrap()` - - If you call `unwrap()`, anywhere, even in a test, you MUST include - an inline comment stating how the unwrap will either 1) never fail, - or 2) should fail (i.e. in a unittest). - - You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the - potential error with the eel operator, `?` or another non panicking way. - For example, consider a function which parses a string into an integer: - - ```rust - fn parse_port_number(config_string: &str) -> u16 { - u16::from_str_radix(config_string, 10).unwrap() - } - ``` - - There are numerous ways this can fail, and the `unwrap()` will cause the - whole program to byte the dust! Instead, either you SHOULD use `ok()` - (or another equivalent function which will return an `Option` or a `Result`) - and change the return type to be compatible: - - ```rust - fn parse_port_number(config_string: &str) -> Option<u16> { - u16::from_str_radix(config_string, 10).ok() - } - ``` - - or you SHOULD use `or()` (or another similar method): - - ```rust - fn parse_port_number(config_string: &str) -> Option<u16> { - u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16") - } - ``` - - Using methods like `or()` can be particularly handy when you must do - something afterwards with the data, for example, if we wanted to guarantee - that the port is high. Combining these methods with the eel operator (`?`) - makes this even easier: - - ```rust - fn parse_port_number(config_string: &str) -> Result<u16, Err> { - let port = u16::from_str_radix(config_string, 10).or(Err("Couldn't parse port into a u16"))?; - - if port > 1024 { - return Ok(port); - } else { - return Err("Low ports not allowed"); - } - } - ``` - -2. `unsafe` - - If you use `unsafe`, you MUST describe a contract in your - documentation which describes how and when the unsafe code may - fail, and what expectations are made w.r.t. the interfaces to - unsafe code. This is also REQUIRED for major pieces of FFI between - C and Rust. - - When creating an FFI in Rust for C code to call, it is NOT REQUIRED - to declare the entire function `unsafe`. For example, rather than doing: - - ```rust - #[no_mangle] - pub unsafe extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 { - for number in &mut numbers { - *number += 1; - } - std::mem::transmute::<[u8; 4], u32>(numbers) - } - ``` - - You SHOULD instead do: - - ```rust - #[no_mangle] - pub extern "C" fn increment_and_combine_numbers(mut numbers: [u8; 4]) -> u32 { - for index in 0..numbers.len() { - numbers[index] += 1; - } - unsafe { - std::mem::transmute::<[u8; 4], u32>(numbers) - } - } - ``` - -3. Pass only C-compatible primitive types and bytes over the boundary - - Rust's C-compatible primitive types are integers and floats. - These types are declared in the [libc crate](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types). - Most Rust objects have different [representations](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types) - in C and Rust, so they can't be passed using FFI. - - Tor currently uses the following Rust primitive types from libc for FFI: - * defined-size integers: `uint32_t` - * native-sized integers: `c_int` - * native-sized floats: `c_double` - * native-sized raw pointers: `* c_void`, `* c_char`, `** c_char` - - TODO: C smartlist to Stringlist conversion using FFI - - The only non-primitive type which may cross the FFI boundary is - bytes, e.g. `&[u8]`. This SHOULD be done on the Rust side by - passing a pointer (`*mut libc::c_char`). The length can be passed - explicitly (`libc::size_t`), or the string can be NUL-byte terminated - C string. - - One might be tempted to do this via doing - `CString::new("blah").unwrap().into_raw()`. This has several problems: - - a) If you do `CString::new("bl\x00ah")` then the unwrap() will fail - due to the additional NULL terminator, causing a dangling - pointer to be returned (as well as a potential use-after-free). - - b) Returning the raw pointer will cause the CString to run its deallocator, - which causes any C code which tries to access the contents to dereference a - NULL pointer. - - c) If we were to do `as_raw()` this would result in a potential double-free - since the Rust deallocator would run and possibly Tor's deallocator. - - d) Calling `into_raw()` without later using the same pointer in Rust to call - `from_raw()` and then deallocate in Rust can result in a - [memory leak](https://doc.rust-lang.org/std/ffi/struct.CString.html#method.into_raw). - - [It was determined](https://github.com/rust-lang/rust/pull/41074) that this - is safe to do if you use the same allocator in C and Rust and also specify - the memory alignment for CString (except that there is no way to specify - the alignment for CString). It is believed that the alignment is always 1, - which would mean it's safe to dealloc the resulting `*mut c_char` in Tor's - C code. However, the Rust developers are not willing to guarantee the - stability of, or a contract for, this behaviour, citing concerns that this - is potentially extremely and subtly unsafe. - -4. Perform an allocation on the other side of the boundary - - After crossing the boundary, the other side MUST perform an - allocation to copy the data and is therefore responsible for - freeing that memory later. - -5. No touching other language's enums - - Rust enums should never be touched from C (nor can they be safely - `#[repr(C)]`) nor vice versa: - - > "The chosen size is the default enum size for the target platform's C - > ABI. Note that enum representation in C is implementation defined, so this is - > really a "best guess". In particular, this may be incorrect when the C code - > of interest is compiled with certain flags." - - (from https://gankro.github.io/nomicon/other-reprs.html) - -6. Type safety - - Wherever possible and sensical, you SHOULD create new types in a - manner which prevents type confusion or misuse. For example, - rather than using an untyped mapping between strings and integers - like so: - - ```rust - use std::collections::HashMap; - - pub fn get_elements_with_over_9000_points(map: &HashMap<String, usize>) -> Vec<String> { - ... - } - ``` - - It would be safer to define a new type, such that some other usage - of `HashMap<String, usize>` cannot be confused for this type: - - ```rust - pub struct DragonBallZPowers(pub HashMap<String, usize>); - - impl DragonBallZPowers { - pub fn over_nine_thousand<'a>(&'a self) -> Vec<&'a String> { - let mut powerful_enough: Vec<&'a String> = Vec::with_capacity(5); - - for (character, power) in &self.0 { - if *power > 9000 { - powerful_enough.push(character); - } - } - powerful_enough - } - } - ``` - - Note the following code, which uses Rust's type aliasing, is valid - but it does NOT meet the desired type safety goals: - - ```rust - pub type Power = usize; - - pub fn over_nine_thousand(power: &Power) -> bool { - if *power > 9000 { - return true; - } - false - } - - // We can still do the following: - let his_power: usize = 9001; - over_nine_thousand(&his_power); - ``` - -7. Unsafe mucking around with lifetimes - - Because lifetimes are technically, in type theory terms, a kind, i.e. a - family of types, individual lifetimes can be treated as types. For example, - one can arbitrarily extend and shorten lifetime using `std::mem::transmute`: - - ```rust - struct R<'a>(&'a i32); - - unsafe fn extend_lifetime<'b>(r: R<'b>) -> R<'static> { - std::mem::transmute::<R<'b>, R<'static>>(r) - } - - unsafe fn shorten_invariant_lifetime<'b, 'c>(r: &'b mut R<'static>) -> &'b mut R<'c> { - std::mem::transmute::<&'b mut R<'static>, &'b mut R<'c>>(r) - } - ``` - - Calling `extend_lifetime()` would cause an `R` passed into it to live forever - for the life of the program (the `'static` lifetime). Similarly, - `shorten_invariant_lifetime()` could be used to take something meant to live - forever, and cause it to disappear! This is incredibly unsafe. If you're - going to be mucking around with lifetimes like this, first, you better have - an extremely good reason, and second, you may as be honest and explicit about - it, and for ferris' sake just use a raw pointer. - - In short, just because lifetimes can be treated like types doesn't mean you - should do it. - -8. Doing excessively unsafe things when there's a safer alternative - - Similarly to #7, often there are excessively unsafe ways to do a task and a - simpler, safer way. You MUST choose the safer option where possible. - - For example, `std::mem::transmute` can be abused in ways where casting with - `as` would be both simpler and safer: - - ```rust - // Don't do this - let ptr = &0; - let ptr_num_transmute = unsafe { std::mem::transmute::<&i32, usize>(ptr)}; - - // Use an `as` cast instead - let ptr_num_cast = ptr as *const i32 as usize; - ``` - - In fact, using `std::mem::transmute` for *any* reason is a code smell and as - such SHOULD be avoided. - -9. Casting integers with `as` - - This is generally fine to do, but it has some behaviours which you should be - aware of. Casting down chops off the high bits, e.g.: - - ```rust - let x: u32 = 4294967295; - println!("{}", x as u16); // prints 65535 - ``` - - Some cases which you MUST NOT do include: - - * Casting an `u128` down to an `f32` or vice versa (e.g. - `u128::MAX as f32` but this isn't only a problem with overflowing - as it is also undefined behaviour for `42.0f32 as u128`), - - * Casting between integers and floats when the thing being cast - cannot fit into the type it is being casted into, e.g.: - - ```rust - println!("{}", 42949.0f32 as u8); // prints 197 in debug mode and 0 in release - println!("{}", 1.04E+17 as u8); // prints 0 in both modes - println!("{}", (0.0/0.0) as i64); // prints whatever the heck LLVM wants - ``` - - Because this behaviour is undefined, it can even produce segfaults in - safe Rust code. For example, the following program built in release - mode segfaults: - - ```rust - #[inline(never)] - pub fn trigger_ub(sl: &[u8; 666]) -> &[u8] { - // Note that the float is out of the range of `usize`, invoking UB when casting. - let idx = 1e99999f64 as usize; - &sl[idx..] // The bound check is elided due to `idx` being of an undefined value. - } - - fn main() { - println!("{}", trigger_ub(&[1; 666])[999999]); // ~ out of bound - } - ``` - - And in debug mode panics with: - - thread 'main' panicked at 'slice index starts at 140721821254240 but ends at 666', /checkout/src/libcore/slice/mod.rs:754:4 diff --git a/doc/HACKING/GettingStarted.md b/doc/HACKING/GettingStarted.md index 6d61be9881..271e2d7517 100644 --- a/doc/HACKING/GettingStarted.md +++ b/doc/HACKING/GettingStarted.md @@ -41,7 +41,7 @@ Once you've reached this point, here's what you need to know. $ git clone https://git.torproject.org/git/tor ``` - This will give you a checkout of the master branch. If you're + This will give you a checkout of the main branch. If you're going to fix a bug that appears in a stable version, check out the appropriate "maint" branch, as in: diff --git a/doc/HACKING/GettingStartedRust.md b/doc/HACKING/GettingStartedRust.md deleted file mode 100644 index beef825226..0000000000 --- a/doc/HACKING/GettingStartedRust.md +++ /dev/null @@ -1,187 +0,0 @@ -# Hacking on Rust in Tor - -## Getting Started - -Please read or review our documentation on Rust coding standards -(`doc/HACKING/CodingStandardsRust.md`) before doing anything. - -Please also read -[the Rust Code of Conduct](https://www.rust-lang.org/en-US/conduct.html). We -aim to follow the good example set by the Rust community and be -excellent to one another. Let's be careful with each other, so we can -be memory-safe together! - -Next, please contact us before rewriting anything! Rust in Tor is still -an experiment. It is an experiment that we very much want to see -succeed, so we're going slowly and carefully. For the moment, it's also -a completely volunteer-driven effort: while many, if not most, of us are -paid to work on Tor, we are not yet funded to write Rust code for Tor. -Please be patient with the other people who are working on getting more -Rust code into Tor, because they are graciously donating their free time -to contribute to this effort. - -## Resources for learning Rust - -**Beginning resources** - -The primary resource for learning Rust is -[The Book](https://doc.rust-lang.org/book/). If you'd like to start writing -Rust immediately, without waiting for anything to install, there is -[an interactive browser-based playground](https://play.rust-lang.org/). - -**Advanced resources** - -If you're interested in playing with various Rust compilers and viewing -a very nicely displayed output of the generated assembly, there is -[the Godbolt compiler explorer](https://rust.godbolt.org/) - -For learning how to write unsafe Rust, read -[The Rustonomicon](https://doc.rust-lang.org/nomicon/). - -For learning everything you ever wanted to know about Rust macros, there -is -[The Little Book of Rust Macros](https://danielkeep.github.io/tlborm/book/index.html). - -For learning more about FFI and Rust, see Jake Goulding's -[Rust FFI Omnibus](https://jakegoulding.com/rust-ffi-omnibus/). - -## Compiling Tor with Rust enabled - -You will need to run the `configure` script with the `--enable-rust` -flag to explicitly build with Rust. Additionally, you will need to -specify where to fetch Rust dependencies, as we allow for either -fetching dependencies from Cargo or specifying a local directory. - -**Fetch dependencies from Cargo** - -```console -$ ./configure --enable-rust --enable-cargo-online-mode -``` - -**Using a local dependency cache** - -You'll need the following Rust dependencies (as of this writing): - - libc==0.2.39 - -We vendor our Rust dependencies in a separate repo using -[cargo-vendor](https://github.com/alexcrichton/cargo-vendor). To use -them, do: - -```console -$ git submodule init -$ git submodule update -``` - -To specify the local directory containing the dependencies, (assuming -you are in the top level of the repository) configure tor with: - -```console -$ TOR_RUST_DEPENDENCIES='path_to_dependencies_directory' ./configure --enable-rust -``` - -(Note that `TOR_RUST_DEPENDENCIES` must be the full path to the directory; it -cannot be relative.) - -Assuming you used the above `git submodule` commands and you're in the -topmost directory of the repository, this would be: - -```console -$ TOR_RUST_DEPENDENCIES=`pwd`/src/ext/rust/crates ./configure --enable-rust -``` - -## Identifying which modules to rewrite - -The places in the Tor codebase that are good candidates for porting to -Rust are: - -1. loosely coupled to other Tor submodules, -2. have high test coverage, and -3. would benefit from being implemented in a memory safe language. - -Help in either identifying places such as this, or working to improve -existing areas of the C codebase by adding regression tests and -simplifying dependencies, would be really helpful. - -Furthermore, as submodules in C are implemented in Rust, this is a good -opportunity to refactor, add more tests, and split modules into smaller -areas of responsibility. - -A good first step is to build a module-level callgraph to understand how -interconnected your target module is. - -```console -$ git clone https://git.torproject.org/user/nickm/calltool.git -$ cd tor -$ CFLAGS=0 ./configure -$ ../calltool/src/main.py module_callgraph -``` - -The output will tell you each module name, along with a set of every module that -the module calls. Modules which call fewer other modules are better targets. - -## Writing your Rust module - -Strive to change the C API as little as possible. - -We are currently targeting Rust stable. (See `CodingStandardsRust.md` for more -details.) - -It is on our TODO list to try to cultivate good -standing with various distro maintainers of `rustc` and `cargo`, in -order to ensure that whatever version we solidify on is readily -available. - -If parts of your Rust code needs to stay in sync with C code (such as -handling enums across the FFI boundary), annonotate these places in a -comment structured as follows: - - `/// C_RUST_COUPLED: <path_to_file> <name_of_c_object>` - -Where `<name_of_c_object>` can be an enum, struct, constant, etc. Then, -do the same in the C code, to note that rust will need to be changed -when the C does. - -## Adding your Rust module to Tor's build system - -0. Your translation of the C module should live in its own crate(s) - in the `src/rust/` directory. -1. Add your crate to `src/rust/Cargo.toml`, in the - `[workspace.members]` section. -2. Add your crate's files to src/rust/include.am - -If your crate should be available to C (rather than just being included as a -dependency of other Rust modules): -0. Declare the crate as a dependency of tor_rust in - `src/rust/tor_util/Cargo.toml` and include it in - `src/rust/tor_rust/lib.rs` - -## How to test your Rust code - -Everything should be tested full stop. Even non-public functionality. - -Be sure to edit `src/test/test_rust.sh` to add the name of your -crate to the `crates` variable! This will ensure that `cargo test` is -run on your crate. - -Configure Tor's build system to build with Rust enabled: - -```console -$ ./configure --enable-fatal-warnings --enable-rust --enable-cargo-online-mode -``` - -Tor's test should be run by doing: - -```console -$ make check -``` - -Tor's integration tests should also pass: - -```console -$ make test-stem -``` - -## Submitting a patch - -Please follow the instructions in `doc/HACKING/GettingStarted.md`. diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md index 0ce59576f0..7849fc67c7 100644 --- a/doc/HACKING/HelpfulTools.md +++ b/doc/HACKING/HelpfulTools.md @@ -33,8 +33,8 @@ OFTC. If they don't, ask #tor-dev (also on OFTC). It's CI/builders. Looks like this: https://jenkins.torproject.org Runs automatically on commits merged to git.torproject.org. We CI the -master branch and all supported tor versions. We also build nightly debian -packages from master. +main branch and all supported tor versions. We also build nightly debian +packages from main. Builds Linux and Windows cross-compilation. Runs Linux tests. diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md index 4d5a7f6b76..267f6d0b58 100644 --- a/doc/HACKING/Maintaining.md +++ b/doc/HACKING/Maintaining.md @@ -7,7 +7,7 @@ The first section describes who is the current Tor maintainer and what are the responsibilities. Tor has one main single maintainer but does have many committers and subsystem maintainers. -The second third section describes how the **alpha and master** branches are +The second third section describes how the **alpha and main** branches are maintained and by whom. Finally, the last section describes how the **stable** branches are maintained @@ -25,7 +25,7 @@ protocol design. Releasing Tor falls under their responsibility. ## Alpha and Master Branches -The Tor repository always has at all times a **master** branch which contains +The Tor repository always has at all times a **main** branch which contains the upstream ongoing development. It may also contain a branch for a released feature freezed version which is @@ -37,7 +37,7 @@ Tor is separated into subsystems and some of those are maintained by other developers than the main maintainer. Those people have commit access to the code base but only commit (in most cases) into the subsystem they maintain. -Upstream merges are restricted to the alpha and master branches. Subsystem +Upstream merges are restricted to the alpha and main branches. Subsystem maintainers should never push a patch into a stable branch which is the responsibility of the [stable branch maintainer](#stable-branches). @@ -69,7 +69,7 @@ maintain the following subsystems: These are the tasks of a subsystem maintainer: 1. Regularly go over `merge_ready` tickets relevant to the related subsystem - and for the current alpha or development (master branch) Milestone. + and for the current alpha or development (main branch) Milestone. 2. A subsystem maintainer is expected to contribute to any design changes (including proposals) or large patch set about the subsystem. @@ -102,7 +102,7 @@ These are few important items to follow when merging code upstream: 4. Tor uses the "merge forward" method, that is, if a patch applies to the alpha branch, it has to be merged there first and then merged forward - into master. + into main. 5. Maintainer should always consult with the network team about any doubts, mis-understandings or unknowns of a patch. Final word will always go to the diff --git a/doc/HACKING/README.1st.md b/doc/HACKING/README.1st.md index 4bc3298c67..1c0decf9ce 100644 --- a/doc/HACKING/README.1st.md +++ b/doc/HACKING/README.1st.md @@ -5,12 +5,11 @@ This directory has helpful information about what you need to know to hack on Tor! -First, read `GettingStarted.md` and `GettingStartedRust.md` -to learn how to get a start in Tor development. +First, read `GettingStarted.md` to learn how to get a start in Tor +development. -If you've decided to write a patch, `CodingStandards.md` and -`CodingStandardsRust.md` will give you a bunch of information -about how we structure our code. +If you've decided to write a patch, `CodingStandards.md` will give you a bunch +of information about how we structure our code. It's important to get the code right! Reading `WritingTests.md` will tell you how to write and run tests in the Tor codebase. @@ -36,6 +35,13 @@ For the latest version of the code, get a copy of git, and $ git clone https://git.torproject.org/git/tor ``` +For a copy of Tor's original design paper, see +[here](https://spec.torproject.org/tor-design). Note that Tor has changed in +many ways since 2004. + +For a large collection of security papers, many of which are related to Tor, +see [Anonbib's Selected Papers in Anonymity](https://www.freehaven.net/anonbib/). + ## Stay in touch We talk about Tor on the `tor-talk` mailing list. Design proposals and diff --git a/doc/HACKING/ReleaseSeriesLifecycle.md b/doc/HACKING/ReleaseSeriesLifecycle.md index 8536fbbd08..e47ac90fa5 100644 --- a/doc/HACKING/ReleaseSeriesLifecycle.md +++ b/doc/HACKING/ReleaseSeriesLifecycle.md @@ -87,17 +87,17 @@ they do not apply to security-related patch release versions. (Ideally, do this immediately after a release.) -1. Start a new maint-x.y.z branch based on master, and a new - release-x.y.z branch based on master. They should have the same +1. Start a new maint-x.y.z branch based on main, and a new + release-x.y.z branch based on main. They should have the same starting point. - Push both of these branches to the master git repository. + Push both of these branches to the canonical git repository. -2. In master, change the version to "0.x.y.0-alpha-dev". Run the +2. In the main branch, change the version to "0.x.y.0-alpha-dev". Run the update_versions.py script, and commit this version bump. 3. Tag the version bump with "tor-0.x.y.0-alpha-dev". Push the tag - and master. + and main branch. 4. Open tickets for connecting the new branches to various other places. See section 2 above for a list of affected locations. @@ -107,7 +107,7 @@ they do not apply to security-related patch release versions. target in the maint-x.y.z branch only. * Delete the file scripts/maint/practracker/.enable_practracker_in_hooks in the maint-x.y.z branch only. - * Merge to release-x.y.z, but do not forward-port to master. + * Merge to release-x.y.z, but do not forward-port to the main branch. 6. Finally, make sure this document is up to date with our latest process. diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 739ea38795..86feef754e 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -1,234 +1,147 @@ -# Putting out a new release +# How to Release Tor Here are the steps that the maintainer should take when putting out a -new Tor release: +new Tor release. It is split in 3 stages and coupled with our Tor CI Release +pipeline. -## 0. Preliminaries +Before we begin, first rule is to make sure: -1. Get at least two of weasel/arma/Sebastian to put the new - version number in their approved versions list. Give them a few - days to do this if you can. + - Our CIs (*nix and Windows) pass for each version to release + - Coverity has no new alerts -2. If this is going to be an important security release, give these packagers - some advance warning: +## 0. Security Release - - {weasel,sysrqb,mikeperry} at torproject dot org - - {blueness} at gentoo dot org - - {paul} at invizbox dot io - - {vincent} at invizbox dot com - - {lfleischer} at archlinux dot org - - {Nathan} at freitas dot net - - {mike} at tig dot as - - {tails-rm} at boum dot org - - {simon} at sdeziel.info - - {yuri} at freebsd.org - - {mh+tor} at scrit.ch - - {security} at brave.com - -3. Given the release date for Tor, ask the TB team about the likely release - date of a TB that contains it. See note below in "commit, upload, - announce". - -## I. Make sure it works - -1. Make sure that CI passes: have a look at Travis - (https://travis-ci.org/torproject/tor/branches), Appveyor - (https://ci.appveyor.com/project/torproject/tor/history), and - Jenkins (https://jenkins.torproject.org/view/tor/). - Make sure you're looking at the right branches. - - If there are any unexplained failures, try to fix them or figure them - out. - -2. Verify that there are no big outstanding issues. You might find such - issues -- - - * On Trac - - * On coverity scan - - * On OSS-Fuzz - -## II. Write a changelog +To start with, if you are doing a security release, this must be done few days +prior to the release: -1a. (Alpha release variant) + 1. If this is going to be an important security release, give the packagers + advance warning, via `tor-packagers@lists.torproject.org`. - Gather the `changes/*` files into a changelog entry, rewriting many - of them and reordering to focus on what users and funders would find - interesting and understandable. - To do this, run `./scripts/maint/sortChanges.py changes/* > changelog.in` - to combine headings and sort the entries. Copy the changelog.in file into - the ChangeLog. Run `format_changelog.py --inplace` (see below) to clean up - the line breaks. +## 1. Preliminaries - Remove the `changes/*` files that you just merged into the ChangeLog. +The following must be done **2 days** at the very least prior to the release: - After that, it's time to hand-edit and fix the issues that - lintChanges can't find: + 1. Add the version(s) in the dirauth-conf git repository as the + RecommendedVersion and RequiredVersion so they can be approved by the + authorities and be in the consensus before the release. - 1. Within each section, sort by "version it's a bugfix on", else by - numerical ticket order. + 2. Send a pre-release announcement to `tor-project@lists.torproject.org` in + order to inform every teams in Tor of the upcoming release. This is so + we can avoid creating release surprises and sync with other teams. - 2. Clean them up: + 3. Ask the network-team to review the `changes/` files in all versions we + are about to release. This step is encouraged but not mandatory. - Make stuff very terse - Describe the user-visible problem right away +## 2. Tarballs - Mention relevant config options by name. If they're rare or unusual, - remind people what they're for +To build the tarballs to release, we need to launch the CI release pipeline: - Avoid starting lines with open-paren + https://gitlab.torproject.org/tpo/core/tor-ci-release - Present and imperative tense: not past. +The `versions.yml` needs to be modified with the Tor versions you want to +release. Once done, git commit and push to trigger the release pipeline. - "Relays", not "servers" or "nodes" or "Tor relays". +The first two stages (Preliminary and Patches) will be run automatically. The +Build stage needs to be triggered manually once all generated patches have +been merged upstream. - "Onion services", not "hidden services". + 1. Download the generated patches from the `Patches` stage. - "Stop FOOing", not "Fix a bug where we would FOO". + 2. For the ChangeLog and ReleaseNotes, you need to write a blurb at the top + explaining a bit the release. - Try not to let any given section be longer than about a page. Break up - long sections into subsections by some sort of common subtopic. This - guideline is especially important when organizing Release Notes for - new stable releases. + 3. Review, modify if needed, and merged them upstream. - If a given changes stanza showed up in a different release (e.g. - maint-0.2.1), be sure to make the stanzas identical (so people can - distinguish if these are the same change). + 4. Manually trigger the `maintained` job in the `Build` stage so the CI can + build the tarballs without errors. - 3. Clean everything one last time. +Once this is done, each selected developers need to build the tarballs in a +reproducible way using: - 4. Run `./scripts/maint/format_changelog.py --inplace` to make it prettier + https://gitlab.torproject.org/tpo/core/tor-ci-reproducible -1b. (old-stable release variant) +Steps are: - For stable releases that backport things from later, we try to compose - their releases, we try to make sure that we keep the changelog entries - identical to their original versions, with a "backport from 0.x.y.z" - note added to each section. So in this case, once you have the items - from the changes files copied together, don't use them to build a new - changelog: instead, look up the corrected versions that were merged - into ChangeLog in the master branch, and use those. + 1. Run `./build.sh` which will download everything you need, including the + latest tarballs from the release CI, and auto-commit the signatures if + the checksum match. You will need to confim the commits. - Add "backport from X.Y.Z" in the section header for these entries. + 2. If all is good, `git push origin main` your signatures. -2. Compose a short release blurb to highlight the user-facing - changes. Insert said release blurb into the ChangeLog stanza. If it's - a stable release, add it to the ReleaseNotes file too. If we're adding - to a release-* branch, manually commit the changelogs to the later - git branches too. +Once all signatures from all selected developers have been committed: -3. If there are changes that require or suggest operator intervention - before or during the update, mail operators (either dirauth or relays - list) with a headline that indicates that an action is required or - appreciated. + 1. Manually trigger the `signature` job in the `Post-process` stage of the + CI release pipeline. -4. If you're doing the first stable release in a series, you need to - create a ReleaseNotes for the series as a whole. To get started - there, copy all of the Changelog entries from the series into a new - file, and run `./scripts/maint/sortChanges.py` on it. That will - group them by category. Then kill every bugfix entry for fixing - bugs that were introduced within that release series; those aren't - relevant changes since the last series. At that point, it's time - to start sorting and condensing entries. (Generally, we don't edit the - text of existing entries, though.) + 2. If it passes, the tarball(s) and signature(s) will be available as + artifacts and should be used for the release. -## III. Making the source release. + 3. Put them on `dist.torproject.org`: -1. In `maint-0.?.x`, bump the version number in `configure.ac` and run - `make update-versions` to update version numbers in other - places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. + Upload the tarball and its sig to the dist website, i.e. + `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run + "static-update-component dist.torproject.org" on dist-master. - When you merge the maint branch forward to the next maint branch, or into - master, merge it with "-s ours" to avoid conflict with the version - bump. + In the `project/web/tpo.git` repository, update `databags/versions.ini` + to note the new version. Push these changes to `master`. -2. Make distcheck, put the tarball up in somewhere (how about your - homedir on people.torproject.org?) , and tell `#tor-dev` - about it. + (NOTE: Due to #17805, there can only be one stable version listed at once. + Nonetheless, do not call your version "alpha" if it is stable, or people + will get confused.) - If you want, wait until at least one person has built it - successfully. (We used to say "wait for others to test it", but our - CI has successfully caught these kinds of errors for the last several - years.) + (NOTE: It will take a while for the website update scripts to update the + website.) -3. Make sure that the new version is recommended in the latest consensus. - (Otherwise, users will get confused when it complains to them - about its status.) - If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see - item 0.1 at the start of this document. +## 3. Post Process -## IV. Commit, upload, announce +Once the tarballs have been uploaded and are ready to be announced, we need to +do the following: -1. Sign the tarball, then sign and push the git tag: + 1. Tag versions (main and maint) using `git tag -s tor-0.x.y.z-<status>` + and then push the tags: `git push origin --tags` -```console -$ gpg -ba <the_tarball> -$ git tag -s tor-0.4.x.y-<status> -$ git push origin tag tor-0.4.x.y-<status> -``` + 2. Merge upstream the artifacts from the `patches` job in the + `Post-process` stage of the CI release pipeline. - (You must do this before you update the website: the website scripts - rely on finding the version by tag.) + 3. Write and post the release announcement for the `forum.torproject.net` + in the `News -> Tor Release Announcement` category. - (If your default PGP key is not the one you want to sign with, then say - "-u <keyid>" instead of "-s".) + If possible, mention in which Tor Browser version (with dates) the + release will be in. This usually only applies to the latest stable. -2. scp the tarball and its sig to the dist website, i.e. - `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run - "static-update-component dist.torproject.org" on dist-master. + 4. Inform `tor-talk@lists.torproject.org` with the releasing pointing to + the Forum. Append the ChangeLog there. We do this until we can automate + such post from the forum directly. - In the project/web/tpo.git repository, update `databags/versions.ini` - to note the new version. Push these changes to master. +### New Stable - (NOTE: Due to #17805, there can only be one stable version listed at - once. Nonetheless, do not call your version "alpha" if it is stable, - or people will get confused.) + 1. Create the `maint-x.y.z` and `release-x.y.z` branches and update the + `./scripts/git/git-list-tor-branches.sh` with the new version. - (NOTE: It will take a while for the website update scripts to update - the website.) + 2. Add the new version in `./scripts/ci/ci-driver.sh`. -3. Email the tor-packagers@lists.torproject.org mailing list to tell them - about the new release. + 3. Forward port the ChangeLog and ReleaseNotes into main branch. Remove any + change logs of stable releases in ReleaseNotes. - Also, email tor-packagers@lists.torproject.org. - Mention where to download the tarball (https://dist.torproject.org). +## Appendix: An alternative means to notify packagers - Include a link to the changelog. +If for some reason you need to contact a bunch of packagers without +using the publicly archived tor-packagers list, you can try these +people: -4. Wait for the download page to be updated. (If you don't do this before you - announce, people will be confused.) - -5. Mail the release blurb and ChangeLog to tor-talk (development release) or - tor-announce (stable). - - Post the changelog on the blog as well. You can generate a - blog-formatted version of the changelog with - `./scripts/maint/format_changelog.py -B` - - When you post, include an estimate of when the next TorBrowser - releases will come out that include this Tor release. This will - usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it - can vary. - - For templates to use when announcing, see: - https://gitlab.torproject.org/tpo/core/team/-/wikis/NetworkTeam/AnnouncementTemplates - -## V. Aftermath and cleanup - -1. If it's a stable release, bump the version number in the - `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` - merge to avoid taking that change into master. - -2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that - builds the release every week. (It's ok to skip the weekly build if the - branch was updated in the last 24 hours.) - -3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the - master branch. - -4. Keep an eye on the blog post, to moderate comments and answer questions. + - {weasel,sysrqb,mikeperry} at torproject dot org + - {blueness} at gentoo dot org + - {paul} at invizbox dot io + - {vincent} at invizbox dot com + - {lfleischer} at archlinux dot org + - {Nathan} at freitas dot net + - {mike} at tig dot as + - {tails-rm} at boum dot org + - {simon} at sdeziel.info + - {yuri} at freebsd.org + - {mh+tor} at scrit.ch + - {security} at brave.com diff --git a/doc/HACKING/ReleasingTor.md.old b/doc/HACKING/ReleasingTor.md.old new file mode 100644 index 0000000000..490c100fcb --- /dev/null +++ b/doc/HACKING/ReleasingTor.md.old @@ -0,0 +1,255 @@ +# CHECKLIST + +Here's a summary checklist, with the things that Nick messes up most often. + +Did you: + + * [ ] Copy the ChangeLog to the ReleaseNotes? + * [ ] Check that the new versions got approved? + * [ ] Check the release date in the ChangeLog? + * [ ] Update the GeoIP file? + +# Putting out a new release + +Here are the steps that the maintainer should take when putting out a +new Tor release: + +## 0. Preliminaries + +1. Get at least three of weasel/arma/Sebastian/Sina to put the new + version number in their approved versions list. Give them a few + days to do this if you can. + +2. If this is going to be an important security release, give the packagers + advance warning, via `tor-packagers@lists.torproject.org`. + + +3. Given the release date for Tor, ask the TB team about the likely release + date of a TB that contains it. See note below in "commit, upload, + announce". + +## I. Make sure it works + +1. Make sure that CI passes: have a look at the branches on gitlab. + + _Optionally_, have a look at Travis + (https://travis-ci.org/torproject/tor/branches), Appveyor + (https://ci.appveyor.com/project/torproject/tor/history), and + Jenkins (https://jenkins.torproject.org/view/tor/). + Make sure you're looking at the right branches. + + If there are any unexplained failures, try to fix them or figure them + out. + +2. Verify that there are no big outstanding issues. You might find such + issues -- + + * On Gitlab + + * On coverity scan + + * On OSS-Fuzz + +## II. Write a changelog + + +1a. (Alpha release variant) + + Gather the `changes/*` files into a changelog entry, rewriting many + of them and reordering to focus on what users and funders would find + interesting and understandable. + + To do this, run `./scripts/maint/sortChanges.py changes/* > changelog.in` + to combine headings and sort the entries. Copy the changelog.in file into + the ChangeLog. Run `format_changelog.py --inplace` (see below) to clean up + the line breaks. + + Remove the `changes/*` files that you just merged into the ChangeLog. + + After that, it's time to hand-edit and fix the issues that + lintChanges can't find: + + 1. Within each section, sort by "version it's a bugfix on", else by + numerical ticket order. + + 2. Clean them up: + + Make stuff very terse + + Describe the user-visible problem right away + + Mention relevant config options by name. If they're rare or unusual, + remind people what they're for + + Avoid starting lines with open-paren + + Present and imperative tense: not past. + + "Relays", not "servers" or "nodes" or "Tor relays". + + "Onion services", not "hidden services". + + "Stop FOOing", not "Fix a bug where we would FOO". + + Try not to let any given section be longer than about a page. Break up + long sections into subsections by some sort of common subtopic. This + guideline is especially important when organizing Release Notes for + new stable releases. + + If a given changes stanza showed up in a different release (e.g. + maint-0.2.1), be sure to make the stanzas identical (so people can + distinguish if these are the same change). + + 3. Clean everything one last time. + + 4. Run `./scripts/maint/format_changelog.py --inplace` to make it prettier + +1b. (old-stable release variant) + + For stable releases that backport things from later, we try to compose + their releases, we try to make sure that we keep the changelog entries + identical to their original versions, with a "backport from 0.x.y.z" + note added to each section. So in this case, once you have the items + from the changes files copied together, don't use them to build a new + changelog: instead, look up the corrected versions that were merged + into ChangeLog in the main branch, and use those. + + Add "backport from X.Y.Z" in the section header for these entries. + +2. Compose a short release blurb to highlight the user-facing + changes. Insert said release blurb into the ChangeLog stanza. If it's + a stable release, add it to the ReleaseNotes file too. If we're adding + to a release-* branch, manually commit the changelogs to the later + git branches too. + +3. If there are changes that require or suggest operator intervention + before or during the update, mail operators (either dirauth or relays + list) with a headline that indicates that an action is required or + appreciated. + +4. If you're doing the first stable release in a series, you need to + create a ReleaseNotes for the series as a whole. To get started + there, copy all of the Changelog entries from the series into a new + file, and run `./scripts/maint/sortChanges.py` on it. That will + group them by category. Then kill every bugfix entry for fixing + bugs that were introduced within that release series; those aren't + relevant changes since the last series. At that point, it's time + to start sorting and condensing entries. (Generally, we don't edit the + text of existing entries, though.) + +## III. Making the source release. + +1. In `maint-0.?.x`, bump the version number in `configure.ac` and run + `./scripts/main/update_versions.py` to update version numbers in other + places, and commit. Then merge `maint-0.?.x` into `release-0.?.x`. + + When you merge the maint branch forward to the next maint branch, or into + main, merge it with `-s ours` to avoid conflict with the version + bump. + +2. In `release-0.?.x`, run `make distcheck`, put the tarball up in somewhere + (how about your homedir on people.torproject.org?) , and tell `#tor-dev` + about it. + + If you want, wait until at least one person has built it + successfully. (We used to say "wait for others to test it", but our + CI has successfully caught these kinds of errors for the last several + years.) + +3. Make sure that the new version is recommended in the latest consensus. + (Otherwise, users will get confused when it complains to them + about its status.) + + If it is not, you'll need to poke Roger, Weasel, Sebastian, and Sina + again: see the note at the start of the document. + +## IV. Commit, upload, announce + +1. Sign the tarball, then sign and push the git tag: + +```console +$ gpg -ba <the_tarball> +$ git tag -s tor-0.4.x.y-<status> +$ git push origin tag tor-0.4.x.y-<status> +``` + + (You must do this before you update the website: the website scripts + rely on finding the version by tag.) + + (If your default PGP key is not the one you want to sign with, then say + "-u <keyid>" instead of "-s".) + +2. scp the tarball and its sig to the dist website, i.e. + `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run + "static-update-component dist.torproject.org" on dist-master. + + In the `project/web/tpo.git` repository, update `databags/versions.ini` + to note the new version. Push these changes to `master`. + + (NOTE: Due to #17805, there can only be one stable version listed at + once. Nonetheless, do not call your version "alpha" if it is stable, + or people will get confused.) + + (NOTE: It will take a while for the website update scripts to update + the website.) + +3. Email the tor-packagers@lists.torproject.org mailing list to tell them + about the new release. + + Also, email tor-packagers@lists.torproject.org. + + Mention where to download the tarball (`https://dist.torproject.org/`). + + Include a link to the changelog. + +4. Wait for the download page to be updated. (If you don't do this before you + announce, people will be confused.) + +5. Mail the release blurb and ChangeLog to tor-talk (development release) or + tor-announce (stable). + + Post the changelog on the blog as well. You can generate a + blog-formatted version of the changelog with + `./scripts/maint/format_changelog.py -B` + + When you post, include an estimate of when the next TorBrowser + releases will come out that include this Tor release. This will + usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it + can vary. + + For templates to use when announcing, see: + https://gitlab.torproject.org/tpo/core/team/-/wikis/NetworkTeam/AnnouncementTemplates + +## V. Aftermath and cleanup + +1. If it's a stable release, bump the version number in the + `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours` + merge to avoid taking that change into main. + +2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that + builds the release every week. (It's ok to skip the weekly build if the + branch was updated in the last 24 hours.) + +3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the + main branch. + +4. Keep an eye on the blog post, to moderate comments and answer questions. + +## Appendix: An alternative means to notify packagers + +If for some reason you need to contact a bunch of packagers without +using the publicly archived tor-packagers list, you can try these +people: + + - {weasel,sysrqb,mikeperry} at torproject dot org + - {blueness} at gentoo dot org + - {paul} at invizbox dot io + - {vincent} at invizbox dot com + - {lfleischer} at archlinux dot org + - {Nathan} at freitas dot net + - {mike} at tig dot as + - {tails-rm} at boum dot org + - {simon} at sdeziel.info + - {yuri} at freebsd.org + - {mh+tor} at scrit.ch + - {security} at brave.com diff --git a/doc/asciidoc-helper.sh b/doc/asciidoc-helper.sh index 3706ca2e14..98e216e68a 100755 --- a/doc/asciidoc-helper.sh +++ b/doc/asciidoc-helper.sh @@ -9,7 +9,7 @@ set -e if [ $# != 3 ]; then - exit 1; + exit 1 fi SOURCE_DATE_EPOCH="$(git show --no-patch --format='%ct')" @@ -22,50 +22,49 @@ if [ "$1" = "html" ]; then base=${output%%.html.in} if [ "$2" != none ]; then - TZ=UTC "$2" -f "$(dirname "$0")/nofooter.conf" -d manpage -o "$output" "$input"; + TZ=UTC "$2" -f "$(dirname "$0")/nofooter.conf" -d manpage -o "$output" "$input"; else - echo "=================================="; - echo; - echo "You need asciidoc installed to be able to build the manpage."; - echo "To build without manpages, use the --disable-asciidoc argument"; - echo "when calling configure."; - echo; - echo "=================================="; - exit 1; + echo "==================================" + echo + echo "You need asciidoc installed to be able to build the manpage." + echo "To build without manpages, use the --disable-asciidoc argument" + echo "when calling configure." + echo + echo "==================================" + exit 1 fi elif [ "$1" = "man" ]; then input=${output%%.1.in}.1.txt base=${output%%.1.in} if test "$2" = none; then - echo "=================================="; - echo; - echo "You need asciidoc installed to be able to build the manpage."; - echo "To build without manpages, use the --disable-asciidoc argument"; - echo "when calling configure."; - echo; - echo "=================================="; - exit 1; + echo "==================================" + echo + echo "You need asciidoc installed to be able to build the manpage." + echo "To build without manpages, use the --disable-asciidoc argument" + echo "when calling configure." + echo + echo "==================================" + exit 1 fi if "$2" -f manpage "$input"; then - mv "$base.1" "$output"; + mv "$base.1" "$output" else - cat<<EOF + cat<<EOF ================================== You need a working asciidoc installed to be able to build the manpage. a2x is installed, but for some reason it isn't working. Sometimes this happens because required docbook support files are missing. Please install docbook-xsl, docbook-xml, and xmlto (Debian) or -similar. If you use homebrew on Mac OS X, install the docbook formula +similar. If you use homebrew on Mac OS X, install the docbook formula and add "export XML_CATALOG_FILES=/usr/local/etc/xml/catalog" to your -.bashrc +.bashrc file. Alternatively, to build without manpages, use the --disable-asciidoc argument when calling configure. ================================== EOF - exit 1; + exit 1 fi fi - diff --git a/doc/include.am b/doc/include.am index 7a8a64ed16..d10f380e7f 100644 --- a/doc/include.am +++ b/doc/include.am @@ -51,10 +51,8 @@ EXTRA_DIST+= doc/asciidoc-helper.sh \ doc/TUNING \ doc/HACKING/README.1st.md \ doc/HACKING/CodingStandards.md \ - doc/HACKING/CodingStandardsRust.md \ doc/HACKING/Fuzzing.md \ doc/HACKING/GettingStarted.md \ - doc/HACKING/GettingStartedRust.md \ doc/HACKING/HelpfulTools.md \ doc/HACKING/HowToReview.md \ doc/HACKING/Module.md \ diff --git a/doc/man/tor.1.txt b/doc/man/tor.1.txt index 0af9a9c03d..1814801b71 100644 --- a/doc/man/tor.1.txt +++ b/doc/man/tor.1.txt @@ -71,7 +71,7 @@ The following options in this section are only recognized on the Specify a new configuration file to contain further Tor configuration options, or pass *-* to make Tor read its configuration from standard input. (Default: **`@CONFDIR@/torrc`**, or **`$HOME/.torrc`** if - that file is not found) + that file is not found.) [[opt-allow-missing-torrc]] **`--allow-missing-torrc`**:: Allow the configuration file specified by **`-f`** to be missing, @@ -101,7 +101,7 @@ The following options in this section are only recognized on the [[opt-dump-config]] **`--dump-config`** **`short`**|**`full`**:: Write a list of Tor's configured options to standard output. When the `short` flag is selected, only write the options that - are different from their default values + are different from their default values. When `full` is selected, write every option. [[opt-serviceinstall]] **`--service install`** [**`--options`** __command-line options__]:: @@ -988,20 +988,20 @@ forward slash (/) in the configuration file and on the command line. running. (Default: none) [[TCPProxy]] **TCPProxy** __protocol__ __host__:__port__:: - Tor will use the given protocol to make all its OR (SSL) connections through - a TCP proxy on host:port, rather than connecting directly to servers. You may - want to set **FascistFirewall** to restrict the set of ports you might try to - connect to, if your proxy only allows connecting to certain ports. There is no - equivalent option for directory connections, because all Tor client versions - that support this option download directory documents via OR connections. + + Tor will use the given protocol to make all its OR (SSL) connections through + a TCP proxy on host:port, rather than connecting directly to servers. You may + want to set **FascistFirewall** to restrict the set of ports you might try to + connect to, if your proxy only allows connecting to certain ports. There is no + equivalent option for directory connections, because all Tor client versions + that support this option download directory documents via OR connections. + + - The only protocol supported right now 'haproxy'. This option is only for - clients. (Default: none) + + The only protocol supported right now 'haproxy'. This option is only for + clients. (Default: none) + + - The HAProxy version 1 proxy protocol is described in detail at - https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + + The HAProxy version 1 proxy protocol is described in detail at + https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt + + - Both source IP address and source port will be set to zero. + Both source IP address and source port will be set to zero. [[TruncateLogFile]] **TruncateLogFile** **0**|**1**:: If 1, Tor will overwrite logs at startup and in response to a HUP signal, @@ -1749,6 +1749,13 @@ The following options are useful only for clients (that is, if the guard-n-primary-guards consensus parameter, and default to 3 if the consensus parameter isn't set. (Default: 0) +[[VanguardsLiteEnabled]] **VanguardsLiteEnabled** **0**|**1**|**auto**:: + This option specifies whether clients should use the vanguards-lite + subsystem to protect against guard discovery attacks. If it's set to + 'auto', clients will do what the vanguards-lite-enabled consensus parameter + tells them to do, and will default to enable the subsystem if the consensus + parameter isn't set. (Default: auto) + [[UseMicrodescriptors]] **UseMicrodescriptors** **0**|**1**|**auto**:: Microdescriptors are a smaller version of the information that Tor needs in order to build its circuits. Using microdescriptors makes Tor clients @@ -2847,11 +2854,15 @@ details.) == DENIAL OF SERVICE MITIGATION OPTIONS -Tor has three built-in mitigation options that can be individually -enabled/disabled and fine-tuned, but by default Tor directory authorities will -define reasonable values for relays and no explicit configuration is required -to make use of these protections. The mitigations take place at relays, -and are as follows: +Tor has a series of built-in denial of service mitigation options that can be +individually enabled/disabled and fine-tuned, but by default Tor directory +authorities will define reasonable values for the network and no explicit +configuration is required to make use of these protections. + +The following is a series of configuration options for relays and then options +for onion services and how they work. + +The mitigations take place at relays, and are as follows: 1. If a single client address makes too many concurrent connections (this is configurable via DoSConnectionMaxConcurrentCount), hang up on further @@ -3002,6 +3013,68 @@ Denial of Service mitigation subsystem described above. (Default: auto) +As for onion services, only one possible mitigation exists. It was intended to +protect the network first and thus do not help the service availability or +reachability. + +The mitigation we put in place is a rate limit of the amount of introduction +that happens at the introduction point for a service. In other words, it rates +limit the number of clients that are attempting to reach the service at the +introduction point instead of at the service itself. + +The following options are per onion service: + +[[HiddenServiceEnableIntroDoSDefense]] **HiddenServiceEnableIntroDoSDefense** **0**|**1**:: + Enable DoS defense at the intropoint level. When this is enabled, the + rate and burst parameter (see below) will be sent to the intro point which + will then use them to apply rate limiting for introduction request to this + service. + + + The introduction point honors the consensus parameters except if this is + specifically set by the service operator using this option. The service + never looks at the consensus parameters in order to enable or disable this + defense. (Default: 0) + +//Out of order because it logically belongs after HiddenServiceEnableIntroDoSDefense. +[[HiddenServiceEnableIntroDoSBurstPerSec]] **HiddenServiceEnableIntroDoSBurstPerSec** __NUM__:: + The allowed client introduction burst per second at the introduction + point. If this option is 0, it is considered infinite and thus if + **HiddenServiceEnableIntroDoSDefense** is set, it then effectively + disables the defenses. (Default: 200) + +[[HiddenServiceEnableIntroDoSRatePerSec]] **HiddenServiceEnableIntroDoSRatePerSec** __NUM__:: + The allowed client introduction rate per second at the introduction + point. If this option is 0, it is considered infinite and thus if + **HiddenServiceEnableIntroDoSDefense** is set, it then effectively + disables the defenses. (Default: 25) + +The rate is the maximum number of clients a service will ask its introduction +points to allow every seconds. And the burst is a parameter that allows that +many within one second. + +For example, the default values of 25 and 200 respectively means that for every +introduction points a service has (default 3 but can be configured with +**HiddenServiceNumIntroductionPoints**), 25 clients per seconds will be allowed +to reach the service and 200 at most within 1 second as a burst. This means +that if 200 clients are seen within 1 second, it will take 8 seconds (200/25) +for another client to be able to be allowed to introduce due to the rate of 25 +per second. + +This might be too much for your use case or not, fine tuning these values is +hard and are likely different for each service operator. + +Why is this not helping reachability of the service? Because the defenses are +at the introduction point, an attacker can easily flood all introduction point +rendering the service unavailable due to no client being able to pass through. +But, the service itself is not overwhelmed with connetions allowing it to +function properly for the few clients that were able to go through or other any +services running on the same tor instance. + +The bottom line is that this protects the network by preventing an onion +service to flood the network with new rendezvous circuits that is reducing load +on the network. + + == DIRECTORY AUTHORITY SERVER OPTIONS The following options enable operation as a directory authority, and @@ -3039,6 +3112,11 @@ on the public Tor network. is the same as for exit policies, except that you don't need to say "accept" or "reject", and ports are not needed.) +[[AuthDirMiddleOnly]] **AuthMiddleOnly** __AddressPattern...__:: + Authoritative directories only. A set of address patterns for servers that + will be listed as middle-only in any network status document this authority + publishes, if **AuthDirListMiddleOnly** is set. + + [[AuthDirFastGuarantee]] **AuthDirFastGuarantee** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**:: Authoritative directories only. If non-zero, always vote the Fast flag for any relay advertising this amount of capacity or @@ -3086,6 +3164,13 @@ on the public Tor network. 1 unless you plan to list non-functioning exits as bad; otherwise, you are effectively voting in favor of every declared exit as an exit.) +[[AuthDirListMiddleOnly]] **AuthDirListMiddleOnly** **0**|**1**:: + Authoritative directories only. If set to 1, this directory has some + opinion about which nodes should only be used in the middle position. + (Do not set this to 1 unless you plan to list questionable relays + as "middle only"; otherwise, you are effectively voting _against_ + middle-only status for every relay.) + [[AuthDirMaxServersPerAddr]] **AuthDirMaxServersPerAddr** __NUM__:: Authoritative directories only. The maximum number of servers that we will list as acceptable on a single IP address. Set this to "0" for "no limit". @@ -3104,18 +3189,20 @@ on the public Tor network. authority publishes, or accepted as an OR address in any descriptor submitted for publication by this authority. +[[AuthDirRejectRequestsUnderLoad]] **AuthDirRejectRequestsUnderLoad** **0**|**1**:: + If set, the directory authority will start rejecting directory requests + from non relay connections by sending a 503 error code if it is under + bandwidth pressure (reaching the configured limit if any). Relays will + always tried to be answered even if this is on. (Default: 1) + //Out of order because it logically belongs with the other CCs options. [[AuthDirBadExitCCs]] **AuthDirBadExitCCs** __CC__,... + //Out of order because it logically belongs with the other CCs options. [[AuthDirInvalidCCs]] **AuthDirInvalidCCs** __CC__,... + - -[[AuthDirRejectRequestsUnderLoad]] **AuthDirRejectRequestsUnderLoad** **0**|**1**:: - If set, the directory authority will start rejecting directory requests - from non relay connections by sending a 503 error code if it is under - bandwidth pressure (reaching the configured limit if any). Relays will - always tried to be answered even if this is on. (Default: 1) +//Out of order because it logically belongs with the other CCs options. +[[AuthDirMiddleOnlytCCs]] **AuthDirMiddleOnlyCCs** __CC__,... + [[AuthDirRejectCCs]] **AuthDirRejectCCs** __CC__,...:: Authoritative directories only. These options contain a comma-separated @@ -3280,30 +3367,6 @@ The next section describes the per service options that can only be set only owner is able to read the hidden service directory. (Default: 0) Has no effect on Windows. -[[HiddenServiceEnableIntroDoSDefense]] **HiddenServiceEnableIntroDoSDefense** **0**|**1**:: - Enable DoS defense at the intropoint level. When this is enabled, the - rate and burst parameter (see below) will be sent to the intro point which - will then use them to apply rate limiting for introduction request to this - service. - + - The introduction point honors the consensus parameters except if this is - specifically set by the service operator using this option. The service - never looks at the consensus parameters in order to enable or disable this - defense. (Default: 0) - -//Out of order because it logically belongs after HiddenServiceEnableIntroDoSDefense. -[[HiddenServiceEnableIntroDoSBurstPerSec]] **HiddenServiceEnableIntroDoSBurstPerSec** __NUM__:: - The allowed client introduction burst per second at the introduction - point. If this option is 0, it is considered infinite and thus if - **HiddenServiceEnableIntroDoSDefense** is set, it then effectively - disables the defenses. (Default: 200) - -[[HiddenServiceEnableIntroDoSRatePerSec]] **HiddenServiceEnableIntroDoSRatePerSec** __NUM__:: - The allowed client introduction rate per second at the introduction - point. If this option is 0, it is considered infinite and thus if - **HiddenServiceEnableIntroDoSDefense** is set, it then effectively - disables the defenses. (Default: 25) - [[HiddenServiceExportCircuitID]] **HiddenServiceExportCircuitID** __protocol__:: The onion service will use the given protocol to expose the global circuit identifier of each inbound client circuit. The only @@ -3514,14 +3577,15 @@ The following options are used for running a testing Tor network. [[TestingAuthKeySlop]] **TestingAuthKeySlop** __N__ **seconds**|**minutes**|**hours** + [[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__:: - Initial delay in seconds for when clients should download each bridge descriptor when they - have just started, or when they can not contact any of their bridges. + Initial delay in seconds for how long clients should wait before + downloading a bridge descriptor for a new bridge. Changing this requires that **TestingTorNetwork** is set. (Default: 0) [[TestingBridgeDownloadInitialDelay]] **TestingBridgeDownloadInitialDelay** __N__:: - Initial delay in seconds for when clients should download each bridge descriptor when they - know that one or more of their configured bridges are running. Changing - this requires that **TestingTorNetwork** is set. (Default: 10800) + How long to wait (in seconds) once clients have successfully + downloaded a bridge descriptor, before trying another download for + that same bridge. Changing this requires that **TestingTorNetwork** + is set. (Default: 10800) [[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__:: Initial delay in seconds for when clients should download consensuses. Changing this @@ -3840,7 +3904,12 @@ __KeyDirectory__/**`secret_onion_key_ntor`** and **`secret_onion_key_ntor.old`** by clients that didn't have the new one. __DataDirectory__/**`fingerprint`**:: - Only used by servers. Contains the fingerprint of the server's identity key. + Only used by servers. Contains the fingerprint of the server's RSA + identity key. + +__DataDirectory__/**`fingerprint-ed25519`**:: + Only used by servers. Contains the fingerprint of the server's ed25519 + identity key. __DataDirectory__/**`hashed-fingerprint`**:: Only used by bridges. Contains the hashed fingerprint of the bridge's @@ -3856,7 +3925,8 @@ __DataDirectory__/**`approved-routers`**:: descriptors are accepted, but marked in the vote as not valid. If it is **!badexit**, then the authority will vote for it to receive a BadExit flag, indicating that it shouldn't be used for traffic leaving - the Tor network. + the Tor network. If it is **!middleonly**, then the authority will + vote for it to only be used in the middle of circuits. (Neither rejected nor invalid relays are included in the consensus.) __DataDirectory__/**`v3-status-votes`**:: diff --git a/scripts/README b/scripts/README index 9cd6e74ac7..4cb49370f5 100644 --- a/scripts/README +++ b/scripts/README @@ -12,7 +12,7 @@ never used. maint/checkOptionDocs.pl -- Make sure that Tor options are documented in the manpage, and that the manpage only documents real Tor options. -maint/checkSpaces.pl -- Style checker for the Tor source code. Mainly checks +maint/checkSpace.pl -- Style checker for the Tor source code. Mainly checks whitespace. maint/findMergedChanges.pl -- Find a set of changes/* files that have been @@ -23,7 +23,7 @@ maint/format_changelog.py -- Flow the changelog into the proper format. maint/redox.py -- Find places that should have DOCDOC comments to indicate a need for doxygen comments, and put those comments there. -maint/updateVersions.pl -- Update the version number in the .nsi and windows +maint/update_versions.py -- Update the version number in the .nsi and windows orconfig.h files. diff --git a/scripts/build/combine_libs b/scripts/build/combine_libs index 9c87f68248..a855171dc7 100755 --- a/scripts/build/combine_libs +++ b/scripts/build/combine_libs @@ -8,15 +8,11 @@ ORIGDIR="$(pwd)" trap 'cd "$ORIGDIR" && rm -rf "$TMPDIR"' 0 abspath() { - echo "$(cd "$(dirname "$1")">/dev/null && pwd)/$(basename "$1")" + echo "$(cd "$(dirname "$1")" >/dev/null && pwd)/$(basename "$1")" } TARGET=$(abspath "$1") -#echo ORIGDIR="$ORIGDIR" -#echo AR="$AR" -#echo ARFLAGS="$AFLAGS" - shift for input in "$@"; do @@ -24,12 +20,11 @@ for input in "$@"; do abs=$(abspath "$input") dir="$TMPDIR"/$(basename "$input" .a) mkdir "$dir" - cd "$dir">/dev/null + cd "$dir" >/dev/null "${AR:-ar}" x "$abs" done cd "$TMPDIR" >/dev/null -#echo "${AR:-ar}" "${ARFLAGS:-cru}" library.tmp.a ./*/** "${AR:-ar}" "${ARFLAGS:-cru}" library.tmp.a ./*/** "${RANLIB:-ranlib}" library.tmp.a mv -f library.tmp.a "$TARGET" diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh index a13e0f6c17..b5a9c5cbf1 100755 --- a/scripts/ci/ci-driver.sh +++ b/scripts/ci/ci-driver.sh @@ -30,7 +30,6 @@ RUN_STAGE_TEST="${RUN_STAGE_TEST:-yes}" FATAL_WARNINGS="${FATAL_WARNINGS:-yes}" HARDENING="${HARDENING:-no}" COVERAGE="${COVERAGE:-no}" -RUST="${RUST:-no}" DOXYGEN="${DOXYGEN:-no}" ASCIIDOC="${ASCIIDOC:-no}" TRACING="${TRACING:-no}" @@ -193,7 +192,6 @@ yes_or_no ON_GITLAB yes_or_no FATAL_WARNINGS yes_or_no HARDENING yes_or_no COVERAGE -yes_or_no RUST yes_or_no DOXYGEN yes_or_no ASCIIDOC yes_or_no TRACING @@ -245,9 +243,6 @@ fi if [[ "$COVERAGE" == "yes" ]]; then configure_options+=("--enable-coverage") fi -if [[ "$RUST" == "yes" ]]; then - configure_options+=("--enable-rust") -fi if [[ "$ASCIIDOC" != "yes" ]]; then configure_options+=("--disable-asciidoc") fi @@ -452,6 +447,8 @@ if [[ "${CHUTNEY}" = "yes" ]]; then start_section "Chutney" export CHUTNEY_TOR_SANDBOX=0 export CHUTNEY_ALLOW_FAILURES=2 + # Send 5MB for every verify check. + export CHUTNEY_DATA_BYTES=5000000 if runcmd make "${CHUTNEY_MAKE_TARGET}"; then hooray "Chutney tests have succeeded" else diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index b52b956f81..d5d5cb0154 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -6,16 +6,18 @@ from __future__ import print_function from __future__ import unicode_literals FUZZERS = """ + address + addressPTR consensus descriptor diff diff-apply extrainfo - hsdescv2 hsdescv3 + hsdescv3-inner + hsdescv3-middle http http-connect - iptsv2 microdesc socks strops @@ -32,7 +34,6 @@ FUZZING_LDFLAG = \ @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @@ -47,11 +48,10 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh index 2bcd4722b4..29e91dd1b6 100755 --- a/scripts/git/git-list-tor-branches.sh +++ b/scripts/git/git-list-tor-branches.sh @@ -8,7 +8,7 @@ SCRIPT_NAME=$(basename "$0") function usage() { - echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R]" + echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R|-M]" echo echo " arguments:" echo " -h: show this help text" @@ -20,6 +20,7 @@ function usage() echo " ( branch parent path suffix parent_suffix ) arrays" echo echo " -R: omit release branches." + echo " -M: omit maint branches." } # list : just a list of branch names. @@ -27,9 +28,10 @@ function usage() # suffix: write a list of suffixes. # merge: branch, upstream, path, suffix, upstream suffix. mode="list" +skip_maint_branches="no" skip_release_branches="no" -while getopts "hblmsR" opt ; do +while getopts "hblmsRM" opt ; do case "$opt" in h) usage exit 0 @@ -42,6 +44,8 @@ while getopts "hblmsR" opt ; do ;; m) mode="merge" ;; + M) skip_maint_branches="yes" + ;; R) skip_release_branches="yes" ;; *) echo "Unknown option" @@ -73,13 +77,16 @@ branch() { # location: where the branch can be found. - if [[ "$brname" == "master" ]]; then - suffix="_master" + if [[ "$brname" == "main" ]]; then + suffix="_main" location="\$GIT_PATH/\$TOR_MASTER_NAME" elif [[ "$brname" =~ ^maint- ]]; then suffix="_${brname_nodots#maint-}" location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" is_maint="yes" + if [[ "$skip_maint_branches" = "yes" ]]; then + return + fi elif [[ "$brname" =~ ^release- ]]; then suffix="_r${brname_nodots#release-}" location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" @@ -139,12 +146,12 @@ finish() { branch maint-0.3.5 branch release-0.3.5 -branch maint-0.4.4 -branch release-0.4.4 - branch maint-0.4.5 branch release-0.4.5 -branch master +branch maint-0.4.6 +branch release-0.4.6 + +branch main finish diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 7c72f8478d..d5d663d558 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -12,8 +12,8 @@ function usage() echo " (default: run commands)" echo " -t: test branch mode: create new branches from the commits checked" echo " out in each maint directory. Call these branches prefix_035," - echo " prefix_040, ... , prefix_master." - echo " (default: merge forward maint-*, release-*, and master)" + echo " prefix_040, ... , prefix_main." + echo " (default: merge forward maint-*, release-*, and main)" echo " -u: in test branch mode, if a prefix_* branch already exists," echo " skip creating that branch. Use after a merge error, to" echo " restart the merge forward at the first unmerged branch." @@ -28,7 +28,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -45,7 +45,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The main branch git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -106,7 +106,7 @@ DRY_RUN=0 # Controlled by the -t <test-branch-prefix> option. The test branch base # name option makes git-merge-forward.sh create new test branches: -# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward. +# <tbbn>_035, <tbbn>_040, ... , <tbbn>_main, and merge forward. TEST_BRANCH_PREFIX= # Controlled by the -u option. The use existing option checks for existing diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 52a5c6140c..bbe2576d8e 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -20,7 +20,7 @@ usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -37,7 +37,7 @@ usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -51,7 +51,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -159,20 +159,7 @@ function goto_repo function fetch_origin { local cmd="git fetch origin" - printf " %s Fetching origin..." "$MARKER" - if [ $DRY_RUN -eq 0 ]; then - msg=$( eval "$cmd" 2>&1 ) - validate_ret $? "$msg" - else - printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" - fi -} - -# Fetch tor-github pull requests. No arguments. -function fetch_tor_github -{ - local cmd="git fetch tor-github" - printf " %s Fetching tor-github..." "$MARKER" + printf "%s Fetching origin..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -185,7 +172,7 @@ function fetch_tor_github function fetch_tor_gitlab { local cmd="git fetch tor-gitlab" - printf " %s Fetching tor-gitlab..." "$MARKER" + printf "%s Fetching tor-gitlab..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -198,11 +185,10 @@ function fetch_tor_gitlab # Entry point # ############### -# First, fetch tor-github. +# Get into our origin repository. goto_repo "$ORIGIN_PATH" -fetch_tor_github -# Then tor-gitlab +# First, fetch tor-gitlab fetch_tor_gitlab # Then, fetch the origin. diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 558ea8d01c..e5c16e615f 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -21,10 +21,10 @@ function usage() echo " -r: push to remote-name, rather than the default upstream remote." echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" echo " -t: test branch mode: push test branches to remote-name. Pushes" - echo " branches prefix_035, prefix_040, ... , prefix_master." - echo " (default: push maint-*, release-*, and master)" + echo " branches prefix_035, prefix_040, ... , prefix_main." + echo " (default: push maint-*, release-*, and main)" echo " -s: push branches whose tips match upstream maint, release, or" - echo " master branches. The default is to skip these branches," + echo " main branches. The default is to skip these branches," echo " because they do not contain any new code. Use -s to test for" echo " CI environment failures, using code that previously passed CI." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" @@ -45,7 +45,7 @@ function usage() echo " (default: use the current directory for pushes;" echo " current: $TOR_FULL_GIT_PATH)" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." @@ -55,9 +55,9 @@ function usage() echo " Overridden by <git push options> after --." echo " (default: git push --atomic; current: $GIT_PUSH)" echo " TOR_PUSH_SAME: push branches whose tips match upstream maint," - echo " release, or master branches. Inverted by -s." + echo " release, or main branches. Inverted by -s." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" - echo " TOR_PUSH_DELAY: pushes the master and maint branches separately," + echo " TOR_PUSH_DELAY: pushes the main and maint branches separately," echo " so that CI runs in a sensible order." echo " (default: push all branches immediately; current: $PUSH_DELAY)" echo " we recommend that you set these env vars in your ~/.profile" @@ -71,7 +71,7 @@ set -e # Don't change this configuration - set the env vars in your .profile # -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # Which directory do we push from? @@ -87,7 +87,7 @@ UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} # Push (1) or skip (0) test branches that are the same as an upstream -# maint/master branch. Push if you are testing that the CI environment still +# maint/main branch. Push if you are testing that the CI environment still # works on old code, skip if you are testing new code in the branch. # Default: skip unchanged branches. # Inverted by the -s option. @@ -99,7 +99,7 @@ PUSH_SAME=${TOR_PUSH_SAME:-0} # Controlled by the -t <test-branch-prefix> option. The test branch prefix # option makes git-merge-forward.sh create new test branches: -# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into +# <tbp>_035, <tbp>_040, ... , <tbp>_main, and merge each branch forward into # the next one. TEST_BRANCH_PREFIX= @@ -213,7 +213,7 @@ if [ "$TEST_BRANCH_PREFIX" ]; then # upstream branches (they have already been tested) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" else - # Skip the local maint-*, release-*, master branches that are the same as the + # Skip the local maint-*, release-*, main branches that are the same as the # current upstream branches, but ignore the default upstream # (we want to update a non-default remote, even if it matches the default) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" @@ -264,8 +264,8 @@ if [ "$PUSH_DELAY" -le 0 ]; then else # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) - MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ - || true # Skipped master branch + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep main$) \ + || true # Skipped main branch if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ || true # Skipped all maint branches @@ -273,7 +273,7 @@ else tr "\\n" " ") || true # Skipped all release branches else # Actually test branches based on maint branches - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v main$) \ || true # Skipped all maint test branches # No release branches RELEASE_BRANCHES= @@ -295,9 +295,9 @@ else # shellcheck disable=SC2086 for b in $MASTER_BRANCH $MAINT_BRANCHES; do $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" - # If we are pushing more than one branch, delay. - # In the unlikely scenario where we are pushing maint without master, - # or maint without release, there may be an extra delay + # If we are pushing more than one branch, delay. In the unlikely scenario + # where we are pushing maint branches without the main branch, or maint + # without release, there may be an extra delay if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then sleep "$PUSH_DELAY" fi diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh index 5a9ae41cbd..c502f74f58 100755 --- a/scripts/git/git-setup-dirs.sh +++ b/scripts/git/git-setup-dirs.sh @@ -22,7 +22,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -65,7 +65,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -102,7 +102,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -519,11 +519,11 @@ for ((i=0; i<COUNT; i++)); do repo_path=${!WORKTREE[$i]:1:1} printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" - # We cloned the repository, and master is the default branch - if [ "$branch" = "master" ]; then - if [ "$TOR_MASTER_NAME" != "master" ]; then - # Set up a master link in the worktree directory - make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + # We cloned the repository, and main is the default branch + if [ "$branch" = "main" ]; then + if [ "$TOR_MASTER_NAME" != "main" ]; then + # Set up a main branch link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/main" fi else # git makes worktree directories if they don't exist diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index eae4f999e7..b458630d26 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -36,8 +36,8 @@ check_for_script_update() { } cur_branch=$(git rev-parse --abbrev-ref HEAD) -if [ "$cur_branch" != "master" ]; then - echo "post-merge: Not a master branch. Skipping." +if [ "$cur_branch" != "main" ]; then + echo "post-merge: Not a main branch. Skipping." exit 0 fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f0a3a250ec..0f016df592 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -2,9 +2,9 @@ # git pre-push hook script to: # 0) Call the pre-commit hook, if it is available -# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# 1) prevent "fixup!" and "squash!" commit from ending up in main, release-* # or maint-* -# 2) Disallow pushing branches other than master, release-* +# 2) Disallow pushing branches other than main, release-* # and maint-* to origin (e.g. gitweb.torproject.org) # # To install this script, copy it into .git/hooks/pre-push path in your @@ -39,7 +39,7 @@ remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || + if [ "$1" == "refs/heads/main" ] || [[ "$1" == refs/heads/release-* ]] || [[ "$1" == refs/heads/maint-* ]]; then return 1 @@ -54,8 +54,8 @@ do : else if [ "$remote_sha" = $z40 ]; then - # New branch, examine commits not in master - range="master...$local_sha" + # New branch, examine commits not in main + range="main...$local_sha" else # Update to existing branch, examine new commits range="$remote_sha..$local_sha" diff --git a/scripts/maint/checkOptionDocs.pl.in b/scripts/maint/checkOptionDocs.pl.in index bb8008c2e8..2d4a7884f5 100644 --- a/scripts/maint/checkOptionDocs.pl.in +++ b/scripts/maint/checkOptionDocs.pl.in @@ -43,7 +43,7 @@ open(F, "@abs_top_srcdir@/doc/man/tor.1.txt") or die; while (<F>) { if (m!^(?:\[\[([A-za-z0-9_]+)\]\] *)?\*\*([A-Za-z0-9_]+)\*\*!) { $manPageOptions{$2} = 1; - print "Missing an anchor: $2\n" unless (defined $1 or $2 eq 'tor'); + print "Missing an anchor: $2\n" unless (defined $1 or $2 eq 'tor'); } } close F; diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh index 0a423be29e..f7e260bbd2 100755 --- a/scripts/maint/checkShellScripts.sh +++ b/scripts/maint/checkShellScripts.sh @@ -41,7 +41,6 @@ rm -f "$TOPLEVEL/contrib/dist/suse/tor.sh" "$TOPLEVEL/contrib/dist/tor.sh" find "$TOPLEVEL/contrib" "$TOPLEVEL/doc" "$TOPLEVEL/scripts" "$TOPLEVEL/src" \ -name "*.sh" \ -not -path "$TOPLEVEL/src/ext/*" \ - -not -path "$TOPLEVEL/src/rust/registry/*" \ -exec shellcheck {} + # Check scripts that aren't named *.sh diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 857ce6f6f1..8ecbf414cf 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -7,13 +7,13 @@ my $found = 0; my $COLON_POS = 10; sub msg { - $found = 1; - my $v = shift; - $v =~ /^\s*([^:]+):(.*)$/; - chomp(my $errtype = $1); - my $rest = $2; - my $padding = ' ' x ($COLON_POS - length $errtype); - print "$padding$errtype:$rest\n"; + $found = 1; + my $v = shift; + $v =~ /^\s*([^:]+):(.*)$/; + chomp(my $errtype = $1); + my $rest = $2; + my $padding = ' ' x ($COLON_POS - length $errtype); + print "$padding$errtype:$rest\n"; } my $C = 0; diff --git a/scripts/maint/check_config_macros.pl b/scripts/maint/check_config_macros.pl index bcde2beccc..1398b9984a 100755 --- a/scripts/maint/check_config_macros.pl +++ b/scripts/maint/check_config_macros.pl @@ -7,7 +7,7 @@ my @macros = (); open(F, 'orconfig.h.in'); while(<F>) { if (/^#undef +([A-Za-z0-9_]*)/) { - push @macros, $1; + push @macros, $1; } } close F; @@ -15,6 +15,6 @@ close F; for my $m (@macros) { my $s = `git grep '$m' src`; if ($s eq '') { - print "Unused: $m\n"; + print "Unused: $m\n"; } } diff --git a/scripts/maint/findMergedChanges.pl b/scripts/maint/findMergedChanges.pl index d6c4105b74..427f2b111d 100755 --- a/scripts/maint/findMergedChanges.pl +++ b/scripts/maint/findMergedChanges.pl @@ -9,7 +9,7 @@ sub nChanges { # requires perl 5.8. Avoids shell issues if we ever get a changes # file named by the parents of Little Johnny Tables. open F, "-|", "git", "log", "--no-merges", "--pretty=format:%H", $branches, "--", $fname - or die "$!"; + or die "$!"; my @changes = <F>; return scalar @changes } @@ -22,13 +22,13 @@ Usage: findMergedChanges.pl [--merged/--unmerged/--weird/--list] [--branch=<branchname] [--head=<branchname>] changes/* A change is "merged" if it has ever been merged to release-0.2.4 and it has had -no subsequent changes in master. +no subsequent changes in main. A change is "unmerged" if it has never been merged to release-0.2.4 and it -has had changes in master. +has had changes in main. A change is "weird" if it has been merged to release-0.2.4 and it *has* had -subsequent changes in master. +subsequent changes in main. Suggested application: findMergedChanges.pl --merged changes/* | xargs -n 1 git rm @@ -37,18 +37,18 @@ EOF } my $target_branch = "origin/release-0.2.4"; -my $head = "origin/master"; +my $head = "origin/main"; while (@ARGV and $ARGV[0] =~ /^--/) { my $flag = shift @ARGV; if ($flag =~ /^--(weird|merged|unmerged|list)/) { - $look_for_type = $1; + $look_for_type = $1; } elsif ($flag =~ /^--branch=(\S+)/) { $target_branch = $1; } elsif ($flag =~ /^--head=(\S+)/) { $head = $1; } else { - die "Unrecognized flag $flag"; + die "Unrecognized flag $flag"; } } @@ -58,16 +58,16 @@ for my $changefile (@ARGV) { my $type; if ($n_merged != 0 and $n_postmerged == 0) { - $type = "merged"; + $type = "merged"; } elsif ($n_merged == 0 and $n_postmerged != 0) { - $type = "unmerged"; + $type = "unmerged"; } else { - $type = "weird"; + $type = "weird"; } if ($type eq $look_for_type) { - print "$changefile\n"; + print "$changefile\n"; } elsif ($look_for_type eq 'list') { - printf "% 8s: %s\n", $type, $changefile; + printf "% 8s: %s\n", $type, $changefile; } } diff --git a/scripts/maint/gen_ccls_file.sh b/scripts/maint/gen_ccls_file.sh index b1fa55c973..04e31d22a8 100755 --- a/scripts/maint/gen_ccls_file.sh +++ b/scripts/maint/gen_ccls_file.sh @@ -19,13 +19,13 @@ echo "clang" > "$CCLS_FILE" # Add these include so the ccls server can properly check new files that are # not in the compile_commands.json yet { - echo "-I." - echo "-I./src" - echo "-I./src/ext" - echo "-I./src/ext/trunnel" + echo "-I." + echo "-I./src" + echo "-I./src/ext" + echo "-I./src/ext/trunnel" } >> "$CCLS_FILE" # Add all defines (-D). for p in $PRIVATE_DEFS; do - echo "-D$p" >> "$CCLS_FILE" + echo "-D$p" >> "$CCLS_FILE" done diff --git a/scripts/maint/geoip/update_geoip.sh b/scripts/maint/geoip/update_geoip.sh index 9289e7a969..743683ab62 100755 --- a/scripts/maint/geoip/update_geoip.sh +++ b/scripts/maint/geoip/update_geoip.sh @@ -5,7 +5,7 @@ set -e DIR=$(cd "$(dirname "$0")" && pwd) TMP=$(mktemp -d) -location update +location --quiet update location dump "$TMP/geoip-dump.txt" OLDDIR=$(pwd) diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py index a5ee728824..46630d987f 100755 --- a/scripts/maint/practracker/includes.py +++ b/scripts/maint/practracker/includes.py @@ -40,11 +40,13 @@ def warn(msg): print(msg, file=sys.stderr) def fname_is_c(fname): - """ Return true iff 'fname' is the name of a file that we should - search for possibly disallowed #include directives. """ - if fname.endswith(".h") or fname.endswith(".c"): + """ + Return true if 'fname' is the name of a file that we should + search for possibly disallowed #include directives. + """ + if fname.endswith((".c", ".h")): bname = os.path.basename(fname) - return not (bname.startswith(".") or bname.startswith("#")) + return not bname.startswith((".", "#")) else: return False diff --git a/scripts/maint/practracker/test_practracker.sh b/scripts/maint/practracker/test_practracker.sh index e29b9106de..bb734ad9cd 100755 --- a/scripts/maint/practracker/test_practracker.sh +++ b/scripts/maint/practracker/test_practracker.sh @@ -1,15 +1,15 @@ #!/bin/sh -# Fail if any subprocess fails unexpectedly +# Fail this script if any subprocess fails unexpectedly. set -e umask 077 unset TOR_DISABLE_PRACTRACKER TMPDIR="" -clean () { +clean() { if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then - rm -rf "$TMPDIR" + rm -rf "$TMPDIR" fi } trap clean EXIT HUP INT TERM @@ -27,7 +27,7 @@ PRACTRACKER_DIR="scripts/maint/practracker" TMPDIR="$(mktemp -d -t pracktracker.test.XXXXXX)" if test -z "${TMPDIR}" || test ! -d "${TMPDIR}" ; then echo >&2 "mktemp failed." - exit 1; + exit 1 fi DATA="${PRACTRACKER_DIR}/testdata" @@ -43,6 +43,7 @@ run_practracker() { --terse \ "${DATA}/" "$@" || echo "practracker exit status: $?" } + compare() { # we can't use cmp because we need to use -b for windows diff -b -u "$@" > "${TMPDIR}/test-diff" || true diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index c52ca2fbbf..6ab10a8de7 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -7,8 +7,7 @@ import os # We don't want to run metrics for unittests, automatically-generated C files, # external libraries or git leftovers. -EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/rust/", - "src/ext/" } +EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/ext/" } EXCLUDE_FILES = {"orconfig.h"} diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index b0268322f4..025a49cd03 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -5,8 +5,8 @@ set -e if test "x$CALLTOOL_PATH" != "x"; then - PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}" - export PYTHONPATH + PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}" + export PYTHONPATH fi mkdir -p callgraph @@ -14,8 +14,8 @@ mkdir -p callgraph SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgraph module_scc module_scc_weaklinks" for calculation in $SUBITEMS; do - echo "======== $calculation" - python -m calltool "$calculation" > callgraph/"$calculation" + echo "======== $calculation" + python -m calltool "$calculation" > callgraph/"$calculation" done cat <<EOF > callgraph/README diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh deleted file mode 100755 index 6d0587351f..0000000000 --- a/scripts/maint/updateRustDependencies.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Tor Project, Inc. -# Copyright (c) 2018 isis agora lovecruft -# See LICENSE for license information -# -# updateRustDependencies.sh -# ------------------------- -# Update our vendored Rust dependencies, either adding/removing -# dependencies and/or upgrading current dependencies to newer -# versions. -# -# To use this script, first add your dependencies, exactly specifying -# their versions, into the appropriate *crate-level* Cargo.toml in -# src/rust/ (i.e. *not* /src/rust/Cargo.toml, but instead the one for -# your crate). -# -# Next, run this script. Then, go into src/ext/rust and commit the -# changes to the tor-rust-dependencies repo. - -set -e - -HERE=$(dirname "$(realpath "$0")") -TOPLEVEL=$(dirname "$(dirname "$HERE")") -TOML="$TOPLEVEL/src/rust/Cargo.toml" -VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=$(command -v cargo) - -if ! test -f "$TOML" ; then - printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" -fi - -if ! test -d "$VENDORED" ; then - printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" -fi - -if test -z "$CARGO" ; then - printf "Error: cargo must be installed and in your \$PATH\\n" -fi - -if test -z "$(cargo --list | grep vendor)" ; then - printf "Error: cargo-vendor not installed\\n" -fi - -$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" diff --git a/src/app/config/config.c b/src/app/config/config.c index 2877bc1e6a..8df5275cc6 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -193,6 +193,7 @@ static const config_abbrev_t option_abbrevs_[] = { PLURAL(AuthDirBadDirCC), PLURAL(AuthDirBadExitCC), PLURAL(AuthDirInvalidCC), + PLURAL(AuthDirMiddleOnlyCC), PLURAL(AuthDirRejectCC), PLURAL(EntryNode), PLURAL(ExcludeNode), @@ -331,6 +332,8 @@ static const config_var_t option_vars_[] = { V(AuthDirBadExitCCs, CSV, ""), V(AuthDirInvalid, LINELIST, NULL), V(AuthDirInvalidCCs, CSV, ""), + V(AuthDirMiddleOnly, LINELIST, NULL), + V(AuthDirMiddleOnlyCCs, CSV, ""), V(AuthDirReject, LINELIST, NULL), V(AuthDirRejectCCs, CSV, ""), OBSOLETE("AuthDirRejectUnlisted"), @@ -616,6 +619,7 @@ static const config_var_t option_vars_[] = { V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(CircuitPadding, BOOL, "1"), + V(ReconfigDropsBridgeDescs, BOOL, "0"), V(ReducedCircuitPadding, BOOL, "0"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), @@ -668,6 +672,7 @@ static const config_var_t option_vars_[] = { VAR("UseEntryGuards", BOOL, UseEntryGuards_option, "1"), OBSOLETE("UseEntryGuardsAsDirGuards"), V(UseGuardFraction, AUTOBOOL, "auto"), + V(VanguardsLiteEnabled, AUTOBOOL, "auto"), V(UseMicrodescriptors, AUTOBOOL, "auto"), OBSOLETE("UseNTorHandshake"), V_IMMUTABLE(User, STRING, NULL), @@ -2309,6 +2314,8 @@ options_act,(const or_options_t *old_options)) } if (transition_affects_guards) { + if (options->ReconfigDropsBridgeDescs) + routerlist_drop_bridge_descriptors(); if (guards_update_all()) { abandon_circuits = 1; } @@ -5468,6 +5475,77 @@ pt_parse_transport_line(const or_options_t *options, return r; } +/** + * Parse a flag describing an extra dirport for a directory authority. + * + * Right now, the supported format is exactly: + * `{upload,download,voting}=http://[IP:PORT]/`. + * Other URL schemes, and other suffixes, might be supported in the future. + * + * Only call this function if `flag` starts with one of the above strings. + * + * Return 0 on success, and -1 on failure. + * + * If `ds` is provided, then add any parsed dirport to `ds`. If `ds` is NULL, + * take no action other than parsing. + **/ +static int +parse_dirauth_dirport(dir_server_t *ds, const char *flag) +{ + tor_assert(flag); + + auth_dirport_usage_t usage; + + if (!strcasecmpstart(flag, "upload=")) { + usage = AUTH_USAGE_UPLOAD; + } else if (!strcasecmpstart(flag, "download=")) { + usage = AUTH_USAGE_DOWNLOAD; + } else if (!strcasecmpstart(flag, "vote=")) { + usage = AUTH_USAGE_VOTING; + } else { + // We shouldn't get called with a flag that we don't recognize. + tor_assert_nonfatal_unreached(); + return -1; + } + + const char *eq = strchr(flag, '='); + tor_assert(eq); + const char *target = eq + 1; + + // Find the part inside the http://{....}/ + if (strcmpstart(target, "http://")) { + log_warn(LD_CONFIG, "Unsupported URL scheme in authority flag %s", flag); + return -1; + } + const char *addr = target + strlen("http://"); + + const char *eos = strchr(addr, '/'); + size_t addr_len; + if (eos && strcmp(eos, "/")) { + log_warn(LD_CONFIG, "Unsupported URL prefix in authority flag %s", flag); + return -1; + } else if (eos) { + addr_len = eos - addr; + } else { + addr_len = strlen(addr); + } + + // Finally, parse the addr:port part. + char *addr_string = tor_strndup(addr, addr_len); + tor_addr_port_t dirport; + memset(&dirport, 0, sizeof(dirport)); + int rv = tor_addr_port_parse(LOG_WARN, addr_string, + &dirport.addr, &dirport.port, -1); + if (ds != NULL && rv == 0) { + trusted_dir_server_add_dirport(ds, usage, &dirport); + } else if (rv == -1) { + log_warn(LD_CONFIG, "Unable to parse address in authority flag %s",flag); + } + + tor_free(addr_string); + return rv; +} + /** Read the contents of a DirAuthority line from <b>line</b>. If * <b>validate_only</b> is 0, and the line is well-formed, and it * shares any bits with <b>required_type</b> or <b>required_type</b> @@ -5488,6 +5566,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char v3_digest[DIGEST_LEN]; dirinfo_type_t type = 0; double weight = 1.0; + smartlist_t *extra_dirports = smartlist_new(); memset(v3_digest, 0, sizeof(v3_digest)); @@ -5556,6 +5635,12 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } ipv6_addrport_ptr = &ipv6_addrport; } + } else if (!strcasecmpstart(flag, "upload=") || + !strcasecmpstart(flag, "download=") || + !strcasecmpstart(flag, "vote=")) { + // We'll handle these after creating the authority object. + smartlist_add(extra_dirports, flag); + flag = NULL; // prevent double-free. } else { log_warn(LD_CONFIG, "Unrecognized flag '%s' on DirAuthority line", flag); @@ -5599,6 +5684,13 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, goto err; } + if (validate_only) { + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(NULL, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); + } + if (!validate_only && (!required_type || required_type & type)) { dir_server_t *ds; if (required_type) @@ -5610,16 +5702,23 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, ipv6_addrport_ptr, digest, v3_digest, type, weight))) goto err; + + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(ds, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); dir_server_add(ds); } r = 0; goto done; - err: + err: r = -1; - done: + done: + SMARTLIST_FOREACH(extra_dirports, char*, s, tor_free(s)); + smartlist_free(extra_dirports); SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 151b77c457..3a1acad044 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -293,6 +293,13 @@ struct or_options_t { * disabled. */ int CircuitPadding; + /** Boolean: if true, then this client will discard cached bridge + * descriptors on a setconf or other config change that impacts guards + * or bridges (see options_transition_affects_guards() for exactly which + * config changes trigger it). Useful for tools that test bridge + * reachability by fetching fresh descriptors. */ + int ReconfigDropsBridgeDescs; + /** Boolean: if true, then this client will only use circuit padding * algorithms that are known to use a low amount of overhead. If false, * we will use all available circuit padding algorithms. @@ -492,6 +499,9 @@ struct or_options_t { struct smartlist_t *NodeFamilySets; struct config_line_t *AuthDirBadExit; /**< Address policy for descriptors to * mark as bad exits. */ + /** Address policy for descriptors to mark as only suitable for the + * middle position in circuits. */ + struct config_line_t *AuthDirMiddleOnly; struct config_line_t *AuthDirReject; /**< Address policy for descriptors to * reject. */ struct config_line_t *AuthDirInvalid; /**< Address policy for descriptors to @@ -505,6 +515,7 @@ struct or_options_t { */ struct smartlist_t *AuthDirBadExitCCs; struct smartlist_t *AuthDirInvalidCCs; + struct smartlist_t *AuthDirMiddleOnlyCCs; struct smartlist_t *AuthDirRejectCCs; /**@}*/ @@ -587,6 +598,9 @@ struct or_options_t { * If 0, use value from NumEntryGuards. */ int NumPrimaryGuards; /**< How many primary guards do we want? */ + /** Boolean: Switch to toggle the vanguards-lite subsystem */ + int VanguardsLiteEnabled; + int RephistTrackTime; /**< How many seconds do we keep rephist info? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ diff --git a/src/app/include.am b/src/app/include.am index 2e2180deca..5494d904a3 100644 --- a/src/app/include.am +++ b/src/app/include.am @@ -17,7 +17,6 @@ src_app_tor_SOURCES = src/app/main/tor_main.c src_app_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ @TOR_STATIC_LDFLAGS@ src_app_tor_LDADD = libtor.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ diff --git a/src/app/main/main.c b/src/app/main/main.c index 89564490e6..f7b532f0ab 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -27,6 +27,8 @@ #include "core/or/channel.h" #include "core/or/channelpadding.h" #include "core/or/circuitpadding.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" #include "core/or/circuitlist.h" #include "core/or/command.h" #include "core/or/connection_or.h" @@ -100,12 +102,6 @@ #include <systemd/sd-daemon.h> #endif /* defined(HAVE_SYSTEMD) */ -#ifdef HAVE_RUST -// helper function defined in Rust to output a log message indicating if tor is -// running with Rust enabled. See src/rust/tor_util -void rust_log_welcome_string(void); -#endif - /********* PROTOTYPES **********/ static void dumpmemusage(int severity); @@ -609,10 +605,6 @@ tor_init(int argc, char *argv[]) tor_compress_log_init_warnings(); } -#ifdef HAVE_RUST - rust_log_welcome_string(); -#endif /* defined(HAVE_RUST) */ - /* Warn _if_ the tracing subsystem is built in. */ tracing_log_warning(); @@ -630,6 +622,8 @@ tor_init(int argc, char *argv[]) * until we get a consensus */ channelpadding_new_consensus_params(NULL); circpad_new_consensus_params(NULL); + congestion_control_new_consensus_params(NULL); + flow_control_new_consensus_params(NULL); /* Initialize circuit padding to defaults+torrc until we get a consensus */ circpad_machines_init(); @@ -1343,6 +1337,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) pubsub_connect(); if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { +#ifdef ENABLE_FRAGILE_HARDENING + log_warn(LD_CONFIG, "Sandbox is enabled but this Tor was built using " + "fragile compiler hardening. The sandbox may be unable to filter " + "requests to open files and directories and its overall " + "effectiveness will be reduced."); +#endif + sandbox_cfg_t* cfg = sandbox_init_filter(); if (sandbox_init(cfg)) { diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 7f43cd324e..667ab294b4 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -224,4 +224,4 @@ ## mechanisms like https://bridges.torproject.org/. If you want to run ## a private bridge, for example because you'll give out your bridge ## address manually to your friends, uncomment this line: -#PublishServerDescriptor 0 +#BridgeDistribution none diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 5d593871dd..edc30d043c 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -239,7 +239,7 @@ ## mechanisms like https://bridges.torproject.org/. If you want to run ## a private bridge, for example because you'll give out your bridge ## address manually to your friends, uncomment this line: -#PublishServerDescriptor 0 +#BridgeDistribution none ## Configuration options can be imported from files or folders using the %include ## option with the value being a path. This path can have wildcards. Wildcards are diff --git a/src/core/crypto/include.am b/src/core/crypto/include.am index 28b7e22905..2d53b3cb0b 100644 --- a/src/core/crypto/include.am +++ b/src/core/crypto/include.am @@ -5,6 +5,7 @@ LIBTOR_APP_A_SOURCES += \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ src/core/crypto/onion_ntor.c \ + src/core/crypto/onion_ntor_v3.c \ src/core/crypto/onion_tap.c \ src/core/crypto/relay_crypto.c @@ -14,5 +15,6 @@ noinst_HEADERS += \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ src/core/crypto/onion_ntor.h \ + src/core/crypto/onion_ntor_v3.h \ src/core/crypto/onion_tap.h \ src/core/crypto/relay_crypto.h diff --git a/src/core/crypto/onion_ntor_v3.c b/src/core/crypto/onion_ntor_v3.c new file mode 100644 index 0000000000..491c69cf8d --- /dev/null +++ b/src/core/crypto/onion_ntor_v3.c @@ -0,0 +1,760 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file onion_ntor_v3.c + * @brief Implements the version 3 ntor handshake as first specified in + * proposal 332. + * + * The v3 ntor handshake differs from the earlier versions (ntor and hs-ntor) + * primarily in that it allows the client to send an authenticated encrypted + * message as part of its onion skin, and allows the relay to send and + * encrypted authenticated reply as part of its response. + * + * It also takes a "verification string" -- the handshake cannot succeed + * unless both parties use the same value for their verification stream. + **/ + +#define ONION_NTOR_V3_PRIVATE + +#include "orconfig.h" +#include "core/crypto/onion_ntor_v3.h" + +#include "lib/arch/bytes.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/ctime/di_ops.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/* Parameters used to keep the outputs of this handshake from colliding with + * others. These are defined in the specification. */ +#define PROTOID "ntor3-curve25519-sha3_256-1" +#define TWEAK(A) (PROTOID ":" A) + +#define T_MSGKDF TWEAK("kdf_phase1") +#define T_MSGMAC TWEAK("msg_mac") +#define T_KEY_SEED TWEAK("key_seed") +#define T_VERIFY TWEAK("verify") +#define T_FINAL TWEAK("kdf_final") +#define T_AUTH TWEAK("auth_final") + +/** + * Add @a len bytes of @a data as input to the provided @a xof. + * + * (This is provided just for abbreviation). + **/ +#define xof_add(xof, data, len) crypto_xof_add_bytes((xof), (data), (len)) +/** + * Add @a len bytes of @a data as input to the provided @a xof, + * prefixed with an encoding of the length. + * + * This is equivalent to ENCAP(data) in the spec. + **/ +static void +xof_add_encap(crypto_xof_t *xof, const uint8_t *data, size_t len) +{ + uint64_t len64 = tor_htonll(len); + xof_add(xof, (uint8_t *)(&len64), 8); + xof_add(xof, data, len); +} +/** + * Add an encapsulated tweak to the provided xof. + **/ +#define xof_add_tweak(d, s) xof_add_encap((d), (const uint8_t *)(s), strlen(s)) + +/** + * Add @a len bytes of @a data to the provided @a digest. + * + * This is provided as an abbreviation, and to get the types right. + **/ +static void +d_add(crypto_digest_t *digest, const uint8_t *data, size_t len) +{ + crypto_digest_add_bytes(digest, (const char *)data, len); +} +/** + * Add @a len bytes of @a data to the provided @a digest, prefixed + * with the encoded length. + * + * This is equivalent to ENCAP(data) from the spec. + **/ +static void +d_add_encap(crypto_digest_t *digest, const uint8_t *data, size_t len) +{ + uint64_t len64 = tor_htonll(len); + d_add(digest, (const uint8_t *)(&len64), 8); + d_add(digest, data, len); +} +/** + * Add an encapsulated tweak to the provided digest. + **/ +#define d_add_tweak(d, s) d_add_encap((d), (const uint8_t *)(s), strlen(s)) + +/** + * Helper: copy @a len bytes of @a data onto *@a ptr, and advance @a ptr + * forward by @a len bytes. + * + * Asserts that @a ptr will not be advanced beyond @a endptr. + **/ +static void +push(uint8_t **ptr, const uint8_t *endptr, const uint8_t *data, size_t len) +{ + size_t remaining = endptr - *ptr; + tor_assert(len <= remaining); + memcpy(*ptr, data, len); + *ptr += len; +} + +/** + * Helper: Drop storage held by @a state, after wiping it. + **/ +void +ntor3_handshake_state_free_(ntor3_handshake_state_t *state) +{ + if (!state) + return; + + memwipe(state, 0, sizeof(*state)); + tor_free(state); +} + +/** + * Perform a client-side v3 ntor handshake with a given relay. + * + * As inputs this function takes the relay's Ed25519 identity (@a relay_id), + * the relay's current ntor onion key (@a relay_key), a verification string + * (@a verification_len bytes at @a verification), and a message to send + * as part of the handshake (@a message_len bytes at @a message). + * + * The message will be encrypted and authenticated to the relay, but will not + * receive the same forward secrecy as the rest of the handshake. We should + * not put any super-confidential data in it. + * + * The handshake will only succeed if the relay uses the same verification + * string as we are using. + * + * As outputs, this function returns 0 on success and -1 on failure. On + * success, it sets @a onion_skin_out and @a onion_skin_len_out to a newly + * allocated handshake message that the client can send as part of its CREATE2 + * or EXTEND2 cell. It also sets it sets @a handshake_state_out to a newly + * allocated handshake state object; the client needs to use this object to + * process the relay's eventual reply. + **/ +int +onion_skin_ntor3_create(const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out) +{ + curve25519_keypair_t client_keypair; + if (curve25519_keypair_generate(&client_keypair, 0) < 0) { + return -1; + } + int r = onion_skin_ntor3_create_nokeygen( + &client_keypair, + relay_id, + relay_key, + verification, + verification_len, + message, + message_len, + handshake_state_out, + onion_skin_out, + onion_skin_len_out); + memwipe(&client_keypair, 0, sizeof(client_keypair)); + return r; +} + +/** + * Like onion_skin_ntor3_create, but do not generate a new ephemeral keypair. + * Instead, take the ephemeral keypair (x,X) from @a client_keypair. + * + * (Having a separate function for this lets us test the code for correct + * behavior.) + **/ +STATIC int +onion_skin_ntor3_create_nokeygen( + const curve25519_keypair_t *client_keypair, + const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out) +{ + *handshake_state_out = NULL; + *onion_skin_out = NULL; + *onion_skin_len_out = 0; + + // Set up the handshake state object. + *handshake_state_out = tor_malloc_zero(sizeof(ntor3_handshake_state_t)); + memcpy(&(*handshake_state_out)->client_keypair, client_keypair, + sizeof(*client_keypair)); + memcpy(&(*handshake_state_out)->relay_id, relay_id, sizeof(*relay_id)); + memcpy(&(*handshake_state_out)->relay_key, relay_key, sizeof(*relay_key)); + + // Perform the first DH handshake. + curve25519_handshake((*handshake_state_out)->bx, + &client_keypair->seckey, relay_key); + if (safe_mem_is_zero((*handshake_state_out)->bx, CURVE25519_OUTPUT_LEN)) { + // Okay to return early here, since our behavior here doesn't + // cause a visible timing sidechannel. + return -1; + } + + // Compute phase1_keys. + uint8_t enc_key[CIPHER256_KEY_LEN]; + uint8_t mac_key[DIGEST256_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + // secret_input_phase1 = Bx | ID | X | B | PROTOID | ENCAP(VER) + xof_add_tweak(xof, T_MSGKDF); + xof_add(xof, (*handshake_state_out)->bx, CURVE25519_OUTPUT_LEN); + xof_add(xof, relay_id->pubkey, ED25519_PUBKEY_LEN); + xof_add(xof, client_keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, relay_key->public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, (const uint8_t *)PROTOID, strlen(PROTOID)); + xof_add_encap(xof, verification, verification_len); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, mac_key, sizeof(mac_key)); + crypto_xof_free(xof); + } + + // Compute encrypted message. + uint8_t *encrypted_message = tor_memdup(message, message_len); + { + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_crypt_inplace(c, (char *)encrypted_message, message_len); + crypto_cipher_free(c); + } + + // Compute the MAC value. + { + crypto_digest_t *m = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(m, T_MSGMAC); + d_add_encap(m, mac_key, sizeof(mac_key)); + d_add(m, relay_id->pubkey, ED25519_PUBKEY_LEN); + d_add(m, relay_key->public_key, CURVE25519_PUBKEY_LEN); + d_add(m, client_keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + d_add(m, encrypted_message, message_len); + crypto_digest_get_digest(m, + (char *)(*handshake_state_out)->msg_mac, + DIGEST256_LEN); + crypto_digest_free(m); + } + + // Build the onionskin. + *onion_skin_len_out = (ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN*2 + + DIGEST256_LEN + message_len); + *onion_skin_out = tor_malloc(*onion_skin_len_out); + { + uint8_t *ptr = *onion_skin_out, *end = ptr + *onion_skin_len_out; + + push(&ptr, end, relay_id->pubkey, ED25519_PUBKEY_LEN); + push(&ptr, end, relay_key->public_key, CURVE25519_PUBKEY_LEN); + push(&ptr, end, client_keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + push(&ptr, end, encrypted_message, message_len); + push(&ptr, end, (*handshake_state_out)->msg_mac, DIGEST256_LEN); + tor_assert(ptr == end); + } + + memwipe(&enc_key, 0, sizeof(enc_key)); + memwipe(&mac_key, 0, sizeof(mac_key)); + memwipe(encrypted_message, 0, message_len); + tor_free(encrypted_message); + + return 0; +} + +/** + * Complete a client-side v3 ntor handshake. + * + * Takes a @a handshake_state returned earlier by `onion_skin_ntor3_create()`, + * and the relay's reply to that handshake (@a reply_len bytes at @a + * handshake_reply). Also takes a verification string (@a verification_len + * bytes at @a verification). + * + * Returns 0 on success and -1 on failure. On success, generates @a key_len + * bytes of key material into the provided @a keys_out buffer, and sets @a + * message_out to the message that the relay sent in reply to our message (and + * sets @a message_out_len to that message's length). + **/ +int +onion_ntor3_client_handshake(const ntor3_handshake_state_t *handshake_state, + const uint8_t *handshake_reply, + size_t reply_len, + const uint8_t *verification, + size_t verification_len, + uint8_t *keys_out, + size_t keys_out_len, + uint8_t **message_out, + size_t *message_len_out) +{ + *message_out = NULL; + *message_len_out = 0; + + int problems = 0; + + // Parse the relay's message. + curve25519_public_key_t relay_Y; + uint8_t relay_auth[DIGEST256_LEN]; + size_t encrypted_msg_len; + const uint8_t *encrypted_msg; + { + if (reply_len < CURVE25519_PUBKEY_LEN + DIGEST256_LEN) { + // Okay to return early here, since the message is completely + // ill-formed, so we can't leak anything. + ++problems; + goto done; + } + encrypted_msg_len = reply_len - (CURVE25519_PUBKEY_LEN + DIGEST256_LEN); + + memcpy(&relay_Y.public_key, handshake_reply, CURVE25519_PUBKEY_LEN); + handshake_reply += CURVE25519_PUBKEY_LEN; + memcpy(&relay_auth, handshake_reply, DIGEST256_LEN); + handshake_reply += DIGEST256_LEN; + encrypted_msg = handshake_reply; + } + + // Finish the second diffie hellman handshake. + uint8_t yx[CURVE25519_OUTPUT_LEN]; + curve25519_handshake(yx, &handshake_state->client_keypair.seckey, &relay_Y); + problems |= safe_mem_is_zero(yx, sizeof(yx)); + + // Compute two tweaked hashes of secret_input. + uint8_t key_seed[DIGEST256_LEN], verify[DIGEST256_LEN]; + { + crypto_digest_t *ks = crypto_digest256_new(DIGEST_SHA3_256); + crypto_digest_t *v = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(ks, T_KEY_SEED); + d_add_tweak(v, T_VERIFY); +#define ADD2(s,len) STMT_BEGIN { \ + d_add(ks, (s),(len)); d_add(v, (s), (len)); \ + } STMT_END +#define ADD2_ENCAP(s,len) STMT_BEGIN { \ + d_add_encap(ks, (s),(len)); d_add_encap(v, (s), (len)); \ + } STMT_END + + ADD2(yx, sizeof(yx)); + ADD2(handshake_state->bx, sizeof(handshake_state->bx)); + ADD2(handshake_state->relay_id.pubkey, ED25519_PUBKEY_LEN); + ADD2(handshake_state->relay_key.public_key, CURVE25519_PUBKEY_LEN); + ADD2(handshake_state->client_keypair.pubkey.public_key, + CURVE25519_PUBKEY_LEN); + ADD2(relay_Y.public_key, CURVE25519_PUBKEY_LEN); + ADD2((const uint8_t *)PROTOID, strlen(PROTOID)); + ADD2_ENCAP(verification, verification_len); + + crypto_digest_get_digest(ks, (char*) key_seed, DIGEST256_LEN); + crypto_digest_get_digest(v, (char*) verify, DIGEST256_LEN); + crypto_digest_free(ks); + crypto_digest_free(v); + } + + // compute expected auth value. + uint8_t auth_computed[DIGEST256_LEN]; + { + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(d, T_AUTH); + d_add(d, verify, sizeof(verify)); + d_add(d, handshake_state->relay_id.pubkey, ED25519_PUBKEY_LEN); + d_add(d, handshake_state->relay_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, relay_Y.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, handshake_state->client_keypair.pubkey.public_key, + CURVE25519_PUBKEY_LEN); + d_add(d, handshake_state->msg_mac, DIGEST256_LEN); + d_add_encap(d, encrypted_msg, encrypted_msg_len); + d_add(d, (const uint8_t*)PROTOID, strlen(PROTOID)); + d_add(d, (const uint8_t*)"Server", strlen("Server")); + crypto_digest_get_digest(d, (char *)auth_computed, DIGEST256_LEN); + crypto_digest_free(d); + } + + // Check authentication value. + problems |= tor_memneq(auth_computed, relay_auth, DIGEST256_LEN); + + // Compute keystream, decrypt message, and return. + *message_out = tor_malloc(encrypted_msg_len); + *message_len_out = encrypted_msg_len; + uint8_t enc_key[CIPHER256_KEY_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + xof_add_tweak(xof, T_FINAL); + xof_add(xof, key_seed, sizeof(key_seed)); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, (uint8_t *)keys_out, keys_out_len); + crypto_xof_free(xof); + + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_decrypt(c, (char *)*message_out, + (const char *)encrypted_msg, encrypted_msg_len); + crypto_cipher_free(c); + } + + done: + memwipe(&relay_Y, 0, sizeof(relay_Y)); + memwipe(&relay_auth, 0, sizeof(relay_auth)); + memwipe(&yx, 0, sizeof(yx)); + memwipe(key_seed, 0, sizeof(key_seed)); + memwipe(verify, 0, sizeof(verify)); + memwipe(enc_key, 0, sizeof(enc_key)); + if (problems) { + if (*message_out) { + memwipe(*message_out, 0, *message_len_out); + tor_free(*message_out); // Sets it to NULL. + } + *message_len_out = 0; + crypto_rand((char*)keys_out, keys_out_len); // In case bad code uses it. + return -1; + } + + return 0; +} + +/** + * Wipe a server handshake state, and release the storage it holds. + **/ +void +ntor3_server_handshake_state_free_(ntor3_server_handshake_state_t *state) +{ + if (state == NULL) + return; + + memwipe(state, 0, sizeof(ntor3_server_handshake_state_t)); + tor_free(state); +} + +/** + * As a relay, start handling a client's v3 ntor handshake. + * + * This function performs the _first half_ of the handshake, up to the point + * where the client's message is decoded. After calling it, the relay should + * decide how and whether to reply to the client's message, compose its reply, + * and call `onion_skin_ntor3_server_handshake_part2`. + * + * It takes as input a map of the relay's known onion keys in @a private_keys, + * along with a fake @a junk_key to use if there is a complete mismatch. It + * takes the relay's ed25519 identity in @a my_id, along with the client's + * handshake message (@a client_handshake_len bytes in @a client_handshake), + * and a verification string (@a verification_len bytes in @a verification). + * + * Return 0 on success, and -1 on failure. On success, sets @a + * client_message_out to a newly allocated string holding the plaintext of the + * message that the client sent as part of its handshake, and @a + * client_message_out_len to its length. Also sets @a state_out to a newly + * allocated state object holding the intermediate computation for this + * handshake. + **/ +int +onion_skin_ntor3_server_handshake_part1( + const di_digest256_map_t *private_keys, + const curve25519_keypair_t *junk_key, + const ed25519_public_key_t *my_id, + const uint8_t *client_handshake, + size_t client_handshake_len, + const uint8_t *verification, + size_t verification_len, + uint8_t **client_message_out, + size_t *client_message_len_out, + ntor3_server_handshake_state_t **state_out) +{ + *client_message_out = NULL; + *client_message_len_out = 0; + *state_out = NULL; + + int problems = 0; + + // Initialize state. + (*state_out) = tor_malloc_zero(sizeof(ntor3_server_handshake_state_t)); + memcpy(&(*state_out)->my_id, my_id, sizeof(*my_id)); + + const uint8_t *wanted_id; // [ED25519_PUBKEY_LEN] + const uint8_t *wanted_key; // [CURVE25519_PUBKEY_LEN] + const uint8_t *encrypted_message; + size_t encrypted_message_len; + // Unpack the client handshake. + { + const uint8_t *ptr = client_handshake; + const uint8_t *end = ptr + client_handshake_len; + + if (client_handshake_len < + ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN * 2 + DIGEST256_LEN) { + // Okay to end early; the client knows this is unparseable already. + ++problems; + goto done; + } + wanted_id = ptr; + ptr += ED25519_PUBKEY_LEN; + wanted_key = ptr; + ptr += CURVE25519_PUBKEY_LEN; + memcpy((*state_out)->client_key.public_key, ptr, CURVE25519_PUBKEY_LEN); + ptr += CURVE25519_PUBKEY_LEN; + size_t remaining = (end-ptr); + if (BUG(remaining < DIGEST256_LEN)) { + // Okay to end early; this is a bug. + ++problems; + goto done; + } + encrypted_message = ptr; + encrypted_message_len = remaining - DIGEST256_LEN; + ptr += encrypted_message_len; + remaining = (end-ptr); + tor_assert(remaining == DIGEST256_LEN); + memcpy((*state_out)->msg_mac, ptr, DIGEST256_LEN); + } + + // Check the identity. + problems |= tor_memneq(my_id->pubkey, wanted_id, ED25519_PUBKEY_LEN); + + // Find the correct keypair. + const curve25519_keypair_t *keypair = + dimap_search(private_keys, wanted_key, (void *)junk_key); + tor_assert(keypair); + memcpy(&(*state_out)->my_key, &keypair->pubkey, + sizeof(curve25519_public_key_t)); + + // Do the first diffie hellman handshake. + curve25519_handshake((*state_out)->xb, + &keypair->seckey, &(*state_out)->client_key); + problems |= safe_mem_is_zero((*state_out)->xb, CURVE25519_OUTPUT_LEN); + + // Derive the encryption and mac keys + uint8_t enc_key[CIPHER256_KEY_LEN], mac_key[DIGEST256_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + xof_add_tweak(xof, T_MSGKDF); + xof_add(xof, (*state_out)->xb, CURVE25519_OUTPUT_LEN); + xof_add(xof, wanted_id, ED25519_PUBKEY_LEN); + xof_add(xof, (*state_out)->client_key.public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, (const uint8_t *)PROTOID, strlen(PROTOID)); + xof_add_encap(xof, verification, verification_len); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, mac_key, sizeof(mac_key)); + crypto_xof_free(xof); + } + + // Check the MAC. + uint8_t computed_mac[DIGEST256_LEN]; + { + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(d, T_MSGMAC); + d_add_encap(d, mac_key, sizeof(mac_key)); + d_add(d, my_id->pubkey, ED25519_PUBKEY_LEN); + d_add(d, keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, (*state_out)->client_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, encrypted_message, encrypted_message_len); + crypto_digest_get_digest(d, (char *)computed_mac, DIGEST256_LEN); + crypto_digest_free(d); + } + + problems |= tor_memneq((*state_out)->msg_mac, computed_mac, DIGEST256_LEN); + + // Decrypt the message. + *client_message_out = tor_malloc(encrypted_message_len); + *client_message_len_out = encrypted_message_len; + { + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_decrypt(c, (char *)*client_message_out, + (const char *)encrypted_message, + encrypted_message_len); + crypto_cipher_free(c); + } + + done: + memwipe(enc_key, 0, sizeof(enc_key)); + memwipe(mac_key, 0, sizeof(mac_key)); + memwipe(computed_mac, 0, sizeof(computed_mac)); + if (problems) { + if (*client_message_out) { + memwipe(*client_message_out, 0, *client_message_len_out); + tor_free(*client_message_out); // Sets it to NULL. + } + *client_message_len_out = 0; + ntor3_server_handshake_state_free(*state_out); + return -1; + } + + return 0; +} + +/** + * Finish the relay side of an ntor v3 handshake. + * + * The relay calls this function after it has decided to respond to the + * client's original encrypted message. This function receives the relay's + * message in @a server_message and its length in @a server_message_len, and + * completes the handshake. + * + * Returns 0 on success and -1 on failure. On success, stores the newly + * allocated handshake for the relay to send in @a handshake_out, and its + * length in @a handshake_len_out. Stores @a keys_out_len bytes of generated + * keys in the provided buffer at @a keys_out. + **/ +int +onion_skin_ntor3_server_handshake_part2( + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len) +{ + curve25519_keypair_t relay_keypair; + if (curve25519_keypair_generate(&relay_keypair, 0) < 0) { + return -1; + } + int r = onion_skin_ntor3_server_handshake_part2_nokeygen( + &relay_keypair, + state, + verification, + verification_len, + server_message, + server_message_len, + handshake_out, + handshake_len_out, + keys_out, + keys_out_len); + memwipe(&relay_keypair, 0, sizeof(relay_keypair)); + return r; +} + +/** + * Like `onion_skin_ntor3_server_handshake_part2`, but do not generate + * an ephemeral (y,Y) keypair. + * + * Instead, this function takes that keypair as @a relay_keypair_y. + * + * (Having a separate function for this lets us test the code for correct + * behavior.) + **/ +STATIC int +onion_skin_ntor3_server_handshake_part2_nokeygen( + const curve25519_keypair_t *relay_keypair_y, + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len) +{ + *handshake_out = NULL; + *handshake_len_out = 0; + + int problems = 0; + + // Second diffie-hellman handshake. + uint8_t xy[CURVE25519_OUTPUT_LEN]; + curve25519_handshake(xy, &relay_keypair_y->seckey, &state->client_key); + problems |= safe_mem_is_zero(xy, sizeof(xy)); + + // Compute two tweaked hashes of secret_input. + uint8_t key_seed[DIGEST256_LEN], verify[DIGEST256_LEN]; + { + crypto_digest_t *ks = crypto_digest256_new(DIGEST_SHA3_256); + crypto_digest_t *v = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(ks, T_KEY_SEED); + d_add_tweak(v, T_VERIFY); + ADD2(xy, sizeof(xy)); + ADD2(state->xb, sizeof(state->xb)); + ADD2(state->my_id.pubkey, ED25519_PUBKEY_LEN); + ADD2(state->my_key.public_key, CURVE25519_PUBKEY_LEN); + ADD2(state->client_key.public_key, CURVE25519_PUBKEY_LEN); + ADD2(relay_keypair_y->pubkey.public_key, CURVE25519_PUBKEY_LEN); + ADD2((const uint8_t *)PROTOID, strlen(PROTOID)); + ADD2_ENCAP(verification, verification_len); + crypto_digest_get_digest(ks, (char*) key_seed, DIGEST256_LEN); + crypto_digest_get_digest(v, (char*) verify, DIGEST256_LEN); + crypto_digest_free(ks); + crypto_digest_free(v); + } + + // Compute enc_key and keystream. + uint8_t enc_key[CIPHER256_KEY_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + xof_add_tweak(xof, T_FINAL); + xof_add(xof, key_seed, sizeof(key_seed)); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, keys_out, keys_out_len); + crypto_xof_free(xof); + } + + // Encrypt message. + uint8_t *encrypted_message = tor_memdup(server_message, server_message_len); + { + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_crypt_inplace( + c, (char *)encrypted_message, server_message_len); + crypto_cipher_free(c); + } + + // Compute AUTH digest. + uint8_t auth[DIGEST256_LEN]; + { + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(d, T_AUTH); + d_add(d, verify, sizeof(verify)); + d_add(d, state->my_id.pubkey, ED25519_PUBKEY_LEN); + d_add(d, state->my_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, relay_keypair_y->pubkey.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, state->client_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, state->msg_mac, DIGEST256_LEN); + d_add_encap(d, encrypted_message, server_message_len); + d_add(d, (const uint8_t*)PROTOID, strlen(PROTOID)); + d_add(d, (const uint8_t*)"Server", strlen("Server")); + crypto_digest_get_digest(d, (char *)auth, DIGEST256_LEN); + crypto_digest_free(d); + } + + // Compose the reply. + *handshake_len_out = CURVE25519_PUBKEY_LEN + DIGEST256_LEN + + server_message_len; + *handshake_out = tor_malloc(*handshake_len_out); + uint8_t *ptr = *handshake_out, *end = ptr + *handshake_len_out; + push(&ptr, end, relay_keypair_y->pubkey.public_key, CURVE25519_PUBKEY_LEN); + push(&ptr, end, auth, sizeof(auth)); + push(&ptr, end, encrypted_message, server_message_len); + tor_assert(ptr == end); + + // Clean up and return. + memwipe(xy, 0, sizeof(xy)); + memwipe(key_seed, 0, sizeof(key_seed)); + memwipe(verify, 0, sizeof(verify)); + memwipe(enc_key, 0, sizeof(enc_key)); + memwipe(encrypted_message, 0, server_message_len); + tor_free(encrypted_message); + + if (problems) { + memwipe(*handshake_out, 0, *handshake_len_out); + tor_free(*handshake_out); // Sets it to NULL. + *handshake_len_out = 0; + crypto_rand((char*)keys_out, keys_out_len); // In case bad code uses it. + return -1; + } + return 0; +} diff --git a/src/core/crypto/onion_ntor_v3.h b/src/core/crypto/onion_ntor_v3.h new file mode 100644 index 0000000000..4449eb237d --- /dev/null +++ b/src/core/crypto/onion_ntor_v3.h @@ -0,0 +1,140 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file onion_ntor_v3.h + * @brief Header for core/crypto/onion_ntor_v3.c + **/ + +#ifndef TOR_CORE_CRYPTO_ONION_NTOR_V3_H +#define TOR_CORE_CRYPTO_ONION_NTOR_V3_H + +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_curve25519.h" +#include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/malloc/malloc.h" + +/** + * Client-side state held while an ntor v3 handshake is in progress. + **/ +typedef struct ntor3_handshake_state_t ntor3_handshake_state_t; + +/** + * Server-side state held while the relay is handling a client's + * encapsulated message, before replying to the v3 handshake. + **/ +typedef struct ntor3_server_handshake_state_t ntor3_server_handshake_state_t; + +void ntor3_handshake_state_free_(ntor3_handshake_state_t *st); +#define ntor3_handshake_state_free(ptr) \ + FREE_AND_NULL(ntor3_handshake_state_t, ntor3_handshake_state_free_, (ptr)) +void ntor3_server_handshake_state_free_(ntor3_server_handshake_state_t *st); +#define ntor3_server_handshake_state_free(ptr) \ + FREE_AND_NULL(ntor3_server_handshake_state_t, \ + ntor3_server_handshake_state_free_, (ptr)) + +int onion_skin_ntor3_create(const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out); + +int onion_ntor3_client_handshake( + const ntor3_handshake_state_t *handshake_state, + const uint8_t *handshake_reply, + size_t reply_len, + const uint8_t *verification, + size_t verification_len, + uint8_t *keys_out, + size_t keys_out_len, + uint8_t **message_out, + size_t *message_len_out); + +struct di_digest256_map_t; +int onion_skin_ntor3_server_handshake_part1( + const struct di_digest256_map_t *private_keys, + const curve25519_keypair_t *junk_key, + const ed25519_public_key_t *my_id, + const uint8_t *client_handshake, + size_t client_handshake_len, + const uint8_t *verification, + size_t verification_len, + uint8_t **client_message_out, + size_t *client_message_len_out, + ntor3_server_handshake_state_t **state_out); + +int onion_skin_ntor3_server_handshake_part2( + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len); + +#ifdef ONION_NTOR_V3_PRIVATE +struct ntor3_handshake_state_t { + /** Ephemeral (x,X) keypair. */ + curve25519_keypair_t client_keypair; + /** Relay's ed25519 identity key (ID) */ + ed25519_public_key_t relay_id; + /** Relay's public key (B) */ + curve25519_public_key_t relay_key; + /** Shared secret (Bx). */ + uint8_t bx[CURVE25519_OUTPUT_LEN]; + /** MAC of the client's encrypted message data (MAC) */ + uint8_t msg_mac[DIGEST256_LEN]; +}; + +struct ntor3_server_handshake_state_t { + /** Relay's ed25519 identity key (ID) */ + ed25519_public_key_t my_id; + /** Relay's public key (B) */ + curve25519_public_key_t my_key; + /** Client's public ephemeral key (X). */ + curve25519_public_key_t client_key; + + /** Shared secret (Xb) */ + uint8_t xb[CURVE25519_OUTPUT_LEN]; + /** MAC of the client's encrypted message data */ + uint8_t msg_mac[DIGEST256_LEN]; +}; + +STATIC int onion_skin_ntor3_create_nokeygen( + const curve25519_keypair_t *client_keypair, + const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out); + +STATIC int onion_skin_ntor3_server_handshake_part2_nokeygen( + const curve25519_keypair_t *relay_keypair_y, + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len); + +#endif + +#endif /* !defined(TOR_CORE_CRYPTO_ONION_NTOR_V3_H) */ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index b17d7bf2bd..9271a70914 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -117,6 +117,7 @@ #include "lib/cc/ctassert.h" #include "lib/sandbox/sandbox.h" #include "lib/net/buffers_net.h" +#include "lib/net/address.h" #include "lib/tls/tortls.h" #include "lib/evloop/compat_libevent.h" #include "lib/compress/compress.h" @@ -146,6 +147,8 @@ #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" +#include "core/or/congestion_control_flow.h" + /** * On Windows and Linux we cannot reliably bind() a socket to an * address and port if: 1) There's already a socket bound to wildcard @@ -250,13 +253,13 @@ CONST_TO_LISTENER_CONN(const connection_t *c) } size_t -connection_get_inbuf_len(connection_t *conn) +connection_get_inbuf_len(const connection_t *conn) { return conn->inbuf ? buf_datalen(conn->inbuf) : 0; } size_t -connection_get_outbuf_len(connection_t *conn) +connection_get_outbuf_len(const connection_t *conn) { return conn->outbuf ? buf_datalen(conn->outbuf) : 0; } @@ -612,6 +615,11 @@ entry_connection_new(int type, int socket_family) entry_conn->entry_cfg.ipv4_traffic = 1; else if (socket_family == AF_INET6) entry_conn->entry_cfg.ipv6_traffic = 1; + + /* Initialize the read token bucket to the maximum value which is the same as + * no rate limiting. */ + token_bucket_rw_init(&ENTRY_TO_EDGE_CONN(entry_conn)->bucket, INT32_MAX, + INT32_MAX, monotime_coarse_get_stamp()); return entry_conn; } @@ -623,6 +631,10 @@ edge_connection_new(int type, int socket_family) edge_connection_t *edge_conn = tor_malloc_zero(sizeof(edge_connection_t)); tor_assert(type == CONN_TYPE_EXIT); connection_init(time(NULL), TO_CONN(edge_conn), type, socket_family); + /* Initialize the read token bucket to the maximum value which is the same as + * no rate limiting. */ + token_bucket_rw_init(&edge_conn->bucket, INT32_MAX, INT32_MAX, + monotime_coarse_get_stamp()); return edge_conn; } @@ -1261,7 +1273,7 @@ socket_failed_from_resource_exhaustion(void) */ if (get_max_sockets() > 65535) { /* TCP port exhaustion */ - rep_hist_note_overload(OVERLOAD_GENERAL); + rep_hist_note_tcp_exhaustion(); } else { /* File descriptor exhaustion */ rep_hist_note_overload(OVERLOAD_FD_EXHAUSTED); @@ -3457,6 +3469,19 @@ connection_bucket_read_limit(connection_t *conn, time_t now) base = get_cell_network_size(or_conn->wide_circ_ids); } + /* Edge connection have their own read bucket due to flow control being able + * to set a rate limit for them. However, for exit connections, we still need + * to honor the global bucket as well. */ + if (CONN_IS_EDGE(conn)) { + const edge_connection_t *edge_conn = CONST_TO_EDGE_CONN(conn); + conn_bucket = token_bucket_rw_get_read(&edge_conn->bucket); + if (conn->type == CONN_TYPE_EXIT) { + /* Decide between our limit and the global one. */ + goto end; + } + return conn_bucket; + } + if (!connection_is_rate_limited(conn)) { /* be willing to read on local conns even if our buckets are empty */ return conn_bucket>=0 ? conn_bucket : 1<<14; @@ -3467,6 +3492,7 @@ connection_bucket_read_limit(connection_t *conn, time_t now) global_bucket_val = MIN(global_bucket_val, relayed); } + end: return connection_bucket_get_share(base, priority, global_bucket_val, conn_bucket); } @@ -3644,6 +3670,13 @@ connection_buckets_decrement(connection_t *conn, time_t now, record_num_bytes_transferred_impl(conn, now, num_read, num_written); + /* Edge connection need to decrement the read side of the bucket used by our + * congestion control. */ + if (CONN_IS_EDGE(conn) && num_read > 0) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + token_bucket_rw_dec(&edge_conn->bucket, num_read, 0); + } + if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ @@ -3697,14 +3730,16 @@ connection_write_bw_exhausted(connection_t *conn, bool is_global_bw) void connection_consider_empty_read_buckets(connection_t *conn) { + int is_global = 1; const char *reason; - if (!connection_is_rate_limited(conn)) + if (CONN_IS_EDGE(conn) && + token_bucket_rw_get_read(&TO_EDGE_CONN(conn)->bucket) <= 0) { + reason = "edge connection read bucket exhausted. Pausing."; + is_global = false; + } else if (!connection_is_rate_limited(conn)) { return; /* Always okay. */ - - int is_global = 1; - - if (token_bucket_rw_get_read(&global_bucket) <= 0) { + } else if (token_bucket_rw_get_read(&global_bucket) <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && token_bucket_rw_get_read(&global_relayed_bucket) <= 0) { @@ -3714,8 +3749,9 @@ connection_consider_empty_read_buckets(connection_t *conn) token_bucket_rw_get_read(&TO_OR_CONN(conn)->bucket) <= 0) { reason = "connection read bucket exhausted. Pausing."; is_global = false; - } else + } else { return; /* all good, no need to stop it */ + } LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); connection_read_bw_exhausted(conn, is_global); @@ -3819,6 +3855,10 @@ connection_bucket_refill_single(connection_t *conn, uint32_t now_ts) or_connection_t *or_conn = TO_OR_CONN(conn); token_bucket_rw_refill(&or_conn->bucket, now_ts); } + + if (CONN_IS_EDGE(conn)) { + token_bucket_rw_refill(&TO_EDGE_CONN(conn)->bucket, now_ts); + } } /** @@ -4556,9 +4596,9 @@ connection_handle_write_impl(connection_t *conn, int force) !dont_stop_writing) { /* it's done flushing */ if (connection_finished_flushing(conn) < 0) { /* already marked */ - return -1; + goto err; } - return 0; + goto done; } /* Call even if result is 0, since the global write bucket may @@ -4568,7 +4608,17 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_read > 0 && connection_is_reading(conn)) connection_consider_empty_read_buckets(conn); + done: + /* If this is an edge connection with congestion control, check to see + * if it is time to send an xon */ + if (conn_uses_flow_control(conn)) { + flow_control_decide_xon(TO_EDGE_CONN(conn), n_written); + } + return 0; + + err: + return -1; } /* DOCDOC connection_handle_write */ diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 36c94d6570..8b378b15a4 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -274,8 +274,8 @@ void connection_buf_add_compress(const char *string, size_t len, struct dir_connection_t *conn, int done); void connection_buf_add_buf(struct connection_t *conn, struct buf_t *buf); -size_t connection_get_inbuf_len(struct connection_t *conn); -size_t connection_get_outbuf_len(struct connection_t *conn); +size_t connection_get_inbuf_len(const struct connection_t *conn); +size_t connection_get_outbuf_len(const struct connection_t *conn); struct connection_t *connection_get_by_global_id(uint64_t id); struct connection_t *connection_get_by_type(int type); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 69606c0d53..cd57dea3d4 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -641,6 +641,13 @@ connection_start_reading,(connection_t *conn)) if (connection_should_read_from_linked_conn(conn)) connection_start_reading_from_linked_conn(conn); } else { + if (CONN_IS_EDGE(conn) && TO_EDGE_CONN(conn)->xoff_received) { + /* We should not get called here if we're waiting for an XON, but + * belt-and-suspenders */ + log_notice(LD_NET, + "Request to start reading on an edgeconn blocked with XOFF"); + return; + } if (event_add(conn->read_event, NULL)) log_warn(LD_NET, "Error from libevent setting read event state for %d " "to watched: %s", @@ -1293,6 +1300,7 @@ signewnym_impl(time_t now) circuit_mark_all_dirty_circs_as_unusable(); addressmap_clear_transient(); hs_client_purge_state(); + purge_vanguards_lite(); time_of_last_signewnym = now; signewnym_is_pending = 0; @@ -1370,6 +1378,7 @@ CALLBACK(save_state); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); +CALLBACK(manage_vglite); #undef CALLBACK @@ -1392,6 +1401,9 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { CALLBACK(second_elapsed, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + /* Update vanguards-lite once per hour, if we have networking */ + CALLBACK(manage_vglite, NET_PARTICIPANT, FL(NEED_NET)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), @@ -1662,6 +1674,21 @@ mainloop_schedule_shutdown(int delay_sec) mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); } +/** + * Update vanguards-lite layer2 nodes, once every 15 minutes + */ +static int +manage_vglite_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; +#define VANGUARDS_LITE_INTERVAL (15*60) + + maintain_layer2_guards(); + + return VANGUARDS_LITE_INTERVAL; +} + /** Perform regular maintenance tasks. This function gets run once per * second. */ diff --git a/src/core/or/channel.c b/src/core/or/channel.c index c4f3e76fc8..c46fa93e58 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -2629,24 +2629,42 @@ channel_dump_statistics, (channel_t *chan, int severity)) circuitmux_num_circuits(chan->cmux) : 0); /* Describe timestamps */ - tor_log(severity, LD_GENERAL, - " * Channel %"PRIu64 " was last used by a " - "client at %"PRIu64 " (%"PRIu64 " seconds ago)", - (chan->global_identifier), - (uint64_t)(chan->timestamp_client), - (uint64_t)(now - chan->timestamp_client)); - tor_log(severity, LD_GENERAL, - " * Channel %"PRIu64 " last received a cell " - "at %"PRIu64 " (%"PRIu64 " seconds ago)", - (chan->global_identifier), - (uint64_t)(chan->timestamp_recv), - (uint64_t)(now - chan->timestamp_recv)); - tor_log(severity, LD_GENERAL, - " * Channel %"PRIu64 " last transmitted a cell " - "at %"PRIu64 " (%"PRIu64 " seconds ago)", - (chan->global_identifier), - (uint64_t)(chan->timestamp_xmit), - (uint64_t)(now - chan->timestamp_xmit)); + if (chan->timestamp_client == 0) { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " was never used by a " + "client", (chan->global_identifier)); + } else { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " was last used by a " + "client at %"PRIu64 " (%"PRIu64 " seconds ago)", + (chan->global_identifier), + (uint64_t)(chan->timestamp_client), + (uint64_t)(now - chan->timestamp_client)); + } + if (chan->timestamp_recv == 0) { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " never received a cell", + (chan->global_identifier)); + } else { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " last received a cell " + "at %"PRIu64 " (%"PRIu64 " seconds ago)", + (chan->global_identifier), + (uint64_t)(chan->timestamp_recv), + (uint64_t)(now - chan->timestamp_recv)); + } + if (chan->timestamp_xmit == 0) { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " never transmitted a cell", + (chan->global_identifier)); + } else { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " last transmitted a cell " + "at %"PRIu64 " (%"PRIu64 " seconds ago)", + (chan->global_identifier), + (uint64_t)(chan->timestamp_xmit), + (uint64_t)(now - chan->timestamp_xmit)); + } /* Describe counters and rates */ tor_log(severity, LD_GENERAL, diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 481dafef91..9db8e2392d 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -64,6 +64,7 @@ #include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/extendinfo.h" +#include "core/or/congestion_control_common.h" #include "core/or/cell_st.h" #include "core/or/cell_queue_st.h" @@ -793,7 +794,7 @@ channel_tls_num_cells_writeable_method(channel_t *chan) cell_network_size = get_cell_network_size(tlschan->conn->wide_circ_ids); outbuf_len = connection_get_outbuf_len(TO_CONN(tlschan->conn)); /* Get the number of cells */ - n = CEIL_DIV(OR_CONN_HIGHWATER - outbuf_len, cell_network_size); + n = CEIL_DIV(or_conn_highwatermark() - outbuf_len, cell_network_size); if (n < 0) n = 0; #if SIZEOF_SIZE_T > SIZEOF_INT if (n > INT_MAX) n = INT_MAX; diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 870bcbf7cf..be6429438a 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -22,6 +22,7 @@ struct hs_token_t; struct circpad_machine_spec_t; struct circpad_machine_runtime_t; +struct congestion_control_t; /** Number of padding state machines on a circuit. */ #define CIRCPAD_MAX_MACHINES (2) @@ -244,6 +245,9 @@ struct circuit_t { * that STOP commands actually correspond to the current machine, * and not a previous one. */ uint32_t padding_machine_ctr; + + /** Congestion control fields */ + struct congestion_control_t *ccontrol; }; #endif /* !defined(CIRCUIT_ST_H) */ diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 2bcc642a97..31e3868b65 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1359,7 +1359,9 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) int routelen = DEFAULT_ROUTE_LEN; int known_purpose = 0; - if (circuit_should_use_vanguards(purpose)) { + /* If we're using L3 vanguards, we need longer paths for onion services */ + if (circuit_purpose_is_hidden_service(purpose) && + get_options()->HSLayer3Nodes) { /* Clients want an extra hop for rends to avoid linkability. * Services want it for intro points to avoid publishing their * layer3 guards. They want it for hsdir posts to use @@ -1374,14 +1376,6 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) return routelen+1; - /* If we only have Layer2 vanguards, then we do not need - * the extra hop for linkabilty reasons (see below). - * This means all hops can be of the form: - * S/C - G - L2 - M - R/HSDir/I - */ - if (get_options()->HSLayer2Nodes && !get_options()->HSLayer3Nodes) - return routelen+1; - /* For connections to hsdirs, clients want two extra hops * when using layer3 guards, to avoid linkability. * Same goes for intro points. Note that the route len @@ -1400,16 +1394,14 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) return routelen; switch (purpose) { - /* These two purposes connect to a router that we chose, so - * DEFAULT_ROUTE_LEN is safe. */ - case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: - /* hidden service connecting to introduction point */ + /* These purposes connect to a router that we chose, so DEFAULT_ROUTE_LEN + * is safe: */ case CIRCUIT_PURPOSE_TESTING: /* router reachability testing */ known_purpose = 1; break; - /* These three purposes connect to a router that someone else + /* These purposes connect to a router that someone else * might have chosen, so add an extra hop to protect anonymity. */ case CIRCUIT_PURPOSE_C_GENERAL: case CIRCUIT_PURPOSE_C_HSDIR_GET: @@ -1419,6 +1411,9 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) /* client connecting to introduction point */ case CIRCUIT_PURPOSE_S_CONNECT_REND: /* hidden service connecting to rendezvous point */ + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + /* hidden service connecting to intro point. In this case we want an extra + hop to avoid linkability attacks by the introduction point. */ known_purpose = 1; routelen++; break; @@ -2019,7 +2014,7 @@ cpath_build_state_to_crn_ipv6_extend_flag(const cpath_build_state_t *state, } /** Decide a suitable length for circ's cpath, and pick an exit - * router (or use <b>exit</b> if provided). Store these in the + * router (or use <b>exit_ei</b> if provided). Store these in the * cpath. * * If <b>is_hs_v3_rp_circuit</b> is set, then this exit should be suitable to @@ -2072,7 +2067,7 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, return 0; } -/** Give <b>circ</b> a new exit destination to <b>exit</b>, and add a +/** Give <b>circ</b> a new exit destination to <b>exit_ei</b>, and add a * hop to the cpath reflecting this. Don't send the next extend cell -- * the caller will do this if it wants to. */ @@ -2114,8 +2109,6 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) return -1; } - // XXX: Should cannibalized circuits be dirty or not? Not easy to say.. - return 0; } @@ -2261,8 +2254,14 @@ middle_node_must_be_vanguard(const or_options_t *options, return 0; } - /* If we have sticky L2 nodes, and this is an L2 pick, use vanguards */ - if (options->HSLayer2Nodes && cur_len == 1) { + /* Don't even bother if the feature is disabled */ + if (!vanguards_lite_is_enabled()) { + return 0; + } + + /* If we are a hidden service circuit, always use either vanguards-lite + * or HSLayer2Nodes for 2nd hop. */ + if (cur_len == 1) { return 1; } @@ -2286,7 +2285,8 @@ pick_vanguard_middle_node(const or_options_t *options, /* Pick the right routerset based on the current hop */ if (cur_len == 1) { - vanguard_routerset = options->HSLayer2Nodes; + vanguard_routerset = options->HSLayer2Nodes ? + options->HSLayer2Nodes : get_layer2_guards(); } else if (cur_len == 2) { vanguard_routerset = options->HSLayer3Nodes; } else { @@ -2295,6 +2295,10 @@ pick_vanguard_middle_node(const or_options_t *options, return NULL; } + if (BUG(!vanguard_routerset)) { + return NULL; + } + node = pick_restricted_middle_node(flags, vanguard_routerset, options->ExcludeNodes, excluded, cur_len+1); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 4f62284e29..4dbf4d4549 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -64,6 +64,7 @@ #include "core/or/circuitpadding.h" #include "core/or/crypt_path.h" #include "core/or/extendinfo.h" +#include "core/or/status.h" #include "core/or/trace_probes_circuit.h" #include "core/mainloop/connection.h" #include "app/config/config.h" @@ -100,6 +101,7 @@ #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/buf/buffers.h" +#include "core/or/congestion_control_common.h" #include "core/or/ocirc_event.h" @@ -1143,6 +1145,8 @@ circuit_free_(circuit_t *circ) * hs identifier is freed. */ hs_circ_cleanup_on_free(circ); + congestion_control_free(circ->ccontrol); + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; @@ -2343,6 +2347,12 @@ circuit_about_to_free(circuit_t *circ) circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); circuit_set_p_circid_chan(or_circ, 0, NULL); } + + if (or_circ->n_cells_discarded_at_end) { + time_t age = approx_time() - circ->timestamp_created.tv_sec; + note_circ_closed_for_unrecognized_cells( + age, or_circ->n_cells_discarded_at_end); + } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); edge_connection_t *conn; @@ -2586,8 +2596,10 @@ conns_compare_by_buffer_age_(const void **a_, const void **b_) /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ -void + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. + * + * Return the number of bytes removed. */ +size_t circuits_handle_oom(size_t current_allocation) { smartlist_t *circlist; @@ -2597,6 +2609,7 @@ circuits_handle_oom(size_t current_allocation) size_t mem_recovered=0; int n_circuits_killed=0; int n_dirconns_killed=0; + int n_edgeconns_killed = 0; uint32_t now_ts; log_notice(LD_GENERAL, "We're low on memory (cell queues total alloc:" " %"TOR_PRIuSZ" buffer total alloc: %" TOR_PRIuSZ "," @@ -2613,12 +2626,11 @@ circuits_handle_oom(size_t current_allocation) tor_zstd_get_total_allocation(), tor_lzma_get_total_allocation(), hs_cache_get_total_allocation()); - { size_t mem_target = (size_t)(get_options()->MaxMemInQueues * FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) - return; + return 0; mem_to_recover = current_allocation - mem_target; } @@ -2664,12 +2676,19 @@ circuits_handle_oom(size_t current_allocation) if (conn_age < circ->age_tmp) { break; } - if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + /* Also consider edge connections so we don't accumulate bytes on the + * outbuf due to a malicious destination holding off the read on us. */ + if ((conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) || + CONN_IS_EDGE(conn)) { if (!conn->marked_for_close) connection_mark_for_close(conn); mem_recovered += single_conn_free_bytes(conn); - ++n_dirconns_killed; + if (conn->type == CONN_TYPE_DIR) { + ++n_dirconns_killed; + } else { + ++n_edgeconns_killed; + } if (mem_recovered >= mem_to_recover) goto done_recovering_mem; @@ -2697,14 +2716,16 @@ circuits_handle_oom(size_t current_allocation) } SMARTLIST_FOREACH_END(circ); done_recovering_mem: - log_notice(LD_GENERAL, "Removed %"TOR_PRIuSZ" bytes by killing %d circuits; " "%d circuits remain alive. Also killed %d non-linked directory " - "connections.", + "connections. Killed %d edge connections", mem_recovered, n_circuits_killed, smartlist_len(circlist) - n_circuits_killed, - n_dirconns_killed); + n_dirconns_killed, + n_edgeconns_killed); + + return mem_recovered; } /** Verify that circuit <b>c</b> has all of its invariants diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index f5791d7c12..147e2cb2f8 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -232,7 +232,7 @@ int circuit_count_pending_on_channel(channel_t *chan); MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c)); void circuit_free_all(void); -void circuits_handle_oom(size_t current_allocation); +size_t circuits_handle_oom(size_t current_allocation); void circuit_clear_testing_cell_stats(circuit_t *circ); diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c index 0382e62f75..adf256ab05 100644 --- a/src/core/or/circuitmux_ewma.c +++ b/src/core/or/circuitmux_ewma.c @@ -45,7 +45,10 @@ /*** EWMA parameter #defines ***/ /** How long does a tick last (seconds)? */ -#define EWMA_TICK_LEN 10 +#define EWMA_TICK_LEN_DEFAULT 10 +#define EWMA_TICK_LEN_MIN 1 +#define EWMA_TICK_LEN_MAX 600 +static int ewma_tick_len = EWMA_TICK_LEN_DEFAULT; /** The default per-tick scale factor, if it hasn't been overridden by a * consensus or a configuration setting. zero means "disabled". */ @@ -148,7 +151,7 @@ cell_ewma_get_tick(void) monotime_coarse_get(&now); int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, &now); - return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN); + return current_tick_num + msec_diff / (1000*ewma_tick_len); } /** @@ -527,15 +530,15 @@ cell_ewma_get_current_tick_and_fraction(double *remainder_out) monotime_coarse_get(&now); int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, &now); - if (msec_diff > (1000*EWMA_TICK_LEN)) { - unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN); + if (msec_diff > (1000*ewma_tick_len)) { + unsigned ticks_difference = msec_diff / (1000*ewma_tick_len); monotime_coarse_add_msec(&start_of_current_tick, &start_of_current_tick, - ticks_difference * 1000 * EWMA_TICK_LEN); + ticks_difference * 1000 * ewma_tick_len); current_tick_num += ticks_difference; - msec_diff %= 1000*EWMA_TICK_LEN; + msec_diff %= 1000*ewma_tick_len; } - *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN); + *remainder_out = ((double)msec_diff) / (1.0e3 * ewma_tick_len); return current_tick_num; } @@ -605,15 +608,20 @@ cmux_ewma_set_options(const or_options_t *options, /* Both options and consensus can be NULL. This assures us to either get a * valid configured value or the default one. */ halflife = get_circuit_priority_halflife(options, consensus, &source); + ewma_tick_len = networkstatus_get_param(consensus, + "CircuitPriorityTickSecs", + EWMA_TICK_LEN_DEFAULT, + EWMA_TICK_LEN_MIN, + EWMA_TICK_LEN_MAX); /* convert halflife into halflife-per-tick. */ - halflife /= EWMA_TICK_LEN; + halflife /= ewma_tick_len; /* compute per-tick scale factor. */ ewma_scale_factor = exp(LOG_ONEHALF / halflife); log_info(LD_OR, "Enabled cell_ewma algorithm because of value in %s; " "scale factor is %f per %d seconds", - source, ewma_scale_factor, EWMA_TICK_LEN); + source, ewma_scale_factor, ewma_tick_len); } /** Return the multiplier necessary to convert the value of a cell sent in diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6dfe94de01..99dc5f9d83 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2967,6 +2967,8 @@ signed_error_t circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) { int retval = 0; + /* Should we send back a STOP cell? */ + bool respond_with_stop = true; circpad_negotiate_t *negotiate; if (CIRCUIT_IS_ORIGIN(circ)) { @@ -2992,6 +2994,12 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) negotiate->machine_type, negotiate->machine_ctr); goto done; } + + /* If we reached this point we received a STOP command from an old or + unknown machine. Don't reply with our own STOP since there is no one to + handle it on the other end */ + respond_with_stop = false; + if (negotiate->machine_ctr <= circ->padding_machine_ctr) { log_info(LD_CIRC, "Received STOP command for old machine %u, ctr %u", negotiate->machine_type, negotiate->machine_ctr); @@ -3023,10 +3031,13 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) retval = -1; done: - circpad_padding_negotiated(circ, negotiate->machine_type, - negotiate->command, - (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR, - negotiate->machine_ctr); + if (respond_with_stop) { + circpad_padding_negotiated(circ, negotiate->machine_type, + negotiate->command, + (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR, + negotiate->machine_ctr); + } + circpad_negotiate_free(negotiate); return retval; diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 044b30b8b3..2ec391eca0 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1204,25 +1204,6 @@ needs_circuits_for_build(int num) return 0; } -/** - * Launch the appropriate type of predicted circuit for hidden - * services, depending on our options. - */ -static void -circuit_launch_predicted_hs_circ(int flags) -{ - /* K.I.S.S. implementation of bug #23101: If we are using - * vanguards or pinned middles, pre-build a specific purpose - * for HS circs. */ - if (circuit_should_use_vanguards(CIRCUIT_PURPOSE_HS_VANGUARDS)) { - circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); - } else { - /* If no vanguards, then no HS-specific prebuilt circuits are needed. - * Normal GENERAL circs are fine */ - circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); - } -} - /** Determine how many circuits we have open that are clean, * Make sure it's enough for all the upcoming behaviors we predict we'll have. * But put an upper bound on the total number of circuits. @@ -1276,7 +1257,7 @@ circuit_predict_and_launch_new(void) "Have %d clean circs (%d internal), need another internal " "circ for my hidden service.", num, num_internal); - circuit_launch_predicted_hs_circ(flags); + circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); return; } @@ -1295,7 +1276,10 @@ circuit_predict_and_launch_new(void) " another hidden service circ.", num, num_uptime_internal, num_internal); - circuit_launch_predicted_hs_circ(flags); + /* Always launch vanguards purpose circuits for HS clients, + * for vanguards-lite. This prevents us from cannibalizing + * to build these circuits (and thus not use vanguards). */ + circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); return; } @@ -2022,16 +2006,12 @@ circuit_is_hs_v3(const circuit_t *circ) int circuit_should_use_vanguards(uint8_t purpose) { - const or_options_t *options = get_options(); - - /* Only hidden service circuits use vanguards */ - if (!circuit_purpose_is_hidden_service(purpose)) - return 0; - - /* Pinned middles are effectively vanguards */ - if (options->HSLayer2Nodes || options->HSLayer3Nodes) + /* All hidden service circuits use either vanguards or + * vanguards-lite. */ + if (circuit_purpose_is_hidden_service(purpose)) return 1; + /* Everything else is a normal circuit */ return 0; } @@ -2069,13 +2049,11 @@ circuit_should_cannibalize_to_build(uint8_t purpose_to_build, return 0; } - /* For vanguards, the server-side intro circ is not cannibalized - * because we pre-build 4 hop HS circuits, and it only needs a 3 hop - * circuit. It is also long-lived, so it is more important that - * it have lower latency than get built fast. + /* The server-side intro circ is not cannibalized because it only + * needs a 3 hop circuit. It is also long-lived, so it is more + * important that it have lower latency than get built fast. */ - if (circuit_should_use_vanguards(purpose_to_build) && - purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) { + if (purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) { return 0; } diff --git a/src/core/or/command.c b/src/core/or/command.c index 622217a78e..40eb1554c0 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -563,7 +563,7 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) } if ((reason = circuit_receive_relay_cell(cell, circ, direction)) < 0) { - log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL,"circuit_receive_relay_cell " + log_fn(LOG_DEBUG,LD_PROTOCOL,"circuit_receive_relay_cell " "(%s) failed. Closing.", direction==CELL_DIRECTION_OUT?"forward":"backward"); /* Always emit a bandwidth event for closed circs */ diff --git a/src/core/or/congestion_control_common.c b/src/core/or/congestion_control_common.c new file mode 100644 index 0000000000..0919f037db --- /dev/null +++ b/src/core/or/congestion_control_common.c @@ -0,0 +1,1038 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_common.c + * \brief Common code used by all congestion control algorithms. + */ + +#define TOR_CONGESTION_CONTROL_COMMON_PRIVATE + +#include "core/or/or.h" + +#include "core/or/circuitlist.h" +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "core/mainloop/connection.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_vegas.h" +#include "core/or/congestion_control_nola.h" +#include "core/or/congestion_control_westwood.h" +#include "core/or/congestion_control_st.h" +#include "core/or/trace_probes_cc.h" +#include "lib/time/compat_time.h" +#include "feature/nodelist/networkstatus.h" + +/* Consensus parameter defaults. + * + * More details for each of the parameters can be found in proposal 324, + * section 6.5 including tuning notes. */ +#define CIRCWINDOW_INIT (500) +#define SENDME_INC_DFLT (50) + +#define CWND_INC_DFLT (50) +#define CWND_INC_PCT_SS_DFLT (100) +#define CWND_INC_RATE_DFLT (1) +#define CWND_MAX_DFLT (INT32_MAX) +#define CWND_MIN_DFLT (MAX(100, SENDME_INC_DFLT)) + +#define BWE_SENDME_MIN_DFLT (5) +#define EWMA_CWND_COUNT_DFLT (2) + +/* BDP algorithms for each congestion control algorithms use the piecewise + * estimattor. See section 3.1.4 of proposal 324. */ +#define WESTWOOD_BDP_ALG BDP_ALG_PIECEWISE +#define VEGAS_BDP_MIX_ALG BDP_ALG_PIECEWISE +#define NOLA_BDP_ALG BDP_ALG_PIECEWISE + +/* Indicate OR connection buffer limitations used to stop or start accepting + * cells in its outbuf. + * + * These watermarks are historical to tor in a sense that they've been used + * almost from the genesis point. And were likely defined to fit the bounds of + * TLS records of 16KB which would be around 32 cells. + * + * These are defaults of the consensus parameter "orconn_high" and "orconn_low" + * values. */ +#define OR_CONN_HIGHWATER_DFLT (32*1024) +#define OR_CONN_LOWWATER_DFLT (16*1024) + +/* Low and high values of circuit cell queue sizes. They are used to tell when + * to start or stop reading on the streams attached on the circuit. + * + * These are defaults of the consensus parameters "cellq_high" and "cellq_low". + */ +#define CELL_QUEUE_LOW_DFLT (10) +#define CELL_QUEUE_HIGH_DFLT (256) + +static uint64_t congestion_control_update_circuit_rtt(congestion_control_t *, + uint64_t); +static bool congestion_control_update_circuit_bdp(congestion_control_t *, + const circuit_t *, + const crypt_path_t *, + uint64_t, uint64_t); + +/* Consensus parameters cached. The non static ones are extern. */ +static uint32_t cwnd_max = CWND_MAX_DFLT; +int32_t cell_queue_high = CELL_QUEUE_HIGH_DFLT; +int32_t cell_queue_low = CELL_QUEUE_LOW_DFLT; +uint32_t or_conn_highwater = OR_CONN_HIGHWATER_DFLT; +uint32_t or_conn_lowwater = OR_CONN_LOWWATER_DFLT; + +/** + * Update global congestion control related consensus parameter values, + * every consensus update. + */ +void +congestion_control_new_consensus_params(const networkstatus_t *ns) +{ +#define CELL_QUEUE_HIGH_MIN (1) +#define CELL_QUEUE_HIGH_MAX (1000) + cell_queue_high = networkstatus_get_param(ns, "cellq_high", + CELL_QUEUE_HIGH_DFLT, + CELL_QUEUE_HIGH_MIN, + CELL_QUEUE_HIGH_MAX); + +#define CELL_QUEUE_LOW_MIN (1) +#define CELL_QUEUE_LOW_MAX (1000) + cell_queue_low = networkstatus_get_param(ns, "cellq_low", + CELL_QUEUE_LOW_DFLT, + CELL_QUEUE_LOW_MIN, + CELL_QUEUE_LOW_MAX); + +#define OR_CONN_HIGHWATER_MIN (CELL_PAYLOAD_SIZE) +#define OR_CONN_HIGHWATER_MAX (INT32_MAX) + or_conn_highwater = + networkstatus_get_param(ns, "orconn_high", + OR_CONN_HIGHWATER_DFLT, + OR_CONN_HIGHWATER_MIN, + OR_CONN_HIGHWATER_MAX); + +#define OR_CONN_LOWWATER_MIN (CELL_PAYLOAD_SIZE) +#define OR_CONN_LOWWATER_MAX (INT32_MAX) + or_conn_lowwater = + networkstatus_get_param(ns, "orconn_low", + OR_CONN_LOWWATER_DFLT, + OR_CONN_LOWWATER_MIN, + OR_CONN_LOWWATER_MAX); + +#define CWND_MAX_MIN 500 +#define CWND_MAX_MAX (INT32_MAX) + cwnd_max = + networkstatus_get_param(NULL, "cc_cwnd_max", + CWND_MAX_DFLT, + CWND_MAX_MIN, + CWND_MAX_MAX); +} + +/** + * Set congestion control parameters on a circuit's congestion + * control object based on values from the consensus. + * + * cc_alg is the negotiated congestion control algorithm. + * + * sendme_inc is the number of packaged cells that a sendme cell + * acks. This parameter will come from circuit negotiation. + */ +static void +congestion_control_init_params(congestion_control_t *cc, + cc_alg_t cc_alg, + int sendme_inc) +{ +#define CWND_INIT_MIN 100 +#define CWND_INIT_MAX (10000) + cc->cwnd = + networkstatus_get_param(NULL, "cc_cwnd_init", + CIRCWINDOW_INIT, + CWND_INIT_MIN, + CWND_INIT_MAX); + +#define CWND_INC_PCT_SS_MIN 1 +#define CWND_INC_PCT_SS_MAX (500) + cc->cwnd_inc_pct_ss = + networkstatus_get_param(NULL, "cc_cwnd_inc_pct_ss", + CWND_INC_PCT_SS_DFLT, + CWND_INC_PCT_SS_MIN, + CWND_INC_PCT_SS_MAX); + +#define CWND_INC_MIN 1 +#define CWND_INC_MAX (1000) + cc->cwnd_inc = + networkstatus_get_param(NULL, "cc_cwnd_inc", + CWND_INC_DFLT, + CWND_INC_MIN, + CWND_INC_MAX); + +#define CWND_INC_RATE_MIN 1 +#define CWND_INC_RATE_MAX (250) + cc->cwnd_inc_rate = + networkstatus_get_param(NULL, "cc_cwnd_inc_rate", + CWND_INC_RATE_DFLT, + CWND_INC_RATE_MIN, + CWND_INC_RATE_MAX); + +#define SENDME_INC_MIN 10 +#define SENDME_INC_MAX (1000) + cc->sendme_inc = + networkstatus_get_param(NULL, "cc_sendme_inc", + sendme_inc, + SENDME_INC_MIN, + SENDME_INC_MAX); + + // XXX: this min needs to abide by sendme_inc range rules somehow +#define CWND_MIN_MIN sendme_inc +#define CWND_MIN_MAX (1000) + cc->cwnd_min = + networkstatus_get_param(NULL, "cc_cwnd_min", + CWND_MIN_DFLT, + CWND_MIN_MIN, + CWND_MIN_MAX); + +#define EWMA_CWND_COUNT_MIN 1 +#define EWMA_CWND_COUNT_MAX (100) + cc->ewma_cwnd_cnt = + networkstatus_get_param(NULL, "cc_ewma_cwnd_cnt", + EWMA_CWND_COUNT_DFLT, + EWMA_CWND_COUNT_MIN, + EWMA_CWND_COUNT_MAX); + +#define BWE_SENDME_MIN_MIN 2 +#define BWE_SENDME_MIN_MAX (20) + cc->bwe_sendme_min = + networkstatus_get_param(NULL, "cc_bwe_min", + BWE_SENDME_MIN_DFLT, + BWE_SENDME_MIN_MIN, + BWE_SENDME_MIN_MAX); + +#define CC_ALG_MIN 0 +#define CC_ALG_MAX (NUM_CC_ALGS-1) + cc->cc_alg = + networkstatus_get_param(NULL, "cc_alg", + cc_alg, + CC_ALG_MIN, + CC_ALG_MAX); + + bdp_alg_t default_bdp_alg = 0; + + switch (cc->cc_alg) { + case CC_ALG_WESTWOOD: + default_bdp_alg = WESTWOOD_BDP_ALG; + break; + case CC_ALG_VEGAS: + default_bdp_alg = VEGAS_BDP_MIX_ALG; + break; + case CC_ALG_NOLA: + default_bdp_alg = NOLA_BDP_ALG; + break; + case CC_ALG_SENDME: + default: + tor_fragile_assert(); + return; // No alg-specific params + } + + cc->bdp_alg = + networkstatus_get_param(NULL, "cc_bdp_alg", + default_bdp_alg, + 0, + NUM_BDP_ALGS-1); + + /* Algorithm-specific parameters */ + if (cc->cc_alg == CC_ALG_WESTWOOD) { + congestion_control_westwood_set_params(cc); + } else if (cc->cc_alg == CC_ALG_VEGAS) { + congestion_control_vegas_set_params(cc); + } else if (cc->cc_alg == CC_ALG_NOLA) { + congestion_control_nola_set_params(cc); + } +} + +/** + * Allocate and initialize fields in congestion control object. + * + * cc_alg is the negotiated congestion control algorithm. + * + * sendme_inc is the number of packaged cells that a sendme cell + * acks. This parameter will come from circuit negotiation. + */ +static void +congestion_control_init(congestion_control_t *cc, cc_alg_t cc_alg, + int sendme_inc) +{ + cc->sendme_pending_timestamps = smartlist_new(); + cc->sendme_arrival_timestamps = smartlist_new(); + + cc->in_slow_start = 1; + congestion_control_init_params(cc, cc_alg, sendme_inc); + + cc->next_cc_event = CWND_UPDATE_RATE(cc); +} + +/** Allocate and initialize a new congestion control object */ +congestion_control_t * +congestion_control_new(void) +{ + congestion_control_t *cc = tor_malloc_zero(sizeof(congestion_control_t)); + + // XXX: the alg and the sendme_inc need to be negotiated during + // circuit handshake + congestion_control_init(cc, CC_ALG_VEGAS, SENDME_INC_DFLT); + + return cc; +} + +/** + * Free a congestion control object and its asssociated state. + */ +void +congestion_control_free_(congestion_control_t *cc) +{ + if (!cc) + return; + + SMARTLIST_FOREACH(cc->sendme_pending_timestamps, uint64_t *, t, tor_free(t)); + SMARTLIST_FOREACH(cc->sendme_arrival_timestamps, uint64_t *, t, tor_free(t)); + smartlist_free(cc->sendme_pending_timestamps); + smartlist_free(cc->sendme_arrival_timestamps); + + tor_free(cc); +} + +/** + * Enqueue a u64 timestamp to the end of a queue of timestamps. + */ +static inline void +enqueue_timestamp(smartlist_t *timestamps_u64, uint64_t timestamp_usec) +{ + uint64_t *timestamp_ptr = tor_malloc(sizeof(uint64_t)); + *timestamp_ptr = timestamp_usec; + + smartlist_add(timestamps_u64, timestamp_ptr); +} + +/** + * Peek at the head of a smartlist queue of u64 timestamps. + */ +static inline uint64_t +peek_timestamp(const smartlist_t *timestamps_u64_usecs) +{ + uint64_t *timestamp_ptr = smartlist_get(timestamps_u64_usecs, 0); + + if (BUG(!timestamp_ptr)) { + log_err(LD_CIRC, "Congestion control timestamp list became empty!"); + return 0; + } + + return *timestamp_ptr; +} + +/** + * Dequeue a u64 monotime usec timestamp from the front of a + * smartlist of pointers to 64. + */ +static inline uint64_t +dequeue_timestamp(smartlist_t *timestamps_u64_usecs) +{ + uint64_t *timestamp_ptr = smartlist_get(timestamps_u64_usecs, 0); + uint64_t timestamp_u64; + + if (BUG(!timestamp_ptr)) { + log_err(LD_CIRC, "Congestion control timestamp list became empty!"); + return 0; + } + + timestamp_u64 = *timestamp_ptr; + smartlist_del_keeporder(timestamps_u64_usecs, 0); + tor_free(timestamp_ptr); + + return timestamp_u64; +} + +/** + * Returns the number of sendme acks that will be recieved in the + * current congestion window size, rounded to nearest int. + */ +static inline uint64_t +sendme_acks_per_cwnd(const congestion_control_t *cc) +{ + /* We add half a sendme_inc to cwnd to round to the nearest int */ + return ((cc->cwnd + cc->sendme_inc/2)/cc->sendme_inc); +} + +/** + * Get a package window from either old sendme logic, or congestion control. + * + * A package window is how many cells you can still send. + */ +int +congestion_control_get_package_window(const circuit_t *circ, + const crypt_path_t *cpath) +{ + int package_window; + congestion_control_t *cc; + + tor_assert(circ); + + if (cpath) { + package_window = cpath->package_window; + cc = cpath->ccontrol; + } else { + package_window = circ->package_window; + cc = circ->ccontrol; + } + + if (!cc) { + return package_window; + } else { + /* Inflight can be above cwnd if cwnd was just reduced */ + if (cc->inflight > cc->cwnd) + return 0; + /* In the extremely unlikely event that cwnd-inflight is larger than + * INT32_MAX, just return that cap, so old code doesn't explode. */ + else if (cc->cwnd - cc->inflight > INT32_MAX) + return INT32_MAX; + else + return (int)(cc->cwnd - cc->inflight); + } +} + +/** + * Returns the number of cells that are acked by every sendme. + */ +int +sendme_get_inc_count(const circuit_t *circ, const crypt_path_t *layer_hint) +{ + int sendme_inc = CIRCWINDOW_INCREMENT; + congestion_control_t *cc = NULL; + + if (layer_hint) { + cc = layer_hint->ccontrol; + } else { + cc = circ->ccontrol; + } + + if (cc) { + sendme_inc = cc->sendme_inc; + } + + return sendme_inc; +} + +/** Return true iff the next cell we send will result in the other endpoint + * sending a SENDME. + * + * We are able to know that because the package or inflight window value minus + * one cell (the possible SENDME cell) should be a multiple of the + * cells-per-sendme increment value (set via consensus parameter, negotiated + * for the circuit, and passed in as sendme_inc). + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. + */ +bool +circuit_sent_cell_for_sendme(const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + congestion_control_t *cc; + int window; + + tor_assert(circ); + + if (layer_hint) { + window = layer_hint->package_window; + cc = layer_hint->ccontrol; + } else { + window = circ->package_window; + cc = circ->ccontrol; + } + + /* If we are using congestion control and the alg is not + * old-school 'fixed', then use cc->inflight to determine + * when sendmes will be sent */ + if (cc) { + if (!cc->inflight) + return false; + + /* This check must be +1 because this function is called *before* + * inflight is incremented for the sent cell */ + if ((cc->inflight+1) % cc->sendme_inc != 0) + return false; + + return true; + } + + /* At the start of the window, no SENDME will be expected. */ + if (window == CIRCWINDOW_START) { + return false; + } + + /* Are we at the limit of the increment and if not, we don't expect next + * cell is a SENDME. + * + * We test against the window minus 1 because when we are looking if the + * next cell is a SENDME, the window (either package or deliver) hasn't been + * decremented just yet so when this is called, we are currently processing + * the "window - 1" cell. + */ + if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + return false; + } + + /* Next cell is expected to be a SENDME. */ + return true; +} + +/** + * Call-in to tell congestion control code that this circuit sent a cell. + * + * This updates the 'inflight' counter, and if this is a cell that will + * cause the other end to send a SENDME, record the current time in a list + * of pending timestamps, so that we can later compute the circuit RTT when + * the SENDME comes back. */ +void +congestion_control_note_cell_sent(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *cpath) +{ + tor_assert(circ); + tor_assert(cc); + + /* Is this the last cell before a SENDME? The idea is that if the + * package_window reaches a multiple of the increment, after this cell, we + * should expect a SENDME. Note that this function must be called *before* + * we account for the sent cell. */ + if (!circuit_sent_cell_for_sendme(circ, cpath)) { + cc->inflight++; + return; + } + + cc->inflight++; + + /* Record this cell time for RTT computation when SENDME arrives */ + enqueue_timestamp(cc->sendme_pending_timestamps, + monotime_absolute_usec()); +} + +/** + * Returns true if any edge connections are active. + * + * We need to know this so that we can stop computing BDP if the + * edges are not sending on the circuit. + */ +static int +circuit_has_active_streams(const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + const edge_connection_t *streams; + + if (CIRCUIT_IS_ORIGIN(circ)) { + streams = CONST_TO_ORIGIN_CIRCUIT(circ)->p_streams; + } else { + streams = CONST_TO_OR_CIRCUIT(circ)->n_streams; + } + + /* Check linked list of streams */ + for (const edge_connection_t *conn = streams; conn != NULL; + conn = conn->next_stream) { + if (conn->base_.marked_for_close) + continue; + + if (!layer_hint || conn->cpath_layer == layer_hint) { + if (connection_get_inbuf_len(TO_CONN(conn)) > 0) { + log_info(LD_CIRC, "CC: More in edge inbuf..."); + return 1; + } + + /* If we did not reach EOF on this read, there's more */ + if (!TO_CONN(conn)->inbuf_reached_eof) { + log_info(LD_CIRC, "CC: More on edge conn..."); + return 1; + } + + if (TO_CONN(conn)->linked_conn) { + if (connection_get_inbuf_len(TO_CONN(conn)->linked_conn) > 0) { + log_info(LD_CIRC, "CC: More in linked inbuf..."); + return 1; + } + + /* If there is a linked conn, and *it* did not each EOF, + * there's more */ + if (!TO_CONN(conn)->linked_conn->inbuf_reached_eof) { + log_info(LD_CIRC, "CC: More on linked conn..."); + return 1; + } + } + } + } + + return 0; +} + +/** + * Upon receipt of a SENDME, pop the oldest timestamp off the timestamp + * list, and use this to update RTT. + * + * Returns true if circuit estimates were successfully updated, false + * otherwise. + */ +bool +congestion_control_update_circuit_estimates(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + uint64_t now_usec = monotime_absolute_usec(); + + /* Update RTT first, then BDP. BDP needs fresh RTT */ + uint64_t curr_rtt_usec = congestion_control_update_circuit_rtt(cc, now_usec); + return congestion_control_update_circuit_bdp(cc, circ, layer_hint, now_usec, + curr_rtt_usec); +} + +/** + * Returns true if we have enough time data to use heuristics + * to compare RTT to a baseline. + */ +static bool +time_delta_should_use_heuristics(const congestion_control_t *cc) +{ + + /* If we have exited slow start, we should have processed at least + * a cwnd worth of RTTs */ + if (!cc->in_slow_start) { + return true; + } + + /* If we managed to get enough acks to estimate a SENDME BDP, then + * we have enough to estimate clock jumps relative to a baseline, + * too. (This is at least 'cc_bwe_min' acks). */ + if (cc->bdp[BDP_ALG_SENDME_RATE]) { + return true; + } + + /* Not enough data to estimate clock jumps */ + return false; +} + +static bool is_monotime_clock_broken = false; + +/** + * Returns true if the monotime delta is 0, or is significantly + * different than the previous delta. Either case indicates + * that the monotime time source stalled or jumped. + * + * Also caches the clock state in the is_monotime_clock_broken flag, + * so we can also provide a is_monotime_clock_reliable() function, + * used by flow control rate timing. + */ +static bool +time_delta_stalled_or_jumped(const congestion_control_t *cc, + uint64_t old_delta, uint64_t new_delta) +{ +#define DELTA_DISCREPENCY_RATIO_MAX 100 + /* If we have a 0 new_delta, that is definitely a monotime stall */ + if (new_delta == 0) { + static ratelim_t stall_info_limit = RATELIM_INIT(60); + log_fn_ratelim(&stall_info_limit, LOG_INFO, LD_CIRC, + "Congestion control cannot measure RTT due to monotime stall."); + + /* If delta is every 0, the monotime clock has stalled, and we should + * not use it anywhere. */ + is_monotime_clock_broken = true; + + return is_monotime_clock_broken; + } + + /* If the old_delta is 0, we have no previous values on this circuit. + * + * So, return the global monotime status from other circuits, and + * do not update. + */ + if (old_delta == 0) { + return is_monotime_clock_broken; + } + + /* + * For the heuristic cases, we need at least a few timestamps, + * to average out any previous partial stalls or jumps. So until + * than point, let's just use the cached status from other circuits. + */ + if (!time_delta_should_use_heuristics(cc)) { + return is_monotime_clock_broken; + } + + /* If old_delta is significantly larger than new_delta, then + * this means that the monotime clock recently stopped moving + * forward. */ + if (old_delta > new_delta * DELTA_DISCREPENCY_RATIO_MAX) { + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Sudden decrease in circuit RTT (%"PRIu64" vs %"PRIu64 + "), likely due to clock jump.", + new_delta/1000, old_delta/1000); + + is_monotime_clock_broken = true; + + return is_monotime_clock_broken; + } + + /* If new_delta is significantly larger than old_delta, then + * this means that the monotime clock suddenly jumped forward. */ + if (new_delta > old_delta * DELTA_DISCREPENCY_RATIO_MAX) { + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Sudden increase in circuit RTT (%"PRIu64" vs %"PRIu64 + "), likely due to clock jump.", + new_delta/1000, old_delta/1000); + + is_monotime_clock_broken = true; + + return is_monotime_clock_broken; + } + + /* All good! Update cached status, too */ + is_monotime_clock_broken = false; + + return is_monotime_clock_broken; +} + +/** + * Is the monotime clock stalled according to any circuits? + */ +bool +is_monotime_clock_reliable(void) +{ + return !is_monotime_clock_broken; +} + +/** + * Called when we get a SENDME. Updates circuit RTT by pulling off a + * timestamp of when we sent the CIRCWINDOW_INCREMENT-th cell from + * the queue of such timestamps, and comparing that to current time. + * + * Also updates min, max, and EWMA of RTT. + * + * Returns the current circuit RTT in usecs, or 0 if it could not be + * measured (due to clock jump, stall, etc). + */ +static uint64_t +congestion_control_update_circuit_rtt(congestion_control_t *cc, + uint64_t now_usec) +{ + uint64_t rtt, ewma_cnt; + uint64_t sent_at_timestamp; + + tor_assert(cc); + + /* Get the time that we sent the cell that resulted in the other + * end sending this sendme. Use this to calculate RTT */ + sent_at_timestamp = dequeue_timestamp(cc->sendme_pending_timestamps); + + rtt = now_usec - sent_at_timestamp; + + /* Do not update RTT at all if it looks fishy */ + if (time_delta_stalled_or_jumped(cc, cc->ewma_rtt_usec, rtt)) { + return 0; + } + + ewma_cnt = cc->ewma_cwnd_cnt*sendme_acks_per_cwnd(cc); + ewma_cnt = MAX(ewma_cnt, 2); // Use at least 2 + + cc->ewma_rtt_usec = n_count_ewma(rtt, cc->ewma_rtt_usec, ewma_cnt); + + if (rtt > cc->max_rtt_usec) { + cc->max_rtt_usec = rtt; + } + + if (cc->min_rtt_usec == 0 || rtt < cc->min_rtt_usec) { + cc->min_rtt_usec = rtt; + } + + return rtt; +} + +/** + * Called when we get a SENDME. Updates the bandwidth-delay-product (BDP) + * estimates of a circuit. Several methods of computing BDP are used, + * depending on scenario. While some congestion control algorithms only + * use one of these methods, we update them all because it's quick and easy. + * + * - now_usec is the current monotime in usecs. + * - curr_rtt_usec is the current circuit RTT in usecs. It may be 0 if no + * RTT could bemeasured. + * + * Returns true if we were able to update BDP, false otherwise. + */ +static bool +congestion_control_update_circuit_bdp(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint, + uint64_t now_usec, + uint64_t curr_rtt_usec) +{ + int chan_q = 0; + unsigned int blocked_on_chan = 0; + uint64_t timestamp_usec; + uint64_t sendme_rate_bdp = 0; + + tor_assert(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* origin circs use n_chan */ + chan_q = circ->n_chan_cells.n; + blocked_on_chan = circ->streams_blocked_on_n_chan; + } else { + /* Both onion services and exits use or_circuit and p_chan */ + chan_q = CONST_TO_OR_CIRCUIT(circ)->p_chan_cells.n; + blocked_on_chan = circ->streams_blocked_on_p_chan; + } + + /* If we have no EWMA RTT, it is because monotime has been stalled + * or messed up the entire time so far. Set our BDP estimates directly + * to current cwnd */ + if (!cc->ewma_rtt_usec) { + uint64_t cwnd = cc->cwnd; + + /* If the channel is blocked, keep subtracting off the chan_q + * until we hit the min cwnd. */ + if (blocked_on_chan) { + cwnd = MAX(cwnd - chan_q, cc->cwnd_min); + cc->blocked_chan = 1; + } else { + cc->blocked_chan = 0; + } + + cc->bdp[BDP_ALG_CWND_RTT] = cwnd; + cc->bdp[BDP_ALG_INFLIGHT_RTT] = cwnd; + cc->bdp[BDP_ALG_SENDME_RATE] = cwnd; + cc->bdp[BDP_ALG_PIECEWISE] = cwnd; + + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Our clock has been stalled for the entire lifetime of a circuit. " + "Performance may be sub-optimal."); + + return blocked_on_chan; + } + + /* Congestion window based BDP will respond to changes in RTT only, and is + * relative to cwnd growth. It is useful for correcting for BDP + * overestimation, but if BDP is higher than the current cwnd, it will + * underestimate it. + * + * We multiply here first to avoid precision issues from min_RTT being + * close to ewma RTT. Since all fields are u64, there is plenty of + * room here to multiply first. + */ + cc->bdp[BDP_ALG_CWND_RTT] = cc->cwnd*cc->min_rtt_usec/cc->ewma_rtt_usec; + + /* + * If we have no pending streams, we do not have enough data to fill + * the BDP, so preserve our old estimates but do not make any more. + */ + if (!blocked_on_chan && !circuit_has_active_streams(circ, layer_hint)) { + log_info(LD_CIRC, + "CC: Streams drained. Spare package window: %"PRIu64 + ", no BDP update", cc->cwnd - cc->inflight); + + /* Clear SENDME timestamps; they will be wrong with intermittent data */ + SMARTLIST_FOREACH(cc->sendme_arrival_timestamps, uint64_t *, t, + tor_free(t)); + smartlist_clear(cc->sendme_arrival_timestamps); + } else if (curr_rtt_usec && is_monotime_clock_reliable()) { + /* Sendme-based BDP will quickly measure BDP in much less than + * a cwnd worth of data when in use (in 2-10 SENDMEs). + * + * But if the link goes idle, it will be vastly lower than true BDP. Hence + * we only compute it if we have either pending stream data, or streams + * are still blocked on the channel queued data. + * + * We also do not compute it if we do not have a current RTT passed in, + * because that means that monotime is currently stalled or just jumped. + */ + enqueue_timestamp(cc->sendme_arrival_timestamps, now_usec); + + if (smartlist_len(cc->sendme_arrival_timestamps) >= cc->bwe_sendme_min) { + /* If we have more sendmes than fit in a cwnd, trim the list. + * Those are not acurrately measuring throughput, if cwnd is + * currently smaller than BDP */ + while (smartlist_len(cc->sendme_arrival_timestamps) > + cc->bwe_sendme_min && + (uint64_t)smartlist_len(cc->sendme_arrival_timestamps) > + sendme_acks_per_cwnd(cc)) { + (void)dequeue_timestamp(cc->sendme_arrival_timestamps); + } + int sendme_cnt = smartlist_len(cc->sendme_arrival_timestamps); + + /* Calculate SENDME_BWE_COUNT pure average */ + timestamp_usec = peek_timestamp(cc->sendme_arrival_timestamps); + uint64_t delta = now_usec - timestamp_usec; + + /* The acked data is in sendme_cnt-1 chunks, because we are counting the + * data that is processed by the other endpoint *between* all of these + * sendmes. There's one less gap between the sendmes than the number + * of sendmes. */ + uint64_t cells = (sendme_cnt-1)*cc->sendme_inc; + + /* The bandwidth estimate is cells/delta, which when multiplied + * by min RTT obtains the BDP. However, we multiply first to + * avoid precision issues with the RTT being close to delta in size. */ + sendme_rate_bdp = cells*cc->min_rtt_usec/delta; + + /* Calculate BDP_EWMA_COUNT N-EWMA */ + cc->bdp[BDP_ALG_SENDME_RATE] = + n_count_ewma(sendme_rate_bdp, cc->bdp[BDP_ALG_SENDME_RATE], + cc->ewma_cwnd_cnt*sendme_acks_per_cwnd(cc)); + } + + /* In-flight BDP will cause the cwnd to drift down when underutilized. + * It is most useful when the local OR conn is blocked, so we only + * compute it if we're utilized. */ + cc->bdp[BDP_ALG_INFLIGHT_RTT] = + (cc->inflight - chan_q)*cc->min_rtt_usec/ + MAX(cc->ewma_rtt_usec, curr_rtt_usec); + } else { + /* We can still update inflight with just an EWMA RTT, but only + * if there is data flowing */ + cc->bdp[BDP_ALG_INFLIGHT_RTT] = + (cc->inflight - chan_q)*cc->min_rtt_usec/cc->ewma_rtt_usec; + } + + /* The orconn is blocked; use smaller of inflight vs SENDME */ + if (blocked_on_chan) { + log_info(LD_CIRC, "CC: Streams blocked on circ channel. Chanq: %d", + chan_q); + + /* A blocked channel is an immediate congestion signal, but it still + * happens only once per cwnd */ + if (!cc->blocked_chan) { + cc->next_cc_event = 0; + cc->blocked_chan = 1; + } + + if (cc->bdp[BDP_ALG_SENDME_RATE]) { + cc->bdp[BDP_ALG_PIECEWISE] = MIN(cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_SENDME_RATE]); + } else { + cc->bdp[BDP_ALG_PIECEWISE] = cc->bdp[BDP_ALG_INFLIGHT_RTT]; + } + } else { + /* If we were previously blocked, emit a new congestion event + * now that we are unblocked, to re-evaluate cwnd */ + if (cc->blocked_chan) { + cc->blocked_chan = 0; + cc->next_cc_event = 0; + log_info(LD_CIRC, "CC: Streams un-blocked on circ channel. Chanq: %d", + chan_q); + } + + cc->bdp[BDP_ALG_PIECEWISE] = MAX(cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_CWND_RTT]); + } + + /* We can end up with no piecewise value if we didn't have either + * a SENDME estimate or enough data for an inflight estimate. + * It also happens on the very first sendme, since we need two + * to get a BDP. In these cases, use the cwnd method. */ + if (!cc->bdp[BDP_ALG_PIECEWISE]) { + cc->bdp[BDP_ALG_PIECEWISE] = cc->bdp[BDP_ALG_CWND_RTT]; + log_info(LD_CIRC, "CC: No piecewise BDP. Using %"PRIu64, + cc->bdp[BDP_ALG_PIECEWISE]); + } + + if (cc->next_cc_event == 0) { + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: Circuit %d " + "SENDME RTT: %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", " + "BDP estimates: " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64". ", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->min_rtt_usec/1000, + curr_rtt_usec/1000, + cc->ewma_rtt_usec/1000, + cc->max_rtt_usec/1000, + cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_CWND_RTT], + sendme_rate_bdp, + cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_PIECEWISE] + ); + } else { + log_info(LD_CIRC, + "CC: Circuit %"PRIu64":%d " + "SENDME RTT: %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64". ", + // XXX: actually, is this p_chan here? This is + // an or_circuit (exit or onion) + circ->n_chan->global_identifier, circ->n_circ_id, + cc->min_rtt_usec/1000, + curr_rtt_usec/1000, + cc->ewma_rtt_usec/1000, + cc->max_rtt_usec/1000, + cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_CWND_RTT], + sendme_rate_bdp, + cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_PIECEWISE] + ); + } + } + + /* We updated BDP this round if either we had a blocked channel, or + * the curr_rtt_usec was not 0. */ + bool ret = (blocked_on_chan || curr_rtt_usec != 0); + if (ret) { + tor_trace(TR_SUBSYS(cc), TR_EV(bdp_update), circ, cc, curr_rtt_usec, + sendme_rate_bdp); + } + return ret; +} + +/** + * Dispatch the sendme to the appropriate congestion control algorithm. + */ +int +congestion_control_dispatch_cc_alg(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + int ret = -END_CIRC_REASON_INTERNAL; + switch (cc->cc_alg) { + case CC_ALG_WESTWOOD: + ret = congestion_control_westwood_process_sendme(cc, circ, layer_hint); + break; + + case CC_ALG_VEGAS: + ret = congestion_control_vegas_process_sendme(cc, circ, layer_hint); + break; + + case CC_ALG_NOLA: + ret = congestion_control_nola_process_sendme(cc, circ, layer_hint); + break; + + case CC_ALG_SENDME: + default: + tor_assert(0); + } + + if (cc->cwnd > cwnd_max) { + static ratelim_t cwnd_limit = RATELIM_INIT(60); + log_fn_ratelim(&cwnd_limit, LOG_NOTICE, LD_CIRC, + "Congestion control cwnd %"PRIu64" exceeds max %d, clamping.", + cc->cwnd, cwnd_max); + cc->cwnd = cwnd_max; + } + + return ret; +} diff --git a/src/core/or/congestion_control_common.h b/src/core/or/congestion_control_common.h new file mode 100644 index 0000000000..01dbc1ceb4 --- /dev/null +++ b/src/core/or/congestion_control_common.h @@ -0,0 +1,113 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_common.h + * \brief Public APIs for congestion control + **/ + +#ifndef TOR_CONGESTION_CONTROL_COMMON_H +#define TOR_CONGESTION_CONTROL_COMMON_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +typedef struct congestion_control_t congestion_control_t; + +/** Wrapper for the free function, set the CC pointer to NULL after free */ +#define congestion_control_free(cc) \ + FREE_AND_NULL(congestion_control_t, congestion_control_free_, cc) + +void congestion_control_free_(congestion_control_t *cc); + +congestion_control_t *congestion_control_new(void); + +int congestion_control_dispatch_cc_alg(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); + +void congestion_control_note_cell_sent(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *cpath); + +bool congestion_control_update_circuit_estimates(congestion_control_t *, + const circuit_t *, + const crypt_path_t *); + +int congestion_control_get_package_window(const circuit_t *, + const crypt_path_t *); + +int sendme_get_inc_count(const circuit_t *, const crypt_path_t *); +bool circuit_sent_cell_for_sendme(const circuit_t *, const crypt_path_t *); +bool is_monotime_clock_reliable(void); + +void congestion_control_new_consensus_params(const networkstatus_t *ns); + +/* Ugh, C.. these four are private. Use the getter instead, when + * external to the congestion control code. */ +extern uint32_t or_conn_highwater; +extern uint32_t or_conn_lowwater; +extern int32_t cell_queue_high; +extern int32_t cell_queue_low; + +/** Stop writing on an orconn when its outbuf is this large */ +static inline uint32_t +or_conn_highwatermark(void) +{ + return or_conn_highwater; +} + +/** Resume writing on an orconn when its outbuf is less than this */ +static inline uint32_t +or_conn_lowwatermark(void) +{ + return or_conn_lowwater; +} + +/** Stop reading on edge connections when we have this many cells + * waiting on the appropriate queue. */ +static inline int32_t +cell_queue_highwatermark(void) +{ + return cell_queue_high; +} + +/** Start reading from edge connections again when we get down to this many + * cells. */ +static inline int32_t +cell_queue_lowwatermark(void) +{ + return cell_queue_low; +} + +/** + * Compute an N-count EWMA, aka N-EWMA. N-EWMA is defined as: + * EWMA = alpha*value + (1-alpha)*EWMA_prev + * with alpha = 2/(N+1). + * + * This works out to: + * EWMA = value*2/(N+1) + EMA_prev*(N-1)/(N+1) + * = (value*2 + EWMA_prev*(N-1))/(N+1) + */ +static inline uint64_t +n_count_ewma(uint64_t curr, uint64_t prev, uint64_t N) +{ + if (prev == 0) + return curr; + else + return (2*curr + (N-1)*prev)/(N+1); +} + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_COMMON_H) */ diff --git a/src/core/or/congestion_control_flow.c b/src/core/or/congestion_control_flow.c new file mode 100644 index 0000000000..805654664c --- /dev/null +++ b/src/core/or/congestion_control_flow.c @@ -0,0 +1,710 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_flow.c + * \brief Code that implements flow control for congestion controlled + * circuits. + */ + +#define TOR_CONGESTION_CONTROL_FLOW_PRIVATE + +#include "core/or/or.h" + +#include "core/or/relay.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_edge.h" +#include "core/mainloop/mainloop.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" +#include "core/or/congestion_control_st.h" +#include "core/or/circuitlist.h" +#include "core/or/trace_probes_cc.h" +#include "feature/nodelist/networkstatus.h" +#include "trunnel/flow_control_cells.h" + +#include "core/or/connection_st.h" +#include "core/or/cell_st.h" +#include "app/config/config.h" + +/** Cache consensus parameters */ +static uint32_t xoff_client; +static uint32_t xoff_exit; + +static uint32_t xon_change_pct; +static uint32_t xon_ewma_cnt; +static uint32_t xon_rate_bytes; + +/* In normal operation, we can get a burst of up to 32 cells before returning + * to libevent to flush the outbuf. This is a heuristic from hardcoded values + * and strange logic in connection_bucket_get_share(). */ +#define MAX_EXPECTED_CELL_BURST 32 + +/* The following three are for dropmark rate limiting. They define when we + * scale down our XON, XOFF, and xmit byte counts. Early scaling is beneficial + * because it limits the ability of spurious XON/XOFF to be sent after large + * amounts of data without XON/XOFF. At these limits, after 10MB of data (or + * more), an adversary can only inject (log2(10MB)-log2(200*500))*100 ~= 1000 + * cells of fake XOFF/XON before the xmit byte count will be halved enough to + * triggering a limit. */ +#define XON_COUNT_SCALE_AT 200 +#define XOFF_COUNT_SCALE_AT 200 +#define ONE_MEGABYTE (UINT64_C(1) << 20) +#define TOTAL_XMIT_SCALE_AT (10 * ONE_MEGABYTE) + +/** + * Return the congestion control object of the given edge connection. + * + * Returns NULL if the edge connection doesn't have a cpath_layer or not + * attached to a circuit. But also if the cpath_layer or circuit doesn't have a + * congestion control object. + */ +static inline const congestion_control_t * +edge_get_ccontrol(const edge_connection_t *edge) +{ + if (edge->cpath_layer) + return edge->cpath_layer->ccontrol; + else if (edge->on_circuit) + return edge->on_circuit->ccontrol; + else + return NULL; +} + +/** + * Update global congestion control related consensus parameter values, every + * consensus update. + * + * More details for each of the parameters can be found in proposal 324, + * section 6.5 including tuning notes. + */ +void +flow_control_new_consensus_params(const networkstatus_t *ns) +{ +#define CC_XOFF_CLIENT_DFLT 500 +#define CC_XOFF_CLIENT_MIN 1 +#define CC_XOFF_CLIENT_MAX 10000 + xoff_client = networkstatus_get_param(ns, "cc_xoff_client", + CC_XOFF_CLIENT_DFLT, + CC_XOFF_CLIENT_MIN, + CC_XOFF_CLIENT_MAX)*RELAY_PAYLOAD_SIZE; + +#define CC_XOFF_EXIT_DFLT 500 +#define CC_XOFF_EXIT_MIN 1 +#define CC_XOFF_EXIT_MAX 10000 + xoff_exit = networkstatus_get_param(ns, "cc_xoff_exit", + CC_XOFF_EXIT_DFLT, + CC_XOFF_EXIT_MIN, + CC_XOFF_EXIT_MAX)*RELAY_PAYLOAD_SIZE; + +#define CC_XON_CHANGE_PCT_DFLT 25 +#define CC_XON_CHANGE_PCT_MIN 1 +#define CC_XON_CHANGE_PCT_MAX 99 + xon_change_pct = networkstatus_get_param(ns, "cc_xon_change_pct", + CC_XON_CHANGE_PCT_DFLT, + CC_XON_CHANGE_PCT_MIN, + CC_XON_CHANGE_PCT_MAX); + +#define CC_XON_RATE_BYTES_DFLT (500) +#define CC_XON_RATE_BYTES_MIN (1) +#define CC_XON_RATE_BYTES_MAX (5000) + xon_rate_bytes = networkstatus_get_param(ns, "cc_xon_rate", + CC_XON_RATE_BYTES_DFLT, + CC_XON_RATE_BYTES_MIN, + CC_XON_RATE_BYTES_MAX)*RELAY_PAYLOAD_SIZE; + +#define CC_XON_EWMA_CNT_DFLT (2) +#define CC_XON_EWMA_CNT_MIN (1) +#define CC_XON_EWMA_CNT_MAX (100) + xon_ewma_cnt = networkstatus_get_param(ns, "cc_xon_ewma_cnt", + CC_XON_EWMA_CNT_DFLT, + CC_XON_EWMA_CNT_MIN, + CC_XON_EWMA_CNT_MAX); +} + +/** + * Send an XOFF for this stream, and note that we sent one + */ +static void +circuit_send_stream_xoff(edge_connection_t *stream) +{ + xoff_cell_t xoff; + uint8_t payload[CELL_PAYLOAD_SIZE]; + ssize_t xoff_size; + + memset(&xoff, 0, sizeof(xoff)); + memset(payload, 0, sizeof(payload)); + + xoff_cell_set_version(&xoff, 0); + + if ((xoff_size = xoff_cell_encode(payload, CELL_PAYLOAD_SIZE, &xoff)) < 0) { + log_warn(LD_BUG, "Failed to encode xon cell"); + return; + } + + if (connection_edge_send_command(stream, RELAY_COMMAND_XOFF, + (char*)payload, (size_t)xoff_size) == 0) { + stream->xoff_sent = true; + } +} + +/** + * Compute the recent drain rate (write rate) for this edge + * connection and return it, in KB/sec (1000 bytes/sec). + * + * Returns 0 if the monotime clock is busted. + */ +static inline uint32_t +compute_drain_rate(const edge_connection_t *stream) +{ + if (BUG(!is_monotime_clock_reliable())) { + log_warn(LD_BUG, "Computing drain rate with stalled monotime clock"); + return 0; + } + + uint64_t delta = monotime_absolute_usec() - stream->drain_start_usec; + + if (delta == 0) { + log_warn(LD_BUG, "Computing stream drain rate with zero time delta"); + return 0; + } + + /* Overflow checks */ + if (stream->prev_drained_bytes > INT32_MAX/1000 || /* Intermediate */ + stream->prev_drained_bytes/delta > INT32_MAX/1000) { /* full value */ + return INT32_MAX; + } + + /* kb/sec = bytes/usec * 1000 usec/msec * 1000 msec/sec * kb/1000bytes */ + return MAX(1, (uint32_t)(stream->prev_drained_bytes * 1000)/delta); +} + +/** + * Send an XON for this stream, with appropriate advisory rate information. + * + * Reverts the xoff sent status, and stores the rate information we sent, + * in case it changes. + */ +static void +circuit_send_stream_xon(edge_connection_t *stream) +{ + xon_cell_t xon; + uint8_t payload[CELL_PAYLOAD_SIZE]; + ssize_t xon_size; + + memset(&xon, 0, sizeof(xon)); + memset(payload, 0, sizeof(payload)); + + xon_cell_set_version(&xon, 0); + xon_cell_set_kbps_ewma(&xon, stream->ewma_drain_rate); + + if ((xon_size = xon_cell_encode(payload, CELL_PAYLOAD_SIZE, &xon)) < 0) { + log_warn(LD_BUG, "Failed to encode xon cell"); + return; + } + + /* Store the advisory rate information, to send advisory updates if + * it changes */ + stream->ewma_rate_last_sent = stream->ewma_drain_rate; + + if (connection_edge_send_command(stream, RELAY_COMMAND_XON, (char*)payload, + (size_t)xon_size) == 0) { + /* Revert the xoff sent status, so we can send another one if need be */ + stream->xoff_sent = false; + } +} + +/** + * Process a stream XOFF, parsing it, and then stopping reading on + * the edge connection. + * + * Record that we have recieved an xoff, so we know not to resume + * reading on this edge conn until we get an XON. + * + * Returns false if the XOFF did not validate; true if it does. + */ +bool +circuit_process_stream_xoff(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell) +{ + (void)cell; + bool retval = true; + + if (BUG(!conn)) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XOFF on invalid stream?"); + return false; + } + + /* Make sure this XOFF came from the right hop */ + if (layer_hint && layer_hint != conn->cpath_layer) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XOFF from wrong hop."); + return false; + } + + if (edge_get_ccontrol(conn) == NULL) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XOFF for non-congestion control circuit"); + return false; + } + + if (conn->xoff_received) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got multiple XOFF on connection"); + return false; + } + + /* If we are near the max, scale everything down */ + if (conn->num_xoff_recv == XOFF_COUNT_SCALE_AT) { + log_info(LD_EDGE, "Scaling down for XOFF count: %d %d %d", + conn->total_bytes_xmit, + conn->num_xoff_recv, + conn->num_xon_recv); + conn->total_bytes_xmit /= 2; + conn->num_xoff_recv /= 2; + conn->num_xon_recv /= 2; + } + + conn->num_xoff_recv++; + + /* Client-side check to make sure that XOFF is not sent too early, + * for dropmark attacks. The main sidechannel risk is early cells, + * but we also check to make sure that we have not received more XOFFs + * than could have been generated by the bytes we sent. + */ + if (TO_CONN(conn)->type == CONN_TYPE_AP || conn->hs_ident != NULL) { + uint32_t limit = 0; + + /* TODO: This limit technically needs to come from negotiation, + * and be bounds checked for sanity, because the other endpoint + * may have a different consensus */ + if (conn->hs_ident) + limit = xoff_client; + else + limit = xoff_exit; + + if (conn->total_bytes_xmit < limit*conn->num_xoff_recv) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got extra XOFF for bytes sent. Got %d, expected max %d", + conn->num_xoff_recv, conn->total_bytes_xmit/limit); + /* We still process this, because the only dropmark defenses + * in C tor are via the vanguards addon's use of the read valid + * cells. So just signal that we think this is not valid protocol + * data and proceed. */ + retval = false; + } + } + + // TODO: Count how many xoffs we have; log if "too many", for shadow + // analysis of chatter. Possibly add to extra-info? + + log_info(LD_EDGE, "Got XOFF!"); + connection_stop_reading(TO_CONN(conn)); + conn->xoff_received = true; + + return retval; +} + +/** + * Process a stream XON, and if it validates, clear the xoff + * flag and resume reading on this edge connection. + * + * Also, use provided rate information to rate limit + * reading on this edge (or packagaing from it onto + * the circuit), to avoid XON/XOFF chatter. + * + * Returns true if the XON validates, false otherwise. + */ +bool +circuit_process_stream_xon(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell) +{ + xon_cell_t *xon; + bool retval = true; + + if (BUG(!conn)) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XON on invalid stream?"); + return false; + } + + /* Make sure this XON came from the right hop */ + if (layer_hint && layer_hint != conn->cpath_layer) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XON from wrong hop."); + return false; + } + + if (edge_get_ccontrol(conn) == NULL) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XON for non-congestion control circuit"); + return false; + } + + if (xon_cell_parse(&xon, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Received malformed XON cell."); + return false; + } + + /* If we are near the max, scale everything down */ + if (conn->num_xon_recv == XON_COUNT_SCALE_AT) { + log_info(LD_EDGE, "Scaling down for XON count: %d %d %d", + conn->total_bytes_xmit, + conn->num_xoff_recv, + conn->num_xon_recv); + conn->total_bytes_xmit /= 2; + conn->num_xoff_recv /= 2; + conn->num_xon_recv /= 2; + } + + conn->num_xon_recv++; + + /* Client-side check to make sure that XON is not sent too early, + * for dropmark attacks. The main sidechannel risk is early cells, + * but we also check to see that we did not get more XONs than make + * sense for the number of bytes we sent. + */ + if (TO_CONN(conn)->type == CONN_TYPE_AP || conn->hs_ident != NULL) { + uint32_t limit = 0; + + /* TODO: This limit technically needs to come from negotiation, + * and be bounds checked for sanity, because the other endpoint + * may have a different consensus */ + if (conn->hs_ident) + limit = MIN(xoff_client, xon_rate_bytes); + else + limit = MIN(xoff_exit, xon_rate_bytes); + + if (conn->total_bytes_xmit < limit*conn->num_xon_recv) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got extra XON for bytes sent. Got %d, expected max %d", + conn->num_xon_recv, conn->total_bytes_xmit/limit); + + /* We still process this, because the only dropmark defenses + * in C tor are via the vanguards addon's use of the read valid + * cells. So just signal that we think this is not valid protocol + * data and proceed. */ + retval = false; + } + } + + log_info(LD_EDGE, "Got XON: %d", xon->kbps_ewma); + + /* Adjust the token bucket of this edge connection with the drain rate in + * the XON. Rate is in bytes from kilobit (kpbs). */ + uint64_t rate = ((uint64_t) xon_cell_get_kbps_ewma(xon) * 1000); + if (rate == 0 || INT32_MAX < rate) { + /* No rate. */ + rate = INT32_MAX; + } + token_bucket_rw_adjust(&conn->bucket, (uint32_t) rate, (uint32_t) rate); + + if (conn->xoff_received) { + /* Clear the fact that we got an XOFF, so that this edge can + * start and stop reading normally */ + conn->xoff_received = false; + connection_start_reading(TO_CONN(conn)); + } + + xon_cell_free(xon); + + return retval; +} + +/** + * Called from sendme_stream_data_received(), when data arrives + * from a circuit to our edge's outbuf, to decide if we need to send + * an XOFF. + * + * Returns the amount of cells remaining until the buffer is full, at + * which point it sends an XOFF, and returns 0. + * + * Returns less than 0 if we have queued more than a congestion window + * worth of data and need to close the circuit. + */ +int +flow_control_decide_xoff(edge_connection_t *stream) +{ + size_t total_buffered = connection_get_outbuf_len(TO_CONN(stream)); + uint32_t buffer_limit_xoff = 0; + + if (BUG(edge_get_ccontrol(stream) == NULL)) { + log_err(LD_BUG, "Flow control called for non-congestion control circuit"); + return -1; + } + + /* Onion services and clients are typically localhost edges, so they + * need different buffering limits than exits do */ + if (TO_CONN(stream)->type == CONN_TYPE_AP || stream->hs_ident != NULL) { + buffer_limit_xoff = xoff_client; + } else { + buffer_limit_xoff = xoff_exit; + } + + if (total_buffered > buffer_limit_xoff) { + if (!stream->xoff_sent) { + log_info(LD_EDGE, "Sending XOFF: %"TOR_PRIuSZ" %d", + total_buffered, buffer_limit_xoff); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xoff_sending), stream); + + circuit_send_stream_xoff(stream); + + /* Clear the drain rate. It is considered wrong if we + * got all the way to XOFF */ + stream->ewma_drain_rate = 0; + } + } + + /* If the outbuf has accumulated more than the expected burst limit of + * cells, then assume it is not draining, and call decide_xon. We must + * do this because writes only happen when the socket unblocks, so + * may not otherwise notice accumulation of data in the outbuf for + * advisory XONs. */ + if (total_buffered > MAX_EXPECTED_CELL_BURST*RELAY_PAYLOAD_SIZE) { + flow_control_decide_xon(stream, 0); + } + + /* Flow control always takes more data; we rely on the oomkiller to + * handle misbehavior. */ + return 0; +} + +/** + * Returns true if the stream's drain rate has changed significantly. + * + * Returns false if the monotime clock is stalled, or if we have + * no previous drain rate information. + */ +static bool +stream_drain_rate_changed(const edge_connection_t *stream) +{ + if (!is_monotime_clock_reliable()) { + return false; + } + + if (!stream->ewma_rate_last_sent) { + return false; + } + + if (stream->ewma_drain_rate > + (100+(uint64_t)xon_change_pct)*stream->ewma_rate_last_sent/100) { + return true; + } + + if (stream->ewma_drain_rate < + (100-(uint64_t)xon_change_pct)*stream->ewma_rate_last_sent/100) { + return true; + } + + return false; +} + +/** + * Called whenever we drain an edge connection outbuf by writing on + * its socket, to decide if it is time to send an xon. + * + * The n_written parameter tells us how many bytes we have written + * this time, which is used to compute the advisory drain rate fields. + */ +void +flow_control_decide_xon(edge_connection_t *stream, size_t n_written) +{ + size_t total_buffered = connection_get_outbuf_len(TO_CONN(stream)); + + /* Bounds check the number of drained bytes, and scale */ + if (stream->drained_bytes >= UINT32_MAX - n_written) { + /* Cut the bytes in half, and move the start time up halfway to now + * (if we have one). */ + stream->drained_bytes /= 2; + + if (stream->drain_start_usec) { + uint64_t now = monotime_absolute_usec(); + + stream->drain_start_usec = now - (now-stream->drain_start_usec)/2; + } + } + + /* Accumulate drained bytes since last rate computation */ + stream->drained_bytes += n_written; + + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon), stream, n_written); + + /* Check for bad monotime clock and bytecount wrap */ + if (!is_monotime_clock_reliable()) { + /* If the monotime clock ever goes wrong, the safest thing to do + * is just clear our short-term rate info and wait for the clock to + * become reliable again.. */ + stream->drain_start_usec = 0; + stream->drained_bytes = 0; + } else { + /* If we have no drain start timestamp, and we still have + * remaining buffer, start the buffering counter */ + if (!stream->drain_start_usec && total_buffered > 0) { + log_debug(LD_EDGE, "Began edge buffering: %d %d %"TOR_PRIuSZ, + stream->ewma_rate_last_sent, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_drain_start), + stream); + stream->drain_start_usec = monotime_absolute_usec(); + stream->drained_bytes = 0; + } + } + + if (stream->drain_start_usec) { + /* If we have spent enough time in a queued state, update our drain + * rate. */ + if (stream->drained_bytes > xon_rate_bytes) { + /* No previous drained bytes means it is the first time we are computing + * it so use the value we just drained onto the socket as a baseline. It + * won't be accurate but it will be a start towards the right value. + * + * We have to do this in order to have a drain rate else we could be + * sending a drain rate of 0 in an XON which would be undesirable and + * basically like sending an XOFF. */ + if (stream->prev_drained_bytes == 0) { + stream->prev_drained_bytes = stream->drained_bytes; + } + uint32_t drain_rate = compute_drain_rate(stream); + /* Once the drain rate has been computed, note how many bytes we just + * drained so it can be used at the next calculation. We do this here + * because it gets reset once the rate is changed. */ + stream->prev_drained_bytes = stream->drained_bytes; + + if (drain_rate) { + stream->ewma_drain_rate = + (uint32_t)n_count_ewma(drain_rate, + stream->ewma_drain_rate, + xon_ewma_cnt); + log_debug(LD_EDGE, "Updating drain rate: %d %d %"TOR_PRIuSZ, + drain_rate, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_drain_update), + stream, drain_rate); + /* Reset recent byte counts. This prevents us from sending advisory + * XONs more frequent than every xon_rate_bytes. */ + stream->drained_bytes = 0; + stream->drain_start_usec = 0; + } + } + } + + /* If we don't have an XOFF outstanding, consider updating an + * old rate */ + if (!stream->xoff_sent) { + if (stream_drain_rate_changed(stream)) { + /* If we are still buffering and the rate changed, update + * advisory XON */ + log_info(LD_EDGE, "Sending rate-change XON: %d %d %"TOR_PRIuSZ, + stream->ewma_rate_last_sent, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_rate_change), stream); + circuit_send_stream_xon(stream); + } + } else if (total_buffered == 0) { + log_info(LD_EDGE, "Sending XON: %d %d %"TOR_PRIuSZ, + stream->ewma_rate_last_sent, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_partial_drain), stream); + circuit_send_stream_xon(stream); + } + + /* If the buffer has fully emptied, clear the drain timestamp, + * so we can total only bytes drained while outbuf is 0. */ + if (total_buffered == 0) { + stream->drain_start_usec = 0; + + /* After we've spent 'xon_rate_bytes' with the queue fully drained, + * double any rate we sent. */ + if (stream->drained_bytes >= xon_rate_bytes && + stream->ewma_rate_last_sent) { + stream->ewma_drain_rate = MIN(INT32_MAX, 2*stream->ewma_drain_rate); + + log_debug(LD_EDGE, + "Queue empty for xon_rate_limit bytes: %d %d", + stream->ewma_rate_last_sent, + stream->ewma_drain_rate); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_drain_doubled), stream); + /* Resetting the drained bytes count. We need to keep its value as a + * previous so the drain rate calculation takes into account what was + * actually drain the last time. */ + stream->prev_drained_bytes = stream->drained_bytes; + stream->drained_bytes = 0; + } + } + + return; +} + +/** + * Note that we packaged some data on this stream. Used to enforce + * client-side dropmark limits + */ +void +flow_control_note_sent_data(edge_connection_t *stream, size_t len) +{ + /* If we are near the max, scale everything down */ + if (stream->total_bytes_xmit >= TOTAL_XMIT_SCALE_AT-len) { + log_info(LD_EDGE, "Scaling down for flow control xmit bytes:: %d %d %d", + stream->total_bytes_xmit, + stream->num_xoff_recv, + stream->num_xon_recv); + + stream->total_bytes_xmit /= 2; + stream->num_xoff_recv /= 2; + stream->num_xon_recv /= 2; + } + + stream->total_bytes_xmit += len; +} + +/** Returns true if an edge connection uses flow control */ +bool +edge_uses_flow_control(const edge_connection_t *stream) +{ + bool ret = (stream->on_circuit && stream->on_circuit->ccontrol) || + (stream->cpath_layer && stream->cpath_layer->ccontrol); + + /* All circuits with congestion control use flow control */ + return ret; +} + +/** + * Returns the max RTT for the circuit that carries this stream, + * as observed by congestion control. + */ +uint64_t +edge_get_max_rtt(const edge_connection_t *stream) +{ + if (stream->on_circuit && stream->on_circuit->ccontrol) + return stream->on_circuit->ccontrol->max_rtt_usec; + else if (stream->cpath_layer && stream->cpath_layer->ccontrol) + return stream->cpath_layer->ccontrol->max_rtt_usec; + + return 0; +} + +/** Returns true if a connection is an edge conn that uses flow control */ +bool +conn_uses_flow_control(connection_t *conn) +{ + bool ret = false; + + if (CONN_IS_EDGE(conn)) { + edge_connection_t *edge = TO_EDGE_CONN(conn); + + if (edge_uses_flow_control(edge)) { + ret = true; + } + } + + return ret; +} + diff --git a/src/core/or/congestion_control_flow.h b/src/core/or/congestion_control_flow.h new file mode 100644 index 0000000000..6c318027ea --- /dev/null +++ b/src/core/or/congestion_control_flow.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_flow.h + * \brief APIs for stream flow control on congestion controlled circuits. + **/ + +#ifndef TOR_CONGESTION_CONTROL_FLOW_H +#define TOR_CONGESTION_CONTROL_FLOW_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" +#include "core/or/edge_connection_st.h" + +void flow_control_new_consensus_params(const struct networkstatus_t *); + +bool circuit_process_stream_xoff(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell); +bool circuit_process_stream_xon(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell); + +int flow_control_decide_xoff(edge_connection_t *stream); +void flow_control_decide_xon(edge_connection_t *stream, size_t n_written); + +void flow_control_note_sent_data(edge_connection_t *stream, size_t len); + +bool edge_uses_flow_control(const edge_connection_t *stream); + +bool conn_uses_flow_control(connection_t *stream); + +uint64_t edge_get_max_rtt(const edge_connection_t *); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_FLOW_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_FLOW_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_FLOW_H) */ diff --git a/src/core/or/congestion_control_nola.c b/src/core/or/congestion_control_nola.c new file mode 100644 index 0000000000..09f88d4699 --- /dev/null +++ b/src/core/or/congestion_control_nola.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_nola.c + * \brief Code that implements the TOR_NOLA congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_NOLA_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_nola.h" +#include "core/or/circuituse.h" +#include "core/or/circuitlist.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define NOLA_BDP_OVERSHOOT 100 + +/** + * Cache NOLA consensus parameters. + */ +void +congestion_control_nola_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_NOLA); + + cc->nola_params.bdp_overshoot = + networkstatus_get_param(NULL, "cc_nola_overshoot", + NOLA_BDP_OVERSHOOT, + 0, + 1000); +} + +/** +* Process a SENDME and update the congestion window according to the +* rules specified in TOR_NOLA of Proposal #324. +* +* TOR_NOLA updates the congestion window to match the current +* BDP estimate, every sendme. Because this can result in downward +* drift, a fixed overhead is added to the BDP estimate. This will +* cause some queuing, but ensures that the algorithm always uses +* the full BDP. +* +* To handle the case where the local orconn blocks, TOR_NOLA uses +* the 'piecewise' BDP estimate, which uses more a conservative BDP +* estimate method when blocking occurrs, but a more aggressive BDP +* estimate when there is no local blocking. This minimizes local +* client queues. +*/ +int +congestion_control_nola_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + tor_assert(cc && cc->cc_alg == CC_ALG_NOLA); + tor_assert(circ); + + if (cc->next_cc_event) + cc->next_cc_event--; + + /* If we get a congestion event, the only thing NOLA + * does is note this as if we exited slow-start + * (which for NOLA just means we finished our ICW). */ + if (cc->next_cc_event == 0) + cc->in_slow_start = 0; + + /* If we did not successfully update BDP, we must return. Otherwise, + * NOLA can drift downwards */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We overshoot the BDP by the cwnd_inc param amount, because BDP + * may otherwise drift down. This helps us probe for more capacity. + * But there is no sense to do it if the local channel is blocked. */ + if (cc->blocked_chan) + cc->cwnd = cc->bdp[cc->bdp_alg]; + else + cc->cwnd = cc->bdp[cc->bdp_alg] + cc->nola_params.bdp_overshoot; + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC TOR_NOLA: Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC TOR_NOLA: Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->in_slow_start + ); + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_nola.h b/src/core/or/congestion_control_nola.h new file mode 100644 index 0000000000..9c7d6e0635 --- /dev/null +++ b/src/core/or/congestion_control_nola.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_nola.h + * \brief Private-ish APIs for the TOR_NOLA congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_NOLA_H +#define TOR_CONGESTION_CONTROL_NOLA_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_nola_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_nola_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_NOLA_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_NOLA_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_NOLA_H) */ diff --git a/src/core/or/congestion_control_st.h b/src/core/or/congestion_control_st.h new file mode 100644 index 0000000000..251ebd82e3 --- /dev/null +++ b/src/core/or/congestion_control_st.h @@ -0,0 +1,257 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_st.h + * \brief Structure definitions for congestion control. + **/ + +#ifndef CONGESTION_CONTROL_ST_H +#define CONGESTION_CONTROL_ST_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/** Signifies which sendme algorithm to use */ +typedef enum { + /** OG Tor fixed-sized circ and stream windows. It sucks, but it is important + * to make sure that the new algs can compete with the old garbage. */ + CC_ALG_SENDME = 0, + + /** + * Prop#324 TOR_WESTWOOD - Deliberately agressive. Westwood may not even + * converge to fairness in some cases because max RTT will also increase + * on congesgtion, which boosts the Westwood RTT congestion threshhold. So it + * can cause runaway queue bloat, which may or may not lead to a robot + * uprising... Ok that's Westworld, not Westwood. Still, we need to test + * Vegas and NOLA against something more agressive to ensure they do not + * starve in the presence of cheaters. We also need to make sure cheaters + * trigger the oomkiller in those cases. + */ + CC_ALG_WESTWOOD = 1, + + /** + * Prop#324 TOR_VEGAS - TCP Vegas-style BDP tracker. Because Vegas backs off + * whenever it detects queue delay, it can be beaten out by more agressive + * algs. However, in live network testing, it seems to do just fine against + * current SENDMEs. It outperforms Westwood and does not stall. */ + CC_ALG_VEGAS = 2, + + /** + * Prop#324: TOR_NOLA - NOLA looks the BDP right in the eye and uses it + * immediately as CWND. No slow start, no other congestion signals, no delay, + * no bullshit. Like TOR_VEGAS, it also uses agressive BDP estimates, to + * avoid out-competition. It seems a bit better throughput than Vegas, + * but its agressive BDP and rapid updates may lead to more queue latency. */ + CC_ALG_NOLA = 3, +} cc_alg_t; + +/* Total number of CC algs in cc_alg_t enum */ +#define NUM_CC_ALGS (CC_ALG_NOLA+1) + +/** Signifies how we estimate circuit BDP */ +typedef enum { + /* CWND-based BDP will respond to changes in RTT only, and is relative + * to cwnd growth. So in slow-start, this will under-estimate BDP */ + BDP_ALG_CWND_RTT = 0, + + /* Sendme-based BDP will quickly measure BDP in less than + * a cwnd worth of data when in use. So it should be good for slow-start. + * But if the link goes idle, it will be vastly lower than true BDP. Thus, + * this estimate gets reset when the cwnd is not fully utilized. */ + BDP_ALG_SENDME_RATE = 1, + + /* Inflight BDP is similar to the cwnd estimator, except it uses + * packets inflight minus local circuit queues instead of current cwnd. + * Because it is strictly less than or equal to the cwnd, it will cause + * the cwnd to drift downward. It is only used if the local OR connection + * is blocked. */ + BDP_ALG_INFLIGHT_RTT = 2, + + /* The Piecewise BDP estimator uses the CWND estimator before there + * are sufficient SENDMEs to calculate the SENDME estimator. At that + * point, it uses the SENDME estimator, unless the local OR connection + * becomes blocked. In that case, it switches to the inflight estimator. */ + BDP_ALG_PIECEWISE = 3, + +} bdp_alg_t; + +/** Total number of BDP algs in bdp_alg_t enum */ +#define NUM_BDP_ALGS (BDP_ALG_PIECEWISE+1) + +/** Westwood algorithm parameters */ +struct westwood_params_t { + /** Cwnd backoff multiplier upon congestion (as percent) */ + uint8_t cwnd_backoff_m; + /** Max RTT backoff multiplier upon congestion (as percent) */ + uint8_t rtt_backoff_m; + + /** Threshold between min and max RTT, to signal congestion (percent) */ + uint8_t rtt_thresh; + + /** + * If true, use minimum of BDP and backoff multiplication in backoff. + * If false, use maximum of BDP and backoff multiplication in backoff. */ + bool min_backoff; +}; + +/** Vegas algorithm parameters. */ +struct vegas_params_t { + /** The queue use allowed before we exit slow start */ + uint16_t gamma; + /** The queue use below which we increment cwnd */ + uint16_t alpha; + /** The queue use above which we decrement cwnd */ + uint16_t beta; + /** Weighted average (percent) between cwnd estimator and + * piecewise estimator. */ + uint8_t bdp_mix_pct; +}; + +/** NOLA consensus params */ +struct nola_params_t { + /** How many cells to add to BDP estimate to obtain cwnd */ + uint16_t bdp_overshoot; +}; + +/** Fields common to all congestion control algorithms */ +typedef struct congestion_control_t { + /** + * Smartlist of uint64_t monotime usec timestamps of when we sent a data + * cell that is pending a sendme. FIFO queue that is managed similar to + * sendme_last_digests. */ + smartlist_t *sendme_pending_timestamps; + + /** + * Smartlist of uint64_t monotime timestamp of when sendme's arrived. + * FIFO queue that is managed similar to sendme_last_digests. + * Used to estimate circuitbandwidth and BDP. */ + smartlist_t *sendme_arrival_timestamps; + + /** RTT time data for congestion control. */ + uint64_t ewma_rtt_usec; + uint64_t min_rtt_usec; + uint64_t max_rtt_usec; + + /* BDP estimates by algorithm */ + uint64_t bdp[NUM_BDP_ALGS]; + + /** Congestion window */ + uint64_t cwnd; + + /** Number of cells in-flight (sent but awaiting SENDME ack). */ + uint64_t inflight; + + /** + * For steady-state: the number of sendme acks until we will acknowledge + * a congestion event again. It starts out as the number of sendme acks + * in a congestion windowm and is decremented each ack. When this reaches + * 0, it means we should examine our congestion algorithm conditions. + * In this way, we only react to one congestion event per congestion window. + * + * It is also reset to 0 immediately whenever the circuit's orconn is + * blocked, and when a previously blocked orconn is unblocked. + */ + uint64_t next_cc_event; + + /** Are we in slow start? */ + bool in_slow_start; + + /** Is the local channel blocked on us? That's a congestion signal */ + bool blocked_chan; + + /* The following parameters are cached from consensus values upon + * circuit setup. */ + + /** Percent of cwnd to increment by during slow start */ + uint16_t cwnd_inc_pct_ss; + + /** Number of cells to increment cwnd by during steady state */ + uint16_t cwnd_inc; + + /** Minimum congestion window (must be at least sendme_inc) */ + uint16_t cwnd_min; + + /** + * Number of times per congestion window to update based on congestion + * signals */ + uint8_t cwnd_inc_rate; + + /** + * Number of cwnd worth of sendme acks to smooth RTT and BDP with, + * using N_EWMA */ + uint8_t ewma_cwnd_cnt; + + /** + * Minimum number of sendmes before we begin BDP estimates + */ + uint8_t bwe_sendme_min; + + /** + * Number of cells to ack with every sendme. Taken from consensus parameter + * and negotiation during circuit setup. */ + uint8_t sendme_inc; + + /** Which congestion control algorithm to use. Taken from + * consensus parameter and negotiation during circuit setup. */ + cc_alg_t cc_alg; + + /** Which algorithm to estimate circuit bandwidth with. Taken from + * consensus parameter during circuit setup. */ + bdp_alg_t bdp_alg; + + /** Algorithm-specific parameters. The specific struct that is used + * depends upon the algoritghm selected by the cc_alg parameter. + * These should not be accessed anywhere other than the algorithm-specific + * files. */ + union { + struct westwood_params_t westwood_params; + struct vegas_params_t vegas_params; + struct nola_params_t nola_params; + }; +} congestion_control_t; + +/** + * Returns the number of sendme acks we will recieve before we update cwnd. + * + * Congestion control literature recommends only one update of cwnd per + * cwnd worth of acks. However, we can also tune this to be more frequent + * by increasing the 'cc_cwnd_inc_rate' consensus parameter. + * + * If this returns 0 due to high cwnd_inc_rate, the calling code will + * update every sendme ack. + */ +static inline uint64_t CWND_UPDATE_RATE(const congestion_control_t *cc) +{ + /* We add cwnd_inc_rate*sendme_inc/2 to round to nearest integer number + * of acks */ + return ((cc->cwnd + cc->cwnd_inc_rate*cc->sendme_inc/2) + / (cc->cwnd_inc_rate*cc->sendme_inc)); +} + +/** + * Returns the amount to increment the congestion window each update, + * during slow start. + * + * Congestion control literature recommends either doubling the cwnd + * every cwnd during slow start, or some similar exponential growth + * (such as 50% more every cwnd, for Vegas). + * + * This is controlled by a consensus parameter 'cwnd_inc_pct_ss', which + * allows us to specify the percent of the current consensus window + * to update by. + */ +static inline uint64_t CWND_INC_SS(const congestion_control_t *cc) +{ + return (cc->cwnd_inc_pct_ss*cc->cwnd/100); +} + +/** + * Returns the amount to increment (and for Vegas, also decrement) the + * congestion window by, every update period. + * + * This is controlled by the cc_cwnd_inc consensus parameter. + */ +#define CWND_INC(cc) ((cc)->cwnd_inc) + +#endif /* !defined(CONGESTION_CONTROL_ST_H) */ diff --git a/src/core/or/congestion_control_vegas.c b/src/core/or/congestion_control_vegas.c new file mode 100644 index 0000000000..3206821f4c --- /dev/null +++ b/src/core/or/congestion_control_vegas.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_vegas.c + * \brief Code that implements the TOR_VEGAS congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_VEGAS_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_vegas.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define VEGAS_GAMMA(cc) (6*(cc)->sendme_inc) +#define VEGAS_ALPHA(cc) (3*(cc)->sendme_inc) +#define VEGAS_BETA(cc) (6*(cc)->sendme_inc) + +#define VEGAS_BDP_MIX_PCT 0 + +/** + * The original TCP Vegas used only a congestion window BDP estimator. We + * believe that the piecewise estimator is likely to perform better, but + * for purposes of experimentation, we might as well have a way to blend + * them. It also lets us set Vegas to its original estimator while other + * algorithms on the same network use piecewise (by setting the + * 'vegas_bdp_mix_pct' consensus parameter to 100, while leaving the + * 'cc_bdp_alg' parameter set to piecewise). + * + * Returns a percentage weighted average between the CWND estimator and + * the specified consensus BDP estimator. + */ +static inline uint64_t +vegas_bdp_mix(const congestion_control_t *cc) +{ + return cc->vegas_params.bdp_mix_pct*cc->bdp[BDP_ALG_CWND_RTT]/100 + + (100-cc->vegas_params.bdp_mix_pct)*cc->bdp[cc->bdp_alg]/100; +} + +/** + * Cache Vegas consensus parameters. + */ +void +congestion_control_vegas_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_VEGAS); + + cc->vegas_params.gamma = + networkstatus_get_param(NULL, "cc_vegas_gamma", + VEGAS_GAMMA(cc), + 0, + 1000); + + cc->vegas_params.alpha = + networkstatus_get_param(NULL, "cc_vegas_alpha", + VEGAS_ALPHA(cc), + 0, + 1000); + + cc->vegas_params.beta = + networkstatus_get_param(NULL, "cc_vegas_beta", + VEGAS_BETA(cc), + 0, + 1000); + + cc->vegas_params.bdp_mix_pct = + networkstatus_get_param(NULL, "cc_vegas_bdp_mix", + VEGAS_BDP_MIX_PCT, + 0, + 100); +} + +/** + * Process a SENDME and update the congestion window according to the + * rules specified in TOR_VEGAS of Proposal #324. + * + * Essentially, this algorithm attempts to measure queue lengths on + * the circuit by subtracting the bandwidth-delay-product estimate + * from the current congestion window. + * + * If the congestion window is larger than the bandwidth-delay-product, + * then data is assumed to be queuing. We reduce the congestion window + * in that case. + * + * If the congestion window is smaller than the bandwidth-delay-product, + * then there is spare bandwidth capacity on the circuit. We increase the + * congestion window in that case. + * + * The congestion window is updated only once every congestion window worth of + * packets, even if the signal persists. It is also updated whenever the + * upstream orcon blocks, or unblocks. This minimizes local client queues. + */ +int +congestion_control_vegas_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + uint64_t queue_use; + + tor_assert(cc && cc->cc_alg == CC_ALG_VEGAS); + tor_assert(circ); + + /* Update ack counter until next congestion signal event is allowed */ + if (cc->next_cc_event) + cc->next_cc_event--; + + /* Compute BDP and RTT. If we did not update, don't run the alg */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We only update anything once per window */ + if (cc->next_cc_event == 0) { + /* The queue use is the amount in which our cwnd is above BDP; + * if it is below, then 0 queue use. */ + if (vegas_bdp_mix(cc) > cc->cwnd) + queue_use = 0; + else + queue_use = cc->cwnd - vegas_bdp_mix(cc); + + if (cc->in_slow_start) { + if (queue_use < cc->vegas_params.gamma && !cc->blocked_chan) { + /* Grow to BDP immediately, then exponential growth until + * congestion signal */ + cc->cwnd = MAX(cc->cwnd + CWND_INC_SS(cc), + vegas_bdp_mix(cc)); + } else { + /* Congestion signal: Fall back to Vegas equilibrium (BDP) */ + cc->cwnd = vegas_bdp_mix(cc); + cc->in_slow_start = 0; + log_info(LD_CIRC, "CC: TOR_VEGAS exiting slow start"); + } + } else { + if (queue_use > cc->vegas_params.beta || cc->blocked_chan) { + cc->cwnd -= CWND_INC(cc); + } else if (queue_use < cc->vegas_params.alpha) { + cc->cwnd += CWND_INC(cc); + } + } + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + /* Schedule next update */ + cc->next_cc_event = CWND_UPDATE_RATE(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: TOR_VEGAS Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "VBDP: %"PRIu64", " + "QUSE: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + vegas_bdp_mix(cc), + queue_use, + cc->next_cc_event, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC: TOR_VEGAS Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "VBDP: %"PRIu64", " + "QUSE: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + vegas_bdp_mix(cc), + queue_use, + cc->next_cc_event, + cc->in_slow_start + ); + } + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_vegas.h b/src/core/or/congestion_control_vegas.h new file mode 100644 index 0000000000..111345081c --- /dev/null +++ b/src/core/or/congestion_control_vegas.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_vegas.h + * \brief Private-ish APIs for the TOR_VEGAS congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_VEGAS_H +#define TOR_CONGESTION_CONTROL_VEGAS_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_vegas_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_vegas_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_VEGAS_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_VEGAS_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_VEGAS_H) */ diff --git a/src/core/or/congestion_control_westwood.c b/src/core/or/congestion_control_westwood.c new file mode 100644 index 0000000000..4b24234212 --- /dev/null +++ b/src/core/or/congestion_control_westwood.c @@ -0,0 +1,231 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_westwood.c + * \brief Code that implements the TOR_WESTWOOD congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_westwood.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define USEC_ONE_MS (1000) + +#define WESTWOOD_CWND_BACKOFF_M 75 +#define WESTWOOD_RTT_BACKOFF_M 100 +#define WESTWOOD_RTT_THRESH 33 +#define WESTWOOD_MIN_BACKOFF 0 + +/** + * Cache westwood consensus parameters. + */ +void +congestion_control_westwood_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_WESTWOOD); + + cc->westwood_params.cwnd_backoff_m = + networkstatus_get_param(NULL, "cc_westwood_cwnd_m", + WESTWOOD_CWND_BACKOFF_M, + 0, + 100); + + cc->westwood_params.rtt_backoff_m = + networkstatus_get_param(NULL, "cc_westwood_rtt_m", + WESTWOOD_RTT_BACKOFF_M, + 50, + 100); + + cc->westwood_params.rtt_thresh = + networkstatus_get_param(NULL, "cc_westwood_rtt_thresh", + WESTWOOD_RTT_THRESH, + 0, + 100); + + cc->westwood_params.min_backoff = + networkstatus_get_param(NULL, "cc_westwood_min_backoff", + WESTWOOD_MIN_BACKOFF, + 0, + 1); +} + +/** + * Return the RTT threshhold that signals congestion. + * + * Computed from the threshold parameter that specifies a + * percent between the min and max RTT obseved so far. + */ +static inline uint64_t +westwood_rtt_signal(const congestion_control_t *cc) +{ + return ((100 - cc->westwood_params.rtt_thresh)*cc->min_rtt_usec + + cc->westwood_params.rtt_thresh*(cc)->max_rtt_usec)/100; +} + +/** + * Compute a backoff to reduce the max RTT. + * + * This may be necessary to ensure that westwood does not have + * a runaway condition where congestion inflates the max RTT, which + * inflates the congestion threshold. That cannot happen with one + * Westwood instance, but it may happen in aggregate. Hence, this is + * a safety parameter, in case we need it. + */ +static inline uint64_t +westwood_rtt_max_backoff(const congestion_control_t *cc) +{ + return cc->min_rtt_usec + + (cc->westwood_params.rtt_backoff_m * + (cc->max_rtt_usec - cc->min_rtt_usec))/100; +} + +/** + * Returns true if the circuit is experiencing congestion, as per + * TOR_WESTWOOD rules. + */ +static inline bool +westwood_is_congested(const congestion_control_t *cc) +{ + /* If the local channel is blocked, that is always congestion */ + if (cc->blocked_chan) + return true; + + /* If the min RTT is within 1ms of the signal, then there is not enough + * range in RTTs to signify congestion. Treat that as not congested. */ + if (westwood_rtt_signal(cc) < cc->min_rtt_usec || + westwood_rtt_signal(cc) - cc->min_rtt_usec < USEC_ONE_MS) + return false; + + /* If the EWMA-smoothed RTT exceeds the westwood RTT threshhold, + * then it is congestion. */ + if (cc->ewma_rtt_usec > westwood_rtt_signal(cc)) + return true; + + return false; +} + +/** + * Process a SENDME and update the congestion window according to the + * rules specified in TOR_WESTWOOD of Proposal #324. + * + * Essentially, this algorithm uses a threshhold of 'rtt_thresh', which + * is a midpoint between the min and max RTT. If the RTT exceeds this + * threshhold, then queue delay due to congestion is assumed to be present, + * and the algirithm reduces the congestion window. If the RTT is below the + * threshhold, the circuit is not congested (ie: queue delay is low), and we + * increase the congestion window. + * + * The congestion window is updated only once every congestion window worth of + * packets, even if the signal persists. It is also updated whenever the + * upstream orcon blocks, or unblocks. This minimizes local client queues. + */ +int +congestion_control_westwood_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + tor_assert(cc && cc->cc_alg == CC_ALG_WESTWOOD); + tor_assert(circ); + + /* Update ack counter until next congestion signal event is allowed */ + if (cc->next_cc_event) + cc->next_cc_event--; + + /* If we were unable to update our circuit estimates, Westwood must + * *not* update its cwnd, otherwise it could run to infinity, or to 0. + * Just update inflight from the sendme and return. */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We only update anything once per window */ + if (cc->next_cc_event == 0) { + if (!westwood_is_congested(cc)) { + if (cc->in_slow_start) { + cc->cwnd = MAX(cc->cwnd + CWND_INC_SS(cc), + cc->bdp[cc->bdp_alg]); + } else { + cc->cwnd = cc->cwnd + CWND_INC(cc); + } + } else { + if (cc->westwood_params.min_backoff) + cc->cwnd = MIN(cc->cwnd*cc->westwood_params.cwnd_backoff_m/100, + cc->bdp[cc->bdp_alg]); + else + cc->cwnd = MAX(cc->cwnd*cc->westwood_params.cwnd_backoff_m/100, + cc->bdp[cc->bdp_alg]); + + cc->in_slow_start = 0; + + // Because Westwood's congestion can runaway and boost max rtt, + // which increases its congestion signal, we backoff the max rtt + // too. + cc->max_rtt_usec = westwood_rtt_max_backoff(cc); + + log_info(LD_CIRC, "CC: TOR_WESTWOOD congestion. New max RTT: %"PRIu64, + cc->max_rtt_usec/1000); + } + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + /* Schedule next update */ + cc->next_cc_event = CWND_UPDATE_RATE(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: TOR_WESTWOOD Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "WRTT: %"PRIu64", " + "WSIG: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->ewma_rtt_usec/1000, + westwood_rtt_signal(cc)/1000, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC: TOR_WESTWOOD Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "WRTT: %"PRIu64", " + "WSIG: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->ewma_rtt_usec/1000, + westwood_rtt_signal(cc)/1000, + cc->in_slow_start + ); + } + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_westwood.h b/src/core/or/congestion_control_westwood.h new file mode 100644 index 0000000000..c6fd596df4 --- /dev/null +++ b/src/core/or/congestion_control_westwood.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_westwood.h + * \brief Private-ish APIs for the TOR_WESTWOOD congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_WESTWOOD_H +#define TOR_CONGESTION_CONTROL_WESTWOOD_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_westwood_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_westwood_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_WESTWOOD_H) */ diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index d3979b3a7e..ea4bf00735 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -69,6 +69,8 @@ #include "core/or/circuituse.h" #include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" +#include "core/or/congestion_control_flow.h" +#include "core/or/circuitstats.h" #include "core/or/connection_or.h" #include "core/or/extendinfo.h" #include "core/or/policies.h" @@ -614,20 +616,39 @@ connection_half_edge_add(const edge_connection_t *conn, half_conn->stream_id = conn->stream_id; - // How many sendme's should I expect? - half_conn->sendmes_pending = - (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT; - // Is there a connected cell pending? half_conn->connected_pending = conn->base_.state == AP_CONN_STATE_CONNECT_WAIT; - /* Data should only arrive if we're not waiting on a resolved cell. - * It can arrive after waiting on connected, because of optimistic - * data. */ - if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { - // How many more data cells can arrive on this id? - half_conn->data_pending = conn->deliver_window; + if (edge_uses_flow_control(conn)) { + /* If the edge uses the new congestion control flow control, we must use + * time-based limits on half-edge activity. */ + uint64_t timeout_usec = (uint64_t)(get_circuit_build_timeout_ms()*1000); + half_conn->used_ccontrol = 1; + + /* If this is an onion service circuit, double the CBT as an approximate + * value for the other half of the circuit */ + if (conn->hs_ident) { + timeout_usec *= 2; + } + + /* The stream should stop seeing any use after the larger of the circuit + * RTT and the overall circuit build timeout */ + half_conn->end_ack_expected_usec = MAX(timeout_usec, + edge_get_max_rtt(conn)) + + monotime_absolute_usec(); + } else { + // How many sendme's should I expect? + half_conn->sendmes_pending = + (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT; + + /* Data should only arrive if we're not waiting on a resolved cell. + * It can arrive after waiting on connected, because of optimistic + * data. */ + if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { + // How many more data cells can arrive on this id? + half_conn->data_pending = conn->deliver_window; + } } insert_at = smartlist_bsearch_idx(circ->half_streams, &half_conn->stream_id, @@ -688,6 +709,12 @@ connection_half_edge_is_valid_data(const smartlist_t *half_conns, if (!half) return 0; + if (half->used_ccontrol) { + if (monotime_absolute_usec() > half->end_ack_expected_usec) + return 0; + return 1; + } + if (half->data_pending > 0) { half->data_pending--; return 1; @@ -740,6 +767,10 @@ connection_half_edge_is_valid_sendme(const smartlist_t *half_conns, if (!half) return 0; + /* congestion control edges don't use sendmes */ + if (half->used_ccontrol) + return 0; + if (half->sendmes_pending > 0) { half->sendmes_pending--; return 1; @@ -1269,15 +1300,6 @@ connection_ap_rescan_and_attach_pending(void) connection_ap_attach_pending(1); } -#ifdef DEBUGGING_17659 -#define UNMARK() do { \ - entry_conn->marked_pending_circ_line = 0; \ - entry_conn->marked_pending_circ_file = 0; \ - } while (0) -#else /* !defined(DEBUGGING_17659) */ -#define UNMARK() do { } while (0) -#endif /* defined(DEBUGGING_17659) */ - /** Tell any AP streams that are listed as waiting for a new circuit to try * again. If there is an available circuit for a stream, attach it. Otherwise, * launch a new circuit. @@ -1306,21 +1328,18 @@ connection_ap_attach_pending(int retry) connection_t *conn = ENTRY_TO_CONN(entry_conn); tor_assert(conn && entry_conn); if (conn->marked_for_close) { - UNMARK(); continue; } if (conn->magic != ENTRY_CONNECTION_MAGIC) { log_warn(LD_BUG, "%p has impossible magic value %u.", entry_conn, (unsigned)conn->magic); - UNMARK(); continue; } if (conn->state != AP_CONN_STATE_CIRCUIT_WAIT) { - log_warn(LD_BUG, "%p is no longer in circuit_wait. Its current state " - "is %s. Why is it on pending_entry_connections?", - entry_conn, - conn_state_to_string(conn->type, conn->state)); - UNMARK(); + /* The connection_ap_handshake_attach_circuit() call, for onion service, + * can lead to more than one connections in the "pending" list to change + * state and so it is OK to get here. Ignore it because this connection + * won't be in pending_entry_connections list after this point. */ continue; } @@ -1345,7 +1364,6 @@ connection_ap_attach_pending(int retry) /* If we got here, then we either closed the connection, or * we attached it. */ - UNMARK(); } SMARTLIST_FOREACH_END(entry_conn); smartlist_free(pending); @@ -1416,7 +1434,6 @@ connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn) { if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) return; - UNMARK(); smartlist_remove(pending_entry_connections, entry_conn); } @@ -1612,23 +1629,6 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port) return 0; } -/** Return true iff <b>query</b> is a syntactically valid service ID (as - * generated by rend_get_service_id). */ -static int -rend_valid_v2_service_id(const char *query) -{ - /** Length of 'y' portion of 'y.onion' URL. */ -#define REND_SERVICE_ID_LEN_BASE32 16 - - if (strlen(query) != REND_SERVICE_ID_LEN_BASE32) - return 0; - - if (strspn(query, BASE32_CHARS) != REND_SERVICE_ID_LEN_BASE32) - return 0; - - return 1; -} - /** Parse the given hostname in address. Returns true if the parsing was * successful and type_out contains the type of the hostname. Else, false is * returned which means it was not recognized and type_out is set to @@ -1692,14 +1692,6 @@ parse_extended_hostname(char *address, hostname_type_t *type_out) if (q != address) { memmove(address, q, strlen(q) + 1 /* also get \0 */); } - /* v2 onion address check. */ - if (strlen(query) == REND_SERVICE_ID_LEN_BASE32) { - *type_out = ONION_V2_HOSTNAME; - if (rend_valid_v2_service_id(query)) { - goto success; - } - goto failed; - } /* v3 onion address check. */ if (strlen(query) == HS_SERVICE_ADDR_LEN_BASE32) { @@ -1719,8 +1711,7 @@ parse_extended_hostname(char *address, hostname_type_t *type_out) failed: /* otherwise, return to previous state and return 0 */ *s = '.'; - const bool is_onion = (*type_out == ONION_V2_HOSTNAME) || - (*type_out == ONION_V3_HOSTNAME); + const bool is_onion = (*type_out == ONION_V3_HOSTNAME); log_warn(LD_APP, "Invalid %shostname %s; rejecting", is_onion ? "onion " : "", safe_str_client(address)); @@ -2242,7 +2233,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } /* Now, we handle everything that isn't a .onion address. */ - if (addresstype != ONION_V3_HOSTNAME && addresstype != ONION_V2_HOSTNAME) { + if (addresstype != ONION_V3_HOSTNAME) { /* Not a hidden-service request. It's either a hostname or an IP, * possibly with a .exit that we stripped off. We're going to check * if we're allowed to connect/resolve there, and then launch the @@ -2527,28 +2518,6 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return 0; } else { /* If we get here, it's a request for a .onion address! */ - - /* We don't support v2 onions anymore. Log a warning and bail. */ - if (addresstype == ONION_V2_HOSTNAME) { - static bool log_once = false; - if (!log_once) { - log_warn(LD_PROTOCOL, "Tried to connect to a v2 onion address, but " - "this version of Tor no longer supports them. Please " - "encourage the site operator to upgrade. For more " - "information see " - "https://blog.torproject.org/v2-deprecation-timeline."); - log_once = true; - } - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - /* Send back the 0xF6 extended code indicating a bad hostname. This is - * mostly so Tor Browser can make a proper UX with regards to v2 - * addresses. */ - conn->socks_request->socks_extended_error_code = SOCKS5_HS_BAD_ADDRESS; - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; - } - tor_assert(addresstype == ONION_V3_HOSTNAME); tor_assert(!automap); return connection_ap_handle_onion(conn, socks, circ); diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h index 72869f348b..966a9391d8 100644 --- a/src/core/or/connection_edge.h +++ b/src/core/or/connection_edge.h @@ -80,7 +80,6 @@ typedef enum hostname_type_t { BAD_HOSTNAME, EXIT_HOSTNAME, NORMAL_HOSTNAME, - ONION_V2_HOSTNAME, ONION_V3_HOSTNAME, } hostname_type_t; diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index dd31638eb3..db9f93e6f6 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -65,6 +65,7 @@ #include "core/or/scheduler.h" #include "feature/nodelist/torcert.h" #include "core/or/channelpadding.h" +#include "core/or/congestion_control_common.h" #include "feature/dirauth/authmode.h" #include "feature/hs/hs_service.h" @@ -636,7 +637,7 @@ connection_or_flushed_some(or_connection_t *conn) /* If we're under the low water mark, add cells until we're just over the * high water mark. */ datalen = connection_get_outbuf_len(TO_CONN(conn)); - if (datalen < OR_CONN_LOWWATER) { + if (datalen < or_conn_lowwatermark()) { /* Let the scheduler know */ scheduler_channel_wants_writes(TLS_CHAN_TO_BASE(conn->chan)); } @@ -660,9 +661,9 @@ connection_or_num_cells_writeable(or_connection_t *conn) * used to trigger when to start writing after we've stopped. */ datalen = connection_get_outbuf_len(TO_CONN(conn)); - if (datalen < OR_CONN_HIGHWATER) { + if (datalen < or_conn_highwatermark()) { cell_network_size = get_cell_network_size(conn->wide_circ_ids); - n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, cell_network_size); + n = CEIL_DIV(or_conn_highwatermark() - datalen, cell_network_size); } return n; diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 29356d7c2a..7673bc306f 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -27,6 +27,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/extendinfo.h" +#include "core/or/congestion_control_common.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_util.h" @@ -165,6 +166,7 @@ cpath_free(crypt_path_t *victim) onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); + congestion_control_free(victim->ccontrol); memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 2529b6ee41..ddc85eec14 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -29,6 +29,8 @@ struct onion_handshake_state_t { } u; }; +struct congestion_control_t; + /** Macro to encapsulate private members of a struct. * * Renames 'x' to 'x_crypt_path_private_field'. @@ -80,6 +82,9 @@ struct crypt_path_t { int deliver_window; /**< How many cells are we willing to deliver originating * at this step? */ + /** Congestion control info */ + struct congestion_control_t *ccontrol; + /*********************** Private members ****************************/ /** Private member: Cryptographic state used for encrypting and diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h index 0120c3df25..dab32fc8d0 100644 --- a/src/core/or/edge_connection_st.h +++ b/src/core/or/edge_connection_st.h @@ -15,6 +15,7 @@ #include "core/or/or.h" #include "core/or/connection_st.h" +#include "lib/evloop/token_bucket.h" /** Subtype of connection_t for an "edge connection" -- that is, an entry (ap) * connection, or an exit. */ @@ -73,6 +74,60 @@ struct edge_connection_t { * that's going away and being used on channels instead. We still tag * edge connections with dirreq_id from circuits, so it's copied here. */ uint64_t dirreq_id; + + /* The following are flow control fields */ + + /** Used for rate limiting the read side of this edge connection when + * congestion control is enabled on its circuit. The XON cell ewma_drain_rate + * parameter is used to set the bucket limits. */ + token_bucket_rw_t bucket; + + /** + * Monotime timestamp of the last time we sent a flow control message + * for this edge, used to compute advisory rates */ + uint64_t drain_start_usec; + + /** + * Number of bytes written since we either emptied our buffers, + * or sent an advisory drate rate. Can wrap, buf if so, + * we must reset the usec timestamp above. (Or make this u64, idk). + */ + uint32_t drained_bytes; + uint32_t prev_drained_bytes; + + /** + * N_EWMA of the drain rate of writes on this edge conn + * while buffers were present. + */ + uint32_t ewma_drain_rate; + + /** + * The ewma drain rate the last time we sent an xon. + */ + uint32_t ewma_rate_last_sent; + + /** + * The following fields are used to count the total bytes sent on this + * stream, and compare them to the number of XON and XOFFs recieved, so + * that clients can check rate limits of XOFF/XON to prevent dropmark + * attacks. */ + uint32_t total_bytes_xmit; + + /** Number of XOFFs received */ + uint8_t num_xoff_recv; + + /** Number of XONs received */ + uint8_t num_xon_recv; + + /** + * Flag that tells us if an XOFF has been sent; cleared when we send an XON. + * Used to avoid sending multiple */ + uint8_t xoff_sent : 1; + + /** Flag that tells us if an XOFF has been received; cleared when we get + * an XON. Used to ensure that this edge keeps reads on its edge socket + * disabled. */ + uint8_t xoff_received : 1; }; #endif /* !defined(EDGE_CONNECTION_ST_H) */ diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h index c956c7434a..ac97eb19f1 100644 --- a/src/core/or/half_edge_st.h +++ b/src/core/or/half_edge_st.h @@ -31,6 +31,18 @@ typedef struct half_edge_t { * our deliver window */ int data_pending; + /** + * Monotime timestamp of when the other end should have successfuly + * shut down the stream and stop sending data, based on the larger + * of circuit RTT and CBT. Used if 'used_ccontrol' is true, to expire + * the half_edge at this monotime timestamp. */ + uint64_t end_ack_expected_usec; + + /** + * Did this edge use congestion control? If so, use + * timer instead of pending data approach */ + int used_ccontrol : 1; + /** Is there a connected cell pending? */ int connected_pending : 1; } half_edge_t; diff --git a/src/core/or/include.am b/src/core/or/include.am index b578b75673..b08f8509cc 100644 --- a/src/core/or/include.am +++ b/src/core/or/include.am @@ -28,13 +28,17 @@ LIBTOR_APP_A_SOURCES += \ src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ - src/core/or/protover_rust.c \ src/core/or/reasons.c \ src/core/or/relay.c \ src/core/or/scheduler.c \ src/core/or/scheduler_kist.c \ src/core/or/scheduler_vanilla.c \ src/core/or/sendme.c \ + src/core/or/congestion_control_common.c \ + src/core/or/congestion_control_vegas.c \ + src/core/or/congestion_control_nola.c \ + src/core/or/congestion_control_westwood.c \ + src/core/or/congestion_control_flow.c \ src/core/or/status.c \ src/core/or/versions.c @@ -57,6 +61,7 @@ noinst_HEADERS += \ src/core/or/circuitpadding_machines.h \ src/core/or/circuituse.h \ src/core/or/command.h \ + src/core/or/congestion_control_st.h \ src/core/or/connection_edge.h \ src/core/or/connection_or.h \ src/core/or/connection_st.h \ @@ -77,6 +82,7 @@ noinst_HEADERS += \ src/core/or/entry_port_cfg_st.h \ src/core/or/extend_info_st.h \ src/core/or/listener_connection_st.h \ + src/core/or/lttng_cc.inc \ src/core/or/lttng_circuit.inc \ src/core/or/onion.h \ src/core/or/or.h \ @@ -97,6 +103,11 @@ noinst_HEADERS += \ src/core/or/relay_crypto_st.h \ src/core/or/scheduler.h \ src/core/or/sendme.h \ + src/core/or/congestion_control_flow.h \ + src/core/or/congestion_control_common.h \ + src/core/or/congestion_control_vegas.h \ + src/core/or/congestion_control_nola.h \ + src/core/or/congestion_control_westwood.h \ src/core/or/server_port_cfg_st.h \ src/core/or/socks_request_st.h \ src/core/or/status.h \ @@ -106,7 +117,9 @@ noinst_HEADERS += \ if USE_TRACING_INSTRUMENTATION_LTTNG LIBTOR_APP_A_SOURCES += \ + src/core/or/trace_probes_cc.c \ src/core/or/trace_probes_circuit.c noinst_HEADERS += \ + src/core/or/trace_probes_cc.h \ src/core/or/trace_probes_circuit.h endif diff --git a/src/core/or/lttng_cc.inc b/src/core/or/lttng_cc.inc new file mode 100644 index 0000000000..b7bf58e196 --- /dev/null +++ b/src/core/or/lttng_cc.inc @@ -0,0 +1,166 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file lttng_cc.inc + * \brief LTTng tracing probe declaration for the congestion control subsystem. + * It is in this .inc file due to the non C standard syntax and the way + * we guard the header with the LTTng specific + * TRACEPOINT_HEADER_MULTI_READ. + **/ + +#include "orconfig.h" + +/* We only build the following if LTTng instrumentation has been enabled. */ +#ifdef USE_TRACING_INSTRUMENTATION_LTTNG + +/* The following defines are LTTng-UST specific. */ +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER tor_cc + +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "./src/core/or/lttng_cc.inc" + +#if !defined(LTTNG_CC_INC) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define LTTNG_CC_INC + +#include <lttng/tracepoint.h> + +/* + * Flow Control + */ + +/* Emitted everytime the flow_control_decide_xon() function is called. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon, + TP_ARGS(const edge_connection_t *, stream, size_t, n_written), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(size_t, written_bytes, n_written) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when flow control starts measuring the drain rate. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_drain_start, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when the drain rate is updated. The new_drain_rate value is what was + * just computed. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_drain_update, + TP_ARGS(const edge_connection_t *, stream, uint32_t, drain_rate), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, new_drain_rate, drain_rate) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when an XON cell is sent due to a notice in a drain rate change. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_rate_change, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when an XON cell is sent because we partially or fully drained the + * edge connection buffer. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_partial_drain, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when we double the drain rate which is an attempt to see if we can + * speed things up. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_drain_doubled, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* XOFF */ + +/* Emitted when we send an XOFF cell. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xoff_sending, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* + * Congestion Control + */ + +/* Emitted when the BDP value has been updated. */ +TRACEPOINT_EVENT(tor_cc, bdp_update, + TP_ARGS(const circuit_t *, circ, const congestion_control_t *, cc, + uint64_t, curr_rtt_usec, uint64_t, sendme_rate_bdp), + TP_FIELDS( + ctf_integer(uint64_t, circuit_ptr, circ) + ctf_integer(uint32_t, n_circ_id, circ->n_circ_id) + ctf_integer(uint64_t, min_rtt_usec, cc->min_rtt_usec) + ctf_integer(uint64_t, curr_rtt_usec, curr_rtt_usec) + ctf_integer(uint64_t, ewma_rtt_usec, cc->ewma_rtt_usec) + ctf_integer(uint64_t, max_rtt_usec, cc->max_rtt_usec) + ctf_integer(uint64_t, bdp_inflight_rtt, cc->bdp[BDP_ALG_INFLIGHT_RTT]) + ctf_integer(uint64_t, bdp_cwnd_rtt, cc->bdp[BDP_ALG_CWND_RTT]) + ctf_integer(uint64_t, bdp_sendme_rate, cc->bdp[BDP_ALG_SENDME_RATE]) + ctf_integer(uint64_t, bdp_piecewise, cc->bdp[BDP_ALG_PIECEWISE]) + ctf_integer(uint64_t, sendme_rate_bdp, sendme_rate_bdp) + ) +) + +#endif /* LTTNG_CC_INC || TRACEPOINT_HEADER_MULTI_READ */ + +/* Must be included after the probes declaration. */ +#include <lttng/tracepoint-event.h> + +#endif /* USE_TRACING_INSTRUMENTATION_LTTNG */ diff --git a/src/core/or/or.h b/src/core/or/or.h index 99948f26e2..392a848ee7 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -210,6 +210,9 @@ struct curve25519_public_key_t; #define RELAY_COMMAND_PADDING_NEGOTIATE 41 #define RELAY_COMMAND_PADDING_NEGOTIATED 42 +#define RELAY_COMMAND_XOFF 43 +#define RELAY_COMMAND_XON 44 + /* Reasons why an OR connection is closed. */ #define END_OR_CONN_REASON_DONE 1 #define END_OR_CONN_REASON_REFUSED 2 /* connection refused */ @@ -591,18 +594,6 @@ typedef struct or_handshake_state_t or_handshake_state_t; /** Length of Extended ORPort connection identifier. */ #define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */ -/* - * OR_CONN_HIGHWATER and OR_CONN_LOWWATER moved from connection_or.c so - * channeltls.c can see them too. - */ - -/** When adding cells to an OR connection's outbuf, keep adding until the - * outbuf is at least this long, or we run out of cells. */ -#define OR_CONN_HIGHWATER (32*1024) - -/** Add cells to an OR connection's outbuf whenever the outbuf's data length - * drops below this size. */ -#define OR_CONN_LOWWATER (16*1024) typedef struct connection_t connection_t; typedef struct control_connection_t control_connection_t; diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index b8fbf9658e..11695ec301 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -52,6 +52,10 @@ struct or_circuit_t { /** Stores KH for the handshake. */ char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */ + /** Number of cells which we have discarded because of having no next hop, + * despite not recognizing the cell. */ + uint32_t n_cells_discarded_at_end; + /** How many more relay_early cells can we send on this circuit, according * to the specification? */ unsigned int remaining_relay_early_cells : 4; @@ -93,4 +97,3 @@ struct or_circuit_t { }; #endif /* !defined(OR_CIRCUIT_ST_H) */ - diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index 9264077c50..6c86a56000 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -180,6 +180,12 @@ struct origin_circuit_t { unsigned first_hop_from_controller : 1; /** + * If true, this circuit's path has been chosen, in full or in part, + * by the controller API, and it's okay to ignore checks that we'd + * usually do on the path as whole. */ + unsigned int any_hop_from_controller : 1; + + /** * Tristate variable to guard against pathbias miscounting * due to circuit purpose transitions changing the decision * of pathbias_should_count(). This variable is informational diff --git a/src/core/or/policies.c b/src/core/or/policies.c index f91c23ad31..a53849b4d0 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -59,6 +59,9 @@ static smartlist_t *authdir_invalid_policy = NULL; /** Policy that addresses for incoming router descriptors must <b>not</b> * match in order to not be marked as BadExit. */ static smartlist_t *authdir_badexit_policy = NULL; +/** Policy that addresses for incoming router descriptors must <b>not</b> + * match in order to not be marked as MiddleOnly. */ +static smartlist_t *authdir_middleonly_policy = NULL; /** Parsed addr_policy_t describing which addresses we believe we can start * circuits at. */ @@ -1119,6 +1122,17 @@ authdir_policy_badexit_address(const tor_addr_t *addr, uint16_t port) return addr_is_in_cc_list(addr, get_options()->AuthDirBadExitCCs); } +/** Return 1 if <b>addr</b>:<b>port</b> should be marked as MiddleOnly, + * based on <b>authdir_middleonly_policy</b>. Else return 0. + */ +int +authdir_policy_middleonly_address(const tor_addr_t *addr, uint16_t port) +{ + if (!addr_policy_permits_tor_addr(addr, port, authdir_middleonly_policy)) + return 1; + return addr_is_in_cc_list(addr, get_options()->AuthDirMiddleOnlyCCs); +} + #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END @@ -1173,6 +1187,9 @@ validate_addr_policies(const or_options_t *options, char **msg) if (parse_addr_policy(options->AuthDirBadExit, &addr_policy, ADDR_POLICY_REJECT)) REJECT("Error in AuthDirBadExit entry."); + if (parse_addr_policy(options->AuthDirMiddleOnly, &addr_policy, + ADDR_POLICY_REJECT)) + REJECT("Error in AuthDirMiddleOnly entry."); if (parse_addr_policy(options->ReachableAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) @@ -1266,6 +1283,9 @@ policies_parse_from_options(const or_options_t *options) if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; + if (load_policy_from_option(options->AuthDirMiddleOnly, "AuthDirMiddleOnly", + &authdir_middleonly_policy, ADDR_POLICY_REJECT) < 0) + ret = -1; if (parse_metrics_port_policy(options) < 0) { ret = -1; } @@ -3112,6 +3132,8 @@ policies_free_all(void) authdir_invalid_policy = NULL; addr_policy_list_free(authdir_badexit_policy); authdir_badexit_policy = NULL; + addr_policy_list_free(authdir_middleonly_policy); + authdir_middleonly_policy = NULL; if (!HT_EMPTY(&policy_root)) { policy_map_ent_t **ent; diff --git a/src/core/or/policies.h b/src/core/or/policies.h index a32f50ab1d..e11e1d0ff5 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -106,6 +106,7 @@ int metrics_policy_permits_address(const tor_addr_t *addr); int authdir_policy_permits_address(const tor_addr_t *addr, uint16_t port); int authdir_policy_valid_address(const tor_addr_t *addr, uint16_t port); int authdir_policy_badexit_address(const tor_addr_t *addr, uint16_t port); +int authdir_policy_middleonly_address(const tor_addr_t *addr, uint16_t port); int validate_addr_policies(const or_options_t *options, char **msg); void policy_expand_private(smartlist_t **policy); diff --git a/src/core/or/protover.c b/src/core/or/protover.c index 04df8aeeb8..199fc830a0 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -28,8 +28,6 @@ #include "core/or/versions.h" #include "lib/tls/tortls.h" -#ifndef HAVE_RUST - static const smartlist_t *get_supported_protocol_list(void); static int protocol_list_contains(const smartlist_t *protos, protocol_type_t pr, uint32_t ver); @@ -855,5 +853,3 @@ protover_free_all(void) supported_protocol_list = NULL; } } - -#endif /* !defined(HAVE_RUST) */ diff --git a/src/core/or/protover.h b/src/core/or/protover.h index c0739a092e..ae258d74a5 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -103,13 +103,13 @@ typedef struct proto_entry_t { uint64_t bitmask; } proto_entry_t; -#if !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS) +#if defined(TOR_UNIT_TESTS) STATIC struct smartlist_t *parse_protocol_list(const char *s); STATIC char *encode_protocol_list(const struct smartlist_t *sl); STATIC const char *protocol_type_to_str(protocol_type_t pr); STATIC int str_to_protocol_type(const char *s, protocol_type_t *pr_out); STATIC void proto_entry_free_(proto_entry_t *entry); -#endif /* !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS) */ +#endif /* defined(TOR_UNIT_TESTS) */ #define proto_entry_free(entry) \ FREE_AND_NULL(proto_entry_t, proto_entry_free_, (entry)) diff --git a/src/core/or/protover_rust.c b/src/core/or/protover_rust.c deleted file mode 100644 index 31ddfa1bdf..0000000000 --- a/src/core/or/protover_rust.c +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright (c) 2016-2021, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* - * \file protover_rust.c - * \brief Provide a C wrapper for functions exposed in /src/rust/protover, - * and safe translation/handling between the Rust/C boundary. - */ - -#include "core/or/or.h" -#include "core/or/protover.h" - -#ifdef HAVE_RUST - -/* Define for compatibility, used in main.c */ -void -protover_free_all(void) -{ -} - -int protover_contains_long_protocol_names_(const char *s); - -/** - * Return true if the unparsed protover in <b>s</b> would contain a protocol - * name longer than MAX_PROTOCOL_NAME_LENGTH, and false otherwise. - */ -bool -protover_list_is_invalid(const char *s) -{ - return protover_contains_long_protocol_names_(s) != 0; -} - -#endif /* defined(HAVE_RUST) */ - diff --git a/src/core/or/relay.c b/src/core/or/relay.c index f5a9e73856..68fddd1ae7 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -97,6 +97,8 @@ #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" #include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -115,13 +117,6 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ, node_t *node, const tor_addr_t *addr); -/** Stop reading on edge connections when we have this many cells - * waiting on the appropriate queue. */ -#define CELL_QUEUE_HIGHWATER_SIZE 256 -/** Start reading from edge connections again when we get down to this many - * cells. */ -#define CELL_QUEUE_LOWWATER_SIZE 64 - /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? */ @@ -338,8 +333,17 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, } return 0; } - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Didn't recognize cell, but circ stops here! Closing circ."); + if (BUG(CIRCUIT_IS_ORIGIN(circ))) { + /* Should be impossible at this point. */ + return -END_CIRC_REASON_TORPROTOCOL; + } + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + if (++or_circ->n_cells_discarded_at_end == 1) { + time_t seconds_open = approx_time() - circ->timestamp_created.tv_sec; + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Didn't recognize a cell, but circ stops here! Closing circuit. " + "It was created %ld seconds ago.", (long)seconds_open); + } return -END_CIRC_REASON_TORPROTOCOL; } @@ -1574,6 +1578,7 @@ process_sendme_cell(const relay_header_t *rh, const cell_t *cell, } /* Stream level SENDME cell. */ + // TODO: Turn this off for cc_alg=1,2,3; use XON/XOFF instead ret = sendme_process_stream_level(conn, circ, rh->length); if (ret < 0) { /* Means we need to close the circuit with reason ret. */ @@ -1738,6 +1743,44 @@ handle_relay_cell_command(cell_t *cell, circuit_t *circ, } return 0; + case RELAY_COMMAND_XOFF: + if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (relay_crypt_from_last_hop(ocirc, layer_hint) && + connection_half_edge_is_valid_data(ocirc->half_streams, + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); + } + } + return 0; + } + + if (circuit_process_stream_xoff(conn, layer_hint, cell)) { + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + } + } + return 0; + case RELAY_COMMAND_XON: + if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (relay_crypt_from_last_hop(ocirc, layer_hint) && + connection_half_edge_is_valid_data(ocirc->half_streams, + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); + } + } + return 0; + } + + if (circuit_process_stream_xon(conn, layer_hint, cell)) { + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + } + } + return 0; case RELAY_COMMAND_END: reason = rh->length > 0 ? get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; @@ -2091,6 +2134,7 @@ void circuit_reset_sendme_randomness(circuit_t *circ) { circ->have_sent_sufficiently_random_cell = 0; + // XXX: do we need to change this check for congestion control? circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 + crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2); } @@ -2284,7 +2328,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, } /* Handle the stream-level SENDME package window. */ - if (sendme_note_stream_data_packaged(conn) < 0) { + if (sendme_note_stream_data_packaged(conn, length) < 0) { connection_stop_reading(TO_CONN(conn)); log_debug(domain,"conn->package_window reached 0."); circuit_consider_stop_edge_reading(circ, cpath_layer); @@ -2350,15 +2394,16 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, /* How many cells do we have space for? It will be the minimum of * the number needed to exhaust the package window, and the minimum * needed to fill the cell queue. */ - max_to_package = circ->package_window; + + max_to_package = congestion_control_get_package_window(circ, layer_hint); if (CIRCUIT_IS_ORIGIN(circ)) { cells_on_queue = circ->n_chan_cells.n; } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); cells_on_queue = or_circ->p_chan_cells.n; } - if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package) - max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue; + if (cell_queue_highwatermark() - cells_on_queue < max_to_package) + max_to_package = cell_queue_highwatermark() - cells_on_queue; /* Once we used to start listening on the streams in the order they * appeared in the linked list. That leads to starvation on the @@ -2398,7 +2443,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, /* Activate reading starting from the chosen stream */ for (conn=chosen_stream; conn; conn = conn->next_stream) { /* Start reading for the streams starting from here */ - if (conn->base_.marked_for_close || conn->package_window <= 0) + if (conn->base_.marked_for_close || conn->package_window <= 0 || + conn->xoff_received) continue; if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); @@ -2409,7 +2455,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, } /* Go back and do the ones we skipped, circular-style */ for (conn = first_conn; conn != chosen_stream; conn = conn->next_stream) { - if (conn->base_.marked_for_close || conn->package_window <= 0) + if (conn->base_.marked_for_close || conn->package_window <= 0 || + conn->xoff_received) continue; if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); @@ -2495,7 +2542,7 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); log_debug(domain,"considering circ->package_window %d", circ->package_window); - if (circ->package_window <= 0) { + if (congestion_control_get_package_window(circ, layer_hint) <= 0) { log_debug(domain,"yes, not-at-origin. stopped."); for (conn = or_circ->n_streams; conn; conn=conn->next_stream) connection_stop_reading(TO_CONN(conn)); @@ -2506,7 +2553,7 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) /* else, layer hint is defined, use it */ log_debug(domain,"considering layer_hint->package_window %d", layer_hint->package_window); - if (layer_hint->package_window <= 0) { + if (congestion_control_get_package_window(circ, layer_hint) <= 0) { log_debug(domain,"yes, at-origin. stopped."); for (conn = TO_ORIGIN_CIRCUIT(circ)->p_streams; conn; conn=conn->next_stream) { @@ -2722,11 +2769,18 @@ cell_queues_get_total_allocation(void) /** The time at which we were last low on memory. */ static time_t last_time_under_memory_pressure = 0; +/** Statistics on how many bytes were removed by the OOM per type. */ +uint64_t oom_stats_n_bytes_removed_dns = 0; +uint64_t oom_stats_n_bytes_removed_cell = 0; +uint64_t oom_stats_n_bytes_removed_geoip = 0; +uint64_t oom_stats_n_bytes_removed_hsdir = 0; + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ STATIC int cell_queues_check_size(void) { + size_t removed = 0; time_t now = time(NULL); size_t alloc = cell_queues_get_total_allocation(); alloc += half_streams_get_total_allocation(); @@ -2751,20 +2805,27 @@ cell_queues_check_size(void) if (hs_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = hs_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= hs_cache_handle_oom(now, bytes_to_remove); + removed = hs_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_hsdir += removed; + alloc -= removed; } if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = geoip_client_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= geoip_client_cache_handle_oom(now, bytes_to_remove); + removed = geoip_client_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_geoip += removed; + alloc -= removed; } if (dns_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = dns_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= dns_cache_handle_oom(now, bytes_to_remove); + removed = dns_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_dns += removed; + alloc -= removed; } - circuits_handle_oom(alloc); + removed = circuits_handle_oom(alloc); + oom_stats_n_bytes_removed_cell += removed; return 1; } } @@ -3062,7 +3123,7 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max)) /* Is the cell queue low enough to unblock all the streams that are waiting * to write to this circuit? */ - if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) + if (streams_blocked && queue->n <= cell_queue_lowwatermark()) set_streams_blocked_on_circ(circ, chan, 0, 0); /* unblock streams */ /* If n_flushed < max still, loop around and pick another circuit */ @@ -3180,7 +3241,7 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, /* If we have too many cells on the circuit, we should stop reading from * the edge streams for a while. */ - if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE) + if (!streams_blocked && queue->n >= cell_queue_highwatermark()) set_streams_blocked_on_circ(circ, chan, 1, 0); /* block streams */ if (streams_blocked && fromstream) { diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 2f337d5d16..eac920f491 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -49,6 +49,11 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; +extern uint64_t oom_stats_n_bytes_removed_dns; +extern uint64_t oom_stats_n_bytes_removed_cell; +extern uint64_t oom_stats_n_bytes_removed_geoip; +extern uint64_t oom_stats_n_bytes_removed_hsdir; + void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index ce3385ae98..ee670f9d51 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -21,6 +21,8 @@ #include "core/or/or_circuit_st.h" #include "core/or/relay.h" #include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" #include "feature/nodelist/networkstatus.h" #include "lib/ctime/di_ops.h" #include "trunnel/sendme_cell.h" @@ -64,13 +66,6 @@ pop_first_cell_digest(const circuit_t *circ) return NULL; } - /* More cell digest than the SENDME window is never suppose to happen. The - * cell should have been rejected before reaching this point due to its - * package_window down to 0 leading to a circuit close. Scream loudly but - * still pop the element so we don't memory leak. */ - tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <= - CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT); - circ_digest = smartlist_get(circ->sendme_last_digests, 0); smartlist_del_keeporder(circ->sendme_last_digests, 0); return circ_digest; @@ -334,17 +329,18 @@ record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest) /** Return true iff the next cell for the given cell window is expected to be * a SENDME. * - * We are able to know that because the package or deliver window value minus - * one cell (the possible SENDME cell) should be a multiple of the increment - * window value. */ + * We are able to know that because the package or inflight window value minus + * one cell (the possible SENDME cell) should be a multiple of the + * cells-per-sendme increment value (set via consensus parameter, negotiated + * for the circuit, and passed in as sendme_inc). + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. + */ static bool -circuit_sendme_cell_is_next(int window) +circuit_sendme_cell_is_next(int window, int sendme_inc) { - /* At the start of the window, no SENDME will be expected. */ - if (window == CIRCWINDOW_START) { - return false; - } - /* Are we at the limit of the increment and if not, we don't expect next * cell is a SENDME. * @@ -352,11 +348,8 @@ circuit_sendme_cell_is_next(int window) * next cell is a SENDME, the window (either package or deliver) hasn't been * decremented just yet so when this is called, we are currently processing * the "window - 1" cell. - * - * This function is used when recording a cell digest and this is done quite - * low in the stack when decrypting or encrypting a cell. The window is only - * updated once the cell is actually put in the outbuf. */ - if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + */ + if (((window - 1) % sendme_inc) != 0) { return false; } @@ -378,6 +371,10 @@ sendme_connection_edge_consider_sending(edge_connection_t *conn) int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT; + /* If we use flow control, we do not send stream sendmes */ + if (edge_uses_flow_control(conn)) + goto end; + /* Don't send it if we still have data to deliver. */ if (connection_outbuf_too_full(TO_CONN(conn))) { goto end; @@ -419,15 +416,16 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) { bool sent_one_sendme = false; const uint8_t *digest; + int sendme_inc = sendme_get_inc_count(circ, layer_hint); while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= - CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { + CIRCWINDOW_START - sendme_inc) { log_debug(LD_CIRC,"Queuing circuit sendme."); if (layer_hint) { - layer_hint->deliver_window += CIRCWINDOW_INCREMENT; + layer_hint->deliver_window += sendme_inc; digest = cpath_get_sendme_digest(layer_hint); } else { - circ->deliver_window += CIRCWINDOW_INCREMENT; + circ->deliver_window += sendme_inc; digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); } if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { @@ -448,6 +446,9 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) * the length of the SENDME cell payload (excluding the header). The * cell_payload is the payload. * + * This function validates the SENDME's digest, and then dispatches to + * the appropriate congestion control algorithm in use on the circuit. + * * Return 0 on success (the SENDME is valid and the package window has * been updated properly). * @@ -460,6 +461,7 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, { tor_assert(circ); tor_assert(cell_payload); + congestion_control_t *cc; /* Validate the SENDME cell. Depending on the version, different validation * can be done. An invalid SENDME requires us to close the circuit. */ @@ -467,6 +469,34 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, return -END_CIRC_REASON_TORPROTOCOL; } + // Get CC + if (layer_hint) { + cc = layer_hint->ccontrol; + + /* origin circuits need to count valid sendmes as valid protocol data */ + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); + } else { + cc = circ->ccontrol; + } + + /* If there is no CC object, assume fixed alg */ + if (!cc) { + return sendme_process_circuit_level_impl(layer_hint, circ); + } + + return congestion_control_dispatch_cc_alg(cc, circ, layer_hint); +} + +/** + * Process a SENDME for Tor's original fixed window circuit-level flow control. + * Updates the package_window and ensures that it does not exceed the max. + * + * Returns -END_CIRC_REASON_TORPROTOCOL if the max is exceeded, otherwise + * returns 0. + */ +int +sendme_process_circuit_level_impl(crypt_path_t *layer_hint, circuit_t *circ) +{ /* If we are the origin of the circuit, we are the Client so we use the * layer hint (the Exit hop) for the package window tracking. */ if (CIRCUIT_IS_ORIGIN(circ)) { @@ -486,10 +516,6 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, layer_hint->package_window += CIRCWINDOW_INCREMENT; log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.", layer_hint->package_window); - - /* We count circuit-level sendme's as valid delivered data because they - * are rate limited. */ - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); } else { /* We aren't the origin of this circuit so we are the Exit and thus we * track the package window with the circuit object. */ @@ -525,6 +551,12 @@ sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, tor_assert(conn); tor_assert(circ); + if (edge_uses_flow_control(conn)) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Congestion control got stream sendme"); + return -END_CIRC_REASON_TORPROTOCOL; + } + /* Don't allow the other endpoint to request more than our maximum (i.e. * initial) stream SENDME window worth of data. Well-behaved stock clients * will not request more than this max (as per the check in the while loop @@ -582,7 +614,12 @@ int sendme_stream_data_received(edge_connection_t *conn) { tor_assert(conn); - return --conn->deliver_window; + + if (edge_uses_flow_control(conn)) { + return flow_control_decide_xoff(conn); + } else { + return --conn->deliver_window; + } } /* Called when a relay DATA cell is packaged on the given circuit. If @@ -592,34 +629,56 @@ int sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) { int package_window, domain; + congestion_control_t *cc; tor_assert(circ); - if (CIRCUIT_IS_ORIGIN(circ)) { - /* Client side. */ - tor_assert(layer_hint); - --layer_hint->package_window; - package_window = layer_hint->package_window; + if (layer_hint) { + cc = layer_hint->ccontrol; domain = LD_APP; } else { - /* Exit side. */ - tor_assert(!layer_hint); - --circ->package_window; - package_window = circ->package_window; + cc = circ->ccontrol; domain = LD_EXIT; } - log_debug(domain, "Circuit package_window now %d.", package_window); - return package_window; + if (cc) { + congestion_control_note_cell_sent(cc, circ, layer_hint); + } else { + /* Fixed alg uses package_window and must update it */ + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* Client side. */ + tor_assert(layer_hint); + --layer_hint->package_window; + package_window = layer_hint->package_window; + } else { + /* Exit side. */ + tor_assert(!layer_hint); + --circ->package_window; + package_window = circ->package_window; + } + log_debug(domain, "Circuit package_window now %d.", package_window); + } + + /* Return appropriate number designating how many cells can still be sent */ + return congestion_control_get_package_window(circ, layer_hint); } /* Called when a relay DATA cell is packaged for the given edge connection * conn. Update the package window and return its new value. */ int -sendme_note_stream_data_packaged(edge_connection_t *conn) +sendme_note_stream_data_packaged(edge_connection_t *conn, size_t len) { tor_assert(conn); + if (edge_uses_flow_control(conn)) { + flow_control_note_sent_data(conn, len); + if (conn->xoff_received) + return -1; + else + return 1; + } + --conn->package_window; log_debug(LD_APP, "Stream package_window now %d.", conn->package_window); return conn->package_window; @@ -631,20 +690,14 @@ sendme_note_stream_data_packaged(edge_connection_t *conn) void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath) { - int package_window; uint8_t *sendme_digest; tor_assert(circ); - package_window = circ->package_window; - if (cpath) { - package_window = cpath->package_window; - } - /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (!circuit_sendme_cell_is_next(package_window)) { + if (!circuit_sent_cell_for_sendme(circ, cpath)) { return; } @@ -670,7 +723,8 @@ sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath) /* Only record if the next cell is expected to be a SENDME. */ if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window : - circ->deliver_window)) { + circ->deliver_window, + sendme_get_inc_count(circ, cpath))) { return; } @@ -692,8 +746,7 @@ sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath) tor_assert(circ); /* Only record if the next cell is expected to be a SENDME. */ - if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window : - circ->package_window)) { + if (!circuit_sent_cell_for_sendme(circ, cpath)) { goto end; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index a008940905..2abec91a91 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -22,6 +22,7 @@ void sendme_circuit_consider_sending(circuit_t *circ, int sendme_process_circuit_level(crypt_path_t *layer_hint, circuit_t *circ, const uint8_t *cell_payload, uint16_t cell_payload_len); +int sendme_process_circuit_level_impl(crypt_path_t *, circuit_t *); int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, uint16_t cell_body_len); @@ -32,7 +33,7 @@ int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint); /* Update package window functions. */ int sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint); -int sendme_note_stream_data_packaged(edge_connection_t *conn); +int sendme_note_stream_data_packaged(edge_connection_t *conn, size_t len); /* Record cell digest on circuit. */ void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath); diff --git a/src/core/or/status.c b/src/core/or/status.c index 9e7ae70535..1e599aafb3 100644 --- a/src/core/or/status.c +++ b/src/core/or/status.c @@ -147,6 +147,32 @@ note_connection(bool inbound, int family) } } +/** + * @name Counters for unrecognized cells + * + * Track cells that we drop because they are unrecognized and we have + * nobody to send them to. + **/ +/**@{*/ +static unsigned n_circs_closed_for_unrecognized_cells; +static uint64_t n_unrecognized_cells_discarded; +static uint64_t n_secs_on_circs_with_unrecognized_cells; +/**@}*/ + +/** + * Note that a circuit has closed @a n_seconds after having been created, + * because of one or more unrecognized cells. Also note the number of + * unrecognized cells @a n_cells. + */ +void +note_circ_closed_for_unrecognized_cells(time_t n_seconds, uint32_t n_cells) +{ + ++n_circs_closed_for_unrecognized_cells; + n_unrecognized_cells_discarded += n_cells; + if (n_seconds >= 0) + n_secs_on_circs_with_unrecognized_cells += (uint64_t) n_seconds; +} + /** Log a "heartbeat" message describing Tor's status and history so that the * user can know that there is indeed a running Tor. Return 0 on success and * -1 on failure. */ @@ -240,6 +266,23 @@ log_heartbeat(time_t now) (main_loop_idle_count)); } + if (n_circs_closed_for_unrecognized_cells) { + double avg_time_alive = ((double) n_secs_on_circs_with_unrecognized_cells) + / n_circs_closed_for_unrecognized_cells; + double avg_cells = ((double) n_unrecognized_cells_discarded) + / n_circs_closed_for_unrecognized_cells; + log_fn(LOG_NOTICE, LD_HEARTBEAT, + "Since our last heartbeat, %u circuits were closed because of " + "unrecognized cells while we were the last hop. On average, each " + "one was alive for %lf seconds, and had %lf unrecognized cells.", + n_circs_closed_for_unrecognized_cells, + avg_time_alive, + avg_cells); + n_circs_closed_for_unrecognized_cells = 0; + n_unrecognized_cells_discarded = 0; + n_secs_on_circs_with_unrecognized_cells = 0; + } + /** Now, if we are an HS service, log some stats about our usage */ log_onion_service_stats(); diff --git a/src/core/or/status.h b/src/core/or/status.h index 927df9a192..57e28002fc 100644 --- a/src/core/or/status.h +++ b/src/core/or/status.h @@ -12,6 +12,9 @@ #include "lib/testsupport/testsupport.h" void note_connection(bool inbound, int family); +void note_circ_closed_for_unrecognized_cells(time_t n_seconds, + uint32_t n_cells); + int log_heartbeat(time_t now); #ifdef STATUS_PRIVATE diff --git a/src/core/or/trace_probes_cc.c b/src/core/or/trace_probes_cc.c new file mode 100644 index 0000000000..d52646da4f --- /dev/null +++ b/src/core/or/trace_probes_cc.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file trace_probes_cc.c + * \brief Tracepoint provider source file for the cc subsystem. Probes + * are generated within this C file for LTTng-UST + **/ + +#include "orconfig.h" + +/* + * Following section is specific to LTTng-UST. + */ +#ifdef USE_TRACING_INSTRUMENTATION_LTTNG + +/* Header files that the probes need. */ +#include "core/or/or.h" +#include "core/or/channel.h" +#include "core/or/circuit_st.h" +#include "core/or/circuitlist.h" +#include "core/or/congestion_control_st.h" +#include "core/or/connection_st.h" +#include "core/or/edge_connection_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" + +#define TRACEPOINT_DEFINE +#define TRACEPOINT_CREATE_PROBES + +#include "core/or/trace_probes_cc.h" + +#endif /* defined(USE_TRACING_INSTRUMENTATION_LTTNG) */ diff --git a/src/core/or/trace_probes_cc.h b/src/core/or/trace_probes_cc.h new file mode 100644 index 0000000000..1f87528723 --- /dev/null +++ b/src/core/or/trace_probes_cc.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file trace_probes_cc.c + * \brief The tracing probes for the congestion control subsystem. + * Currently, only LTTng-UST probes are available. + **/ + +#ifndef TOR_TRACE_PROBES_CC_H +#define TOR_TRACE_PROBES_CC_H + +#include "lib/trace/events.h" + +/* We only build the following if LTTng instrumentation has been enabled. */ +#ifdef USE_TRACING_INSTRUMENTATION_LTTNG + +#include "core/or/lttng_cc.inc" + +#endif /* USE_TRACING_INSTRUMENTATION_LTTNG */ + +#endif /* !defined(TOR_TRACE_PROBES_CC_H) */ diff --git a/src/ext/ed25519/donna/test-ticks.h b/src/ext/ed25519/donna/test-ticks.h index 0103e03dde..16cec9ba92 100644 --- a/src/ext/ed25519/donna/test-ticks.h +++ b/src/ext/ed25519/donna/test-ticks.h @@ -13,7 +13,7 @@ get_ticks(void) { __asm__ __volatile__("rdtsc" : "=a" (lo), "=d" (hi)); return ((uint64_t)lo | ((uint64_t)hi << 32)); #else - need rdtsc for this compiler + #error need rdtsc for this compiler #endif #elif defined(OS_SOLARIS) return (uint64_t)gethrtime(); @@ -35,7 +35,7 @@ get_ticks(void) { t = ((uint64_t)t2.tv_usec << 32) | (uint64_t)t2.tv_sec; return t; #else - need ticks for this platform + #error need ticks for this platform #endif } diff --git a/src/ext/rust b/src/ext/rust deleted file mode 160000 -Subproject aa37fb84fb829902e83ca11a7244bbc6b86b809 diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c index 051be50b3a..88e91ebfd5 100644 --- a/src/feature/api/tor_api.c +++ b/src/feature/api/tor_api.c @@ -18,9 +18,9 @@ // Include this after the above headers, to insure that they don't // depend on anything else. #include "orconfig.h" +#include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "feature/api/tor_api_internal.h" -#include "lib/cc/compat_compiler.h" #include <stdio.h> #include <stdlib.h> @@ -39,7 +39,9 @@ #include "lib/net/socketpair.h" #define raw_socketpair tor_ersatz_socketpair #define raw_closesocket closesocket +#if !defined(HAVE_SNPRINTF) #define snprintf _snprintf +#endif #else /* !defined(_WIN32) */ #define raw_socketpair socketpair #define raw_closesocket close diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index d40bcc6c8e..9e36d26929 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -943,9 +943,17 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } /** We just learned a descriptor for a bridge. See if that - * digest is in our entry guard list, and add it if not. */ + * digest is in our entry guard list, and add it if not. Schedule the + * next fetch for a long time from now, and initiate any follow-up + * activities like continuing to bootstrap. + * + * <b>from_cache</b> * tells us whether we fetched it from disk (else + * the network) + * + * <b>desc_is_new</b> tells us if we preferred it to the old version we + * had, if any. */ void -learned_bridge_descriptor(routerinfo_t *ri, int from_cache) +learned_bridge_descriptor(routerinfo_t *ri, int from_cache, int desc_is_new) { tor_assert(ri); tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); @@ -961,12 +969,14 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) if (bridge) { /* if we actually want to use this one */ node_t *node; - /* it's here; schedule its re-fetch for a long time from now. */ if (!from_cache) { /* This schedules the re-fetch at a constant interval, which produces * a pattern of bridge traffic. But it's better than trying all * configured bridges several times in the first few minutes. */ download_status_reset(&bridge->fetch_status); + /* it's here; schedule its re-fetch for a long time from now. */ + bridge->fetch_status.next_attempt_at += + get_options()->TestingBridgeDownloadInitialDelay; } node = node_get_mutable_by_id(ri->cache_info.identity_digest); @@ -982,8 +992,10 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) entry_guard_learned_bridge_identity(&bridge->addrport_configured, (const uint8_t*)ri->cache_info.identity_digest); - log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname, - from_cache ? "cached" : "fresh", router_describe(ri)); + if (desc_is_new) + log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", + ri->nickname, + from_cache ? "cached" : "fresh", router_describe(ri)); /* If we didn't have a reachable bridge before this one, try directory * documents again. */ if (first) { diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h index a42363f683..dd3e498a0a 100644 --- a/src/feature/client/bridges.h +++ b/src/feature/client/bridges.h @@ -46,7 +46,8 @@ void learned_router_identity(const tor_addr_t *addr, uint16_t port, void bridge_add_from_config(struct bridge_line_t *bridge_line); void retry_bridge_descriptor_fetch_directly(const char *digest); void fetch_bridge_descriptors(const or_options_t *options, time_t now); -void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); +void learned_bridge_descriptor(routerinfo_t *ri, + int from_cache, int desc_is_new); const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port); diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 9c0ecc56ad..ff9e05a645 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -363,6 +363,17 @@ pathbias_should_count(origin_circuit_t *circ) return 0; } + /* Ignore circuits where the controller helped choose the path. When + * this happens, we can't be sure whether the path was chosen randomly + * or not. */ + if (circ->any_hop_from_controller) { + /* (In this case, we _don't_ check to see if shouldcount is changing, + * since it's possible that an already-created circuit later gets extended + * by the controller. */ + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + /* Completely ignore one hop circuits */ if (circ->build_state->onehop_tunnel || circ->build_state->desired_path_len == 1) { diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 502cb99690..15f29d1c3e 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -132,6 +132,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control_events.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/describe.h" #include "feature/nodelist/microdesc.h" @@ -559,7 +560,7 @@ get_extreme_restriction_threshold(void) int32_t pct = networkstatus_get_param(NULL, "guard-extreme-restriction-percent", DFLT_EXTREME_RESTRICTION_PERCENT, - 1, INT32_MAX); + 1, 100); return pct / 100.0; } @@ -576,6 +577,18 @@ mark_guard_maybe_reachable(entry_guard_t *guard) guard->is_reachable = GUARD_REACHABLE_MAYBE; if (guard->is_filtered_guard) guard->is_usable_filtered_guard = 1; + + /* Check if it is a bridge and we don't have its descriptor yet */ + if (guard->bridge_addr && !guard_has_descriptor(guard)) { + /* Reset the descriptor fetch retry schedule, so it gives it another + * go soon. It's important to keep any "REACHABLE_MAYBE" bridges in + * sync with the descriptor fetch schedule, since we will refuse to + * use the network until our first primary bridges are either + * known-usable or known-unusable. See bug 40396. */ + download_status_t *dl = get_bridge_dl_status_by_id(guard->identity); + if (dl) + download_status_reset(dl); + } } /** @@ -2046,6 +2059,14 @@ entry_guard_consider_retry(entry_guard_t *guard) get_retry_schedule(guard->failing_since, now, guard->is_primary); const time_t last_attempt = guard->last_tried_to_connect; + /* Check if it is a bridge and we don't have its descriptor yet */ + if (guard->bridge_addr && !guard_has_descriptor(guard)) { + /* We want to leave the retry schedule to fetch_bridge_descriptors(), + * so we don't have two retry schedules clobbering each other. See + * bugs 40396 and 40497 for details of why we need this exception. */ + return; + } + if (BUG(last_attempt == 0) || now >= last_attempt + delay) { /* We should mark this retriable. */ @@ -2271,6 +2292,13 @@ entry_guards_note_guard_failure(guard_selection_t *gs, guard->is_primary?"primary ":"", guard->confirmed_idx>=0?"confirmed ":"", entry_guard_describe(guard)); + + /* Schedule a re-assessment of whether we have enough dir info to + * use the network. Counterintuitively, *losing* a bridge might actually + * be just what we need to *resume* using the network, if we had it in + * state GUARD_REACHABLE_MAYBE and we were stalling to learn this + * outcome. See bug 40396 for more details. */ + router_dir_info_changed(); } /** @@ -2295,6 +2323,12 @@ entry_guards_note_guard_success(guard_selection_t *gs, /* If guard was not already marked as reachable, send a GUARD UP signal */ if (guard->is_reachable != GUARD_REACHABLE_YES) { control_event_guard(guard->nickname, guard->identity, "UP"); + + /* Schedule a re-assessment of whether we have enough dir info to + * use the network. One of our guards has just moved to + * GUARD_REACHABLE_YES, so maybe we can resume using the network + * now. */ + router_dir_info_changed(); } guard->is_reachable = GUARD_REACHABLE_YES; @@ -2709,7 +2743,7 @@ entry_guards_upgrade_waiting_circuits(guard_selection_t *gs, {NONPRIMARY_GUARD_CONNECT_TIMEOUT} seconds." */ circuit_guard_state_t *state = origin_circuit_get_guard_state(circ); - if BUG((state == NULL)) + if (BUG(state == NULL)) continue; if (state->state != GUARD_CIRC_STATE_COMPLETE) continue; @@ -3538,6 +3572,11 @@ entry_guards_changed_for_guard_selection(guard_selection_t *gs) entry_guards_update_guards_in_state() */ or_state_mark_dirty(get_or_state(), when); + + /* Schedule a re-assessment of whether we have enough dir info to + * use the network. When we add or remove or disable or enable a + * guard, the decision could shift. */ + router_dir_info_changed(); } /** Our list of entry guards has changed for the default guard selection @@ -3930,6 +3969,253 @@ guard_selection_free_(guard_selection_t *gs) tor_free(gs); } +/**********************************************************************/ + +/** Layer2 guard subsystem (vanguards-lite) used for onion service circuits */ + +/** A simple representation of a layer2 guard. We just need its identity so + * that we feed it into a routerset, and a sampled timestamp to do expiration + * checks. */ +typedef struct layer2_guard_t { + /** Identity of the guard */ + char identity[DIGEST_LEN]; + /** When does this guard expire? (randomized timestamp) */ + time_t expire_on_date; +} layer2_guard_t; + +#define layer2_guard_free(val) \ + FREE_AND_NULL(layer2_guard_t, layer2_guard_free_, (val)) + +/** Return true if the vanguards-lite subsystem is enabled */ +bool +vanguards_lite_is_enabled(void) +{ + /* First check torrc option and then maybe also the consensus parameter. */ + const or_options_t *options = get_options(); + + /* If the option is explicitly disabled, that's the final word here */ + if (options->VanguardsLiteEnabled == 0) { + return false; + } + + /* If the option is set to auto, then check the consensus parameter */ + if (options->VanguardsLiteEnabled == -1) { + return networkstatus_get_param(NULL, "vanguards-lite-enabled", + 1, /* default to "on" */ + 0, 1); + } + + /* else it's enabled */ + tor_assert_nonfatal(options->VanguardsLiteEnabled == 1); + return options->VanguardsLiteEnabled; +} + +static void +layer2_guard_free_(layer2_guard_t *l2) +{ + if (!l2) { + return; + } + + tor_free(l2); +} + +/** Global list and routerset of L2 guards. They are both synced and they get + * updated periodically. We need both the list and the routerset: we use the + * smartlist to keep track of expiration times and the routerset is what we + * return to the users of this subsystem. */ +static smartlist_t *layer2_guards = NULL; +static routerset_t *layer2_routerset = NULL; + +/** Number of L2 guards */ +#define NUMBER_SECOND_GUARDS 4 +/** Make sure that the number of L2 guards is less than the number of + * MAX_SANE_RESTRICTED_NODES */ +CTASSERT(NUMBER_SECOND_GUARDS < 20); + +/** Lifetime of L2 guards: + * 1 to 12 days, for an average of a week using the max(x,x) distribution */ +#define MIN_SECOND_GUARD_LIFETIME (3600*24) +#define MAX_SECOND_GUARD_LIFETIME (3600*24*12) + +/** Return the number of guards our L2 guardset should have */ +static int +get_number_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-number", + NUMBER_SECOND_GUARDS, + 1, 19); +} + +/** Return the minimum lifetime of L2 guards */ +static int +get_min_lifetime_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-lifetime-min", + MIN_SECOND_GUARD_LIFETIME, + 1, INT32_MAX); +} + +/** Return the maximum lifetime of L2 guards */ +static int +get_max_lifetime_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-lifetime-max", + MAX_SECOND_GUARD_LIFETIME, + 1, INT32_MAX); +} + +/** + * Sample and return a lifetime for an L2 guard. + * + * Lifetime randomized uniformly between min and max consensus params. + */ +static int +get_layer2_hs_guard_lifetime(void) +{ + int min = get_min_lifetime_of_layer2_hs_guards(); + int max = get_max_lifetime_of_layer2_hs_guards(); + + if (BUG(min >= max)) { + return min; + } + + return crypto_rand_int_range(min, max); +} + +/** Maintain the L2 guard list. Make sure the list contains enough guards, do + * expirations as necessary, and keep all the data structures of this + * subsystem synchronized */ +void +maintain_layer2_guards(void) +{ + if (!router_have_minimum_dir_info()) { + return; + } + + /* Create the list if it doesn't exist */ + if (!layer2_guards) { + layer2_guards = smartlist_new(); + } + + /* Go through the list and perform any needed expirations */ + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + /* Expire based on expiration date */ + if (g->expire_on_date <= approx_time()) { + log_info(LD_GENERAL, "Removing expired Layer2 guard %s", + safe_str_client(hex_str(g->identity, DIGEST_LEN))); + // Nickname may be gone from consensus and doesn't matter anyway + control_event_guard("None", g->identity, "BAD_L2"); + layer2_guard_free(g); + SMARTLIST_DEL_CURRENT_KEEPORDER(layer2_guards, g); + continue; + } + + /* Expire if relay has left consensus */ + if (router_get_consensus_status_by_id(g->identity) == NULL) { + log_info(LD_GENERAL, "Removing missing Layer2 guard %s", + safe_str_client(hex_str(g->identity, DIGEST_LEN))); + // Nickname may be gone from consensus and doesn't matter anyway + control_event_guard("None", g->identity, "BAD_L2"); + layer2_guard_free(g); + SMARTLIST_DEL_CURRENT_KEEPORDER(layer2_guards, g); + continue; + } + } SMARTLIST_FOREACH_END(g); + + /* Find out how many guards we need to add */ + int new_guards_needed_n = + get_number_of_layer2_hs_guards() - smartlist_len(layer2_guards); + if (new_guards_needed_n <= 0) { + return; + } + + log_info(LD_GENERAL, "Adding %d guards to Layer2 routerset", + new_guards_needed_n); + + /* Add required guards to the list */ + smartlist_t *excluded = smartlist_new(); + for (int i = 0; i < new_guards_needed_n; i++) { + const node_t *choice = NULL; + const or_options_t *options = get_options(); + /* Pick Stable nodes */ + router_crn_flags_t flags = CRN_NEED_DESC|CRN_NEED_UPTIME; + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + if (!choice) { + break; + } + + /* We found our node: create an L2 guard out of it */ + layer2_guard_t *layer2_guard = tor_malloc_zero(sizeof(layer2_guard_t)); + memcpy(layer2_guard->identity, choice->identity, DIGEST_LEN); + layer2_guard->expire_on_date = approx_time() + + get_layer2_hs_guard_lifetime(); + smartlist_add(layer2_guards, layer2_guard); + log_info(LD_GENERAL, "Adding Layer2 guard %s", + safe_str_client(hex_str(layer2_guard->identity, DIGEST_LEN))); + // Nickname can also be None here because it is looked up later + control_event_guard("None", layer2_guard->identity, + "GOOD_L2"); + /* Exclude this node and its family so that we don't double-pick. */ + nodelist_add_node_and_family(excluded, choice); + } + + /* Some cleanup */ + smartlist_free(excluded); + + /* Now that the list is up to date, synchronize the routerset */ + routerset_free(layer2_routerset); + layer2_routerset = routerset_new(); + + SMARTLIST_FOREACH_BEGIN (layer2_guards, layer2_guard_t *, g) { + routerset_parse(layer2_routerset, + hex_str(g->identity, DIGEST_LEN), + "l2 guards"); + } SMARTLIST_FOREACH_END(g); +} + +/** + * Reset vanguards-lite list(s). + * + * Used for SIGNAL NEWNYM. + */ +void +purge_vanguards_lite(void) +{ + if (!layer2_guards) + return; + + /* Go through the list and perform any needed expirations */ + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + layer2_guard_free(g); + } SMARTLIST_FOREACH_END(g); + + smartlist_clear(layer2_guards); + + /* Pick new l2 guards */ + maintain_layer2_guards(); +} + +/** Return a routerset containing the L2 guards or NULL if it's not yet + * initialized. Callers must not free the routerset. Designed for use in + * pick_vanguard_middle_node() and should not be used anywhere else. Do not + * store this pointer -- any future calls to maintain_layer2_guards() and + * purge_vanguards_lite() can invalidate it. */ +const routerset_t * +get_layer2_guards(void) +{ + if (!layer2_guards) { + maintain_layer2_guards(); + } + + return layer2_routerset; +} + +/*****************************************************************************/ + /** Release all storage held by the list of entry guards and related * memory structs. */ void @@ -3946,4 +4232,15 @@ entry_guards_free_all(void) guard_contexts = NULL; } circuit_build_times_free_timeouts(get_circuit_build_times_mutable()); + + if (!layer2_guards) { + return; + } + + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + layer2_guard_free(g); + } SMARTLIST_FOREACH_END(g); + + smartlist_free(layer2_guards); + routerset_free(layer2_routerset); } diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h index 88ed8f649e..08fd7cf745 100644 --- a/src/feature/client/entrynodes.h +++ b/src/feature/client/entrynodes.h @@ -651,4 +651,9 @@ guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, int orig_bandwidth, uint32_t guardfraction_percentage); +bool vanguards_lite_is_enabled(void); +const routerset_t *get_layer2_guards(void); +void maintain_layer2_guards(void); +void purge_vanguards_lite(void); + #endif /* !defined(TOR_ENTRYNODES_H) */ diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 167beb96c6..80903ac9e5 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -843,7 +843,7 @@ handle_methods_done(const managed_proxy_t *mp) tor_assert(mp->transports); if (smartlist_len(mp->transports) == 0) - log_notice(LD_GENERAL, "Managed proxy '%s' was spawned successfully, " + log_warn(LD_GENERAL, "Managed proxy '%s' was spawned successfully, " "but it didn't launch any pluggable transport listeners!", mp->argv[0]); @@ -903,14 +903,22 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) goto err; + /* Log the error but do not kill the managed proxy. + * A proxy may contain several transports and if one + * of them is misconfigured, we still want to use + * the other transports. A managed proxy with no usable + * transports will log a warning. + * See https://gitlab.torproject.org/tpo/core/tor/-/issues/7362 + * */ parse_client_method_error(line); - goto err; + return; } else if (!strcmpstart(line, PROTO_SMETHOD_ERROR)) { if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) goto err; + /* Log the error but do not kill the managed proxy */ parse_server_method_error(line); - goto err; + return; } else if (!strcmpstart(line, PROTO_CMETHOD)) { if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) goto err; diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index bd0d41d29e..b19935e456 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -817,6 +817,8 @@ handle_control_extendcircuit(control_connection_t *conn, circ->first_hop_from_controller = 1; } + circ->any_hop_from_controller = 1; + /* now circ refers to something that is ready to be extended */ first_node = zero_circ; SMARTLIST_FOREACH(nodes, const node_t *, node, @@ -1075,7 +1077,7 @@ static const control_cmd_syntax_t redirectstream_syntax = { .max_args = UINT_MAX, // XXX should be 3. }; -/** Called when we receive a REDIRECTSTERAM command. Try to change the target +/** Called when we receive a REDIRECTSTREAM command. Try to change the target * address of the named AP stream, and report success or failure. */ static int handle_control_redirectstream(control_connection_t *conn, diff --git a/src/feature/dirauth/dirauth_options.inc b/src/feature/dirauth/dirauth_options.inc index 05726b8c2f..4fd07a8859 100644 --- a/src/feature/dirauth/dirauth_options.inc +++ b/src/feature/dirauth/dirauth_options.inc @@ -27,6 +27,10 @@ CONF_VAR(AuthDirHasIPv6Connectivity, BOOL, 0, "0") * good. */ CONF_VAR(AuthDirListBadExits, BOOL, 0, "0") +/** True iff we should list middle-only relays, and vote for all other + * relays as possibly suitable for other positions. */ +CONF_VAR(AuthDirListMiddleOnly, BOOL, 0, "0") + /** Do not permit more than this number of servers per IP address. */ CONF_VAR(AuthDirMaxServersPerAddr, POSINT, 0, "2") @@ -109,4 +113,7 @@ CONF_VAR(VersioningAuthoritativeDirectory, BOOL, 0, "0") * pressure or not. */ CONF_VAR(AuthDirRejectRequestsUnderLoad, BOOL, 0, "1") +/** Boolean: Should we not give bandwidth weight measurements to dirauths? */ +CONF_VAR(AuthDirDontVoteOnDirAuthBandwidth, BOOL, 0, "1") + END_CONF_STRUCT(dirauth_options_t) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index fa906c0c3c..cdd2c132ef 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -1479,6 +1479,21 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes) return result; } +/** Helper: Takes a smartlist of `const char *` flags, and a flag to remove. + * + * Removes that flag if it is present in the list. Doesn't free it. + */ +static void +remove_flag(smartlist_t *sl, const char *flag) +{ + /* We can't use smartlist_string_remove() here, since that doesn't preserve + * order, and since it frees elements from the string. */ + + int idx = smartlist_string_pos(sl, flag); + if (idx >= 0) + smartlist_del_keeporder(sl, idx); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -1633,6 +1648,9 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(votesec_list); tor_free(distsec_list); } + // True if anybody is voting on the BadExit flag. + const bool badexit_flag_is_listed = + smartlist_contains_string(flags, "BadExit"); chunks = smartlist_new(); @@ -1924,7 +1942,7 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0, is_valid = 0; - int is_guard = 0, is_exit = 0, is_bad_exit = 0; + int is_guard = 0, is_exit = 0, is_bad_exit = 0, is_middle_only = 0; int naming_conflict = 0; int n_listing = 0; char microdesc_digest[DIGEST256_LEN]; @@ -2055,7 +2073,6 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Set the flags. */ - smartlist_add(chosen_flags, (char*)"s"); /* for the start of the line. */ SMARTLIST_FOREACH_BEGIN(flags, const char *, fl) { if (!strcmp(fl, "Named")) { if (is_named) @@ -2077,6 +2094,8 @@ networkstatus_compute_consensus(smartlist_t *votes, is_running = 1; else if (!strcmp(fl, "BadExit")) is_bad_exit = 1; + else if (!strcmp(fl, "MiddleOnly")) + is_middle_only = 1; else if (!strcmp(fl, "Valid")) is_valid = 1; } @@ -2093,6 +2112,22 @@ networkstatus_compute_consensus(smartlist_t *votes, if (!is_valid) continue; + /* Starting with consensus method 32, we handle the middle-only + * flag specially: when it is present, we clear some flags, and + * set others. */ + if (is_middle_only && consensus_method >= MIN_METHOD_FOR_MIDDLEONLY) { + remove_flag(chosen_flags, "Exit"); + remove_flag(chosen_flags, "V2Dir"); + remove_flag(chosen_flags, "Guard"); + remove_flag(chosen_flags, "HSDir"); + is_exit = is_guard = 0; + if (! is_bad_exit && badexit_flag_is_listed) { + is_bad_exit = 1; + smartlist_add(chosen_flags, (char *)"BadExit"); + smartlist_sort_strings(chosen_flags); // restore order. + } + } + /* Pick the version. */ if (smartlist_len(versions)) { sort_version_list(versions, 0); @@ -2253,6 +2288,8 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_asprintf(chunks, "m %s\n", m); } /* Next line is all flags. The "\n" is missing. */ + smartlist_add_asprintf(chunks, "s%s", + smartlist_len(chosen_flags)?" ":""); smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -2265,7 +2302,8 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list); } /* Now the weight line. */ - if (rs_out.has_bandwidth) { + if (rs_out.has_bandwidth && (!rs_out.is_authority || + !dirauth_get_options()->AuthDirDontVoteOnDirAuthBandwidth)) { char *guardfraction_str = NULL; int unmeasured = rs_out.bw_is_unmeasured; @@ -4581,6 +4619,7 @@ const char DIRVOTE_UNIVERSAL_FLAGS[] = * depending on our configuration. */ const char DIRVOTE_OPTIONAL_FLAGS[] = "BadExit " + "MiddleOnly " "Running"; /** Return a new networkstatus_t* containing our current opinion. (For v3 @@ -4598,7 +4637,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_t *routers, *routerstatuses; char identity_digest[DIGEST_LEN]; char signing_key_digest[DIGEST_LEN]; - const int listbadexits = d_options->AuthDirListBadExits; + const int list_bad_exits = d_options->AuthDirListBadExits; + const int list_middle_only = d_options->AuthDirListMiddleOnly; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; @@ -4703,7 +4743,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now, - listbadexits); + list_bad_exits, + list_middle_only); if (ri->cache_info.signing_key_cert) { memcpy(vrs->ed25519_id, @@ -4825,8 +4866,10 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); - if (listbadexits) + if (list_bad_exits) smartlist_add_strdup(v3_out->known_flags, "BadExit"); + if (list_middle_only) + smartlist_add_strdup(v3_out->known_flags, "MiddleOnly"); smartlist_sort_strings(v3_out->known_flags); if (d_options->ConsensusParams) { diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index 3420098315..64aaec116e 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -53,7 +53,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 28 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 31 +#define MAX_SUPPORTED_CONSENSUS_METHOD 32 /** * Lowest consensus method where microdescriptor lines are put in canonical @@ -70,6 +70,10 @@ */ #define MIN_METHOD_FOR_CORRECT_BWWEIGHTSCALE 31 +/** Lowest consensus method for which we handle the MiddleOnly flag specially. + */ +#define MIN_METHOD_FOR_MIDDLEONLY 32 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index eca987b8b5..a75f516dca 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -226,6 +226,8 @@ dirserv_load_fingerprint_file(void) add_status = RTR_BADEXIT; } else if (!strcasecmp(nickname, "!invalid")) { add_status = RTR_INVALID; + } else if (!strcasecmp(nickname, "!middleonly")) { + add_status = RTR_MIDDLEONLY; } /* Check if fingerprint is RSA or ed25519 by verifying it. */ @@ -412,11 +414,11 @@ dirserv_rejects_tor_version(const char *platform, return true; } - /* Series between Tor 0.3.6 and 0.4.1 inclusive are unsupported. Reject - * them. 0.3.6.0-alpha-dev only existed for a short time, before it was - * renamed to 0.4.0.0-alpha-dev. */ + /* Series between Tor 0.3.6.x and 0.4.5.5-rc inclusive are unsupported. + * Reject them. 0.3.6.0-alpha-dev only existed for a short time, before it + * was renamed to 0.4.0.0-alpha-dev. */ if (tor_version_as_new_as(platform,"0.3.6.0-alpha-dev") && - !tor_version_as_new_as(platform,"0.4.2.1-alpha")) { + !tor_version_as_new_as(platform,"0.4.5.6")) { if (msg) { *msg = please_upgrade_string; } @@ -496,6 +498,13 @@ dirserv_get_status_impl(const char *id_digest, result |= RTR_BADEXIT; } + if (authdir_policy_middleonly_address(ipv4_addr, ipv4_orport)) { + log_fn(severity, LD_DIRSERV, + "Marking '%s' as middle-only because of address '%s'", + nickname, fmt_addr(ipv4_addr)); + result |= RTR_MIDDLEONLY; + } + if (!authdir_policy_permits_address(ipv4_addr, ipv4_orport)) { log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'", nickname, fmt_addr(ipv4_addr)); @@ -630,6 +639,7 @@ dirserv_set_node_flags_from_authoritative_status(node_t *node, { node->is_valid = (authstatus & RTR_INVALID) ? 0 : 1; node->is_bad_exit = (authstatus & RTR_BADEXIT) ? 1 : 0; + node->is_middle_only = (authstatus & RTR_MIDDLEONLY) ? 1 : 0; } /** True iff <b>a</b> is more severe than <b>b</b>. */ @@ -963,6 +973,11 @@ directory_remove_invalid(void) (r & RTR_BADEXIT) ? "bad" : "good"); node->is_bad_exit = (r&RTR_BADEXIT) ? 1: 0; } + if (bool_neq((r & RTR_MIDDLEONLY), node->is_middle_only)) { + log_info(LD_DIRSERV, "Router '%s' is now %smiddle-only", description, + (r & RTR_MIDDLEONLY) ? "" : "not"); + node->is_middle_only = (r&RTR_MIDDLEONLY) ? 1: 0; + } } SMARTLIST_FOREACH_END(node); routerlist_assert_ok(rl); diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index 6c056d11dd..a509eb1fbe 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -45,7 +45,8 @@ typedef struct authdir_config_t { #define RTR_REJECT 4 /**< We will not publish this router. */ /* 8 Historically used to avoid using this as a dir. */ #define RTR_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ -/* 32 Historically used to indicade Unnamed */ +/** We'll vote to only use this router as a midpoint. */ +#define RTR_MIDDLEONLY 32 #endif /* defined(PROCESS_DESCS_PRIVATE) || defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index d755a270be..05c19ff501 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -565,7 +565,8 @@ dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, const routerinfo_t *ri, time_t now, - int listbadexits) + int listbadexits, + int listmiddleonly) { const or_options_t *options = get_options(); uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri); @@ -597,6 +598,14 @@ dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, /* Override rs->is_bad_exit */ rs->is_bad_exit = listbadexits && node->is_bad_exit; + /* Override rs->is_middle_only and related flags. */ + rs->is_middle_only = listmiddleonly && node->is_middle_only; + if (rs->is_middle_only) { + if (listbadexits) + rs->is_bad_exit = 1; + rs->is_exit = rs->is_possible_guard = rs->is_hs_dir = rs->is_v2_dir = 0; + } + /* Set rs->is_staledesc. */ rs->is_staledesc = (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 818a0bafd2..8371f1c315 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -22,7 +22,8 @@ void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, const routerinfo_t *ri, time_t now, - int listbadexits); + int listbadexits, + int listmiddleonly); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #endif /* defined(HAVE_MODULE_DIRAUTH) */ diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 7fdb1bc70f..7319b96caf 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1569,6 +1569,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, char *url = NULL; const or_options_t *options = get_options(); + (void) body_len; + log_debug(LD_DIRSERV,"Received POST command."); conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h index ed6b00647e..ac45f3787b 100644 --- a/src/feature/dirclient/dir_server_st.h +++ b/src/feature/dirclient/dir_server_st.h @@ -16,6 +16,8 @@ #include "core/or/or.h" #include "feature/nodelist/routerstatus_st.h" +struct smartlist_t; + /** Represents information about a single trusted or fallback directory * server. */ struct dir_server_t { @@ -48,6 +50,10 @@ struct dir_server_t { time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ + /** Authority only. Can be null. If present, a list of auth_dirport_t + * representing HTTP dirports for this authority. */ + struct smartlist_t *auth_dirports; + routerstatus_t fake_status; /**< Used when we need to pass this trusted * dir_server_t to * directory_request_set_routerstatus. diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 0b6a8101a5..4e9c8e2f45 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -1134,6 +1134,7 @@ directory_request_set_routerstatus(directory_request_t *req, { req->routerstatus = status; } + /** * Helper: update the addresses, ports, and identities in <b>req</b> * from the routerstatus object in <b>req</b>. Return 0 on success. @@ -1176,7 +1177,7 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } - /* At this point, if we are a client making a direct connection to a + /* At this point, if we are a client making a direct connection to a * directory server, we have selected a server that has at least one address * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if @@ -1191,6 +1192,37 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } + /* One last thing: If we're talking to an authority, we might want to use + * a special HTTP port for it based on our purpose. + */ + if (req->indirection == DIRIND_DIRECT_CONN && status->is_authority) { + const dir_server_t *ds = router_get_trusteddirserver_by_digest( + status->identity_digest); + if (ds) { + const tor_addr_port_t *v4 = NULL; + if (authdir_mode_v3(get_options())) { + // An authority connecting to another authority should always + // prefer the VOTING usage, if one is specifically configured. + v4 = trusted_dir_server_get_dirport_exact( + ds, AUTH_USAGE_VOTING, AF_INET); + } + if (! v4) { + // Everybody else should prefer a usage dependent on their + // the dir_purpose. + auth_dirport_usage_t usage = + auth_dirport_usage_for_purpose(req->dir_purpose); + v4 = trusted_dir_server_get_dirport(ds, usage, AF_INET); + } + tor_assert_nonfatal(v4); + if (v4) { + // XXXX We could, if we wanted, also select a v6 address. But a v4 + // address must exist here, and we as a relay are required to support + // ipv4. So we just that. + tor_addr_port_copy(&use_dir_ap, v4); + } + } + } + directory_request_set_or_addr_port(req, &use_or_ap); directory_request_set_dir_addr_port(req, &use_dir_ap); directory_request_set_directory_id_digest(req, status->identity_digest); @@ -1209,7 +1241,7 @@ directory_initiate_request,(directory_request_t *request)) tor_assert_nonfatal( ! directory_request_dir_contact_info_specified(request)); if (directory_request_set_dir_from_routerstatus(request) < 0) { - return; + return; // or here XXXX } } @@ -1324,6 +1356,8 @@ directory_initiate_request,(directory_request_t *request)) entry_guard_cancel(&guard_state); } + // XXXX This is the case where we replace. + switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c index 8be2983a5d..c21dd113b4 100644 --- a/src/feature/dirclient/dlstatus.c +++ b/src/feature/dirclient/dlstatus.c @@ -73,15 +73,14 @@ find_dl_min_delay(const download_status_t *dls, const or_options_t *options) } } case DL_SCHED_BRIDGE: - if (options->UseBridges && num_bridges_usable(0) > 0) { - /* A bridge client that is sure that one or more of its bridges are - * running can afford to wait longer to update bridge descriptors. */ - return options->TestingBridgeDownloadInitialDelay; - } else { - /* A bridge client which might have no running bridges, must try to - * get bridge descriptors straight away. */ - return options->TestingBridgeBootstrapDownloadInitialDelay; - } + /* Be conservative here: always return the 'during bootstrap' delay + * value, so we never delay while trying to fetch descriptors + * for new bridges. Once we do succeed at fetching a descriptor + * for our bridge, we will adjust its next_attempt_at based on + * the longer "TestingBridgeDownloadInitialDelay" value. See + * learned_bridge_descriptor() for details. + */ + return options->TestingBridgeBootstrapDownloadInitialDelay; default: tor_assert(0); } diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index c877227adc..323f2bd576 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -1128,7 +1128,7 @@ consdiff_get_digests(const smartlist_t *diff, { const cdline_t *line2 = smartlist_get(diff, 1); char *h = tor_memdup_nulterm(line2->s, line2->len); - smartlist_split_string(hash_words, h, " ", 0, 0); + smartlist_split_string(hash_words, h, " ", 0, 4); tor_free(h); } diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 947b3810a4..cd3e2731be 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -434,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_possible_guard = 1; else if (!strcmp(tok->args[i], "BadExit")) rs->is_bad_exit = 1; + else if (!strcmp(tok->args[i], "MiddleOnly")) + rs->is_middle_only = 1; else if (!strcmp(tok->args[i], "Authority")) rs->is_authority = 1; else if (!strcmp(tok->args[i], "Unnamed") && diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 73f9176186..a76893fe1a 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -548,15 +548,19 @@ config_service(config_line_t *line, const or_options_t *options, tor_assert(service->config.version <= HS_VERSION_MAX); - /* Check permission on service directory that was just parsed. And this must - * be done regardless of the service version. Do not ask for the directory - * to be created, this is done when the keys are loaded because we could be - * in validation mode right now. */ - if (hs_check_service_private_dir(options->User, - service->config.directory_path, - service->config.dir_group_readable, - 0) < 0) { - goto err; + /* If we're running with TestingTorNetwork enabled, we relax the permissions + * check on the hs directory. */ + if (!options->TestingTorNetwork) { + /* Check permission on service directory that was just parsed. And this + * must be done regardless of the service version. Do not ask for the + * directory to be created, this is done when the keys are loaded because + * we could be in validation mode right now. */ + if (hs_check_service_private_dir(options->User, + service->config.directory_path, + service->config.dir_group_readable, + 0) < 0) { + goto err; + } } /* We'll try to learn the service version here by loading the key(s) if @@ -640,6 +644,7 @@ hs_config_service_all(const or_options_t *options, int validate_only) int rv = config_service(section, options, new_service_list); config_free_lines(section); if (rv < 0) { + config_free_lines(remaining); goto err; } } diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 70ff4e9690..a37eab5b5d 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1607,8 +1607,8 @@ decrypt_desc_layer,(const hs_descriptor_t *desc, * put in decrypted_out which contains the superencrypted layer of the * descriptor. Return the length of decrypted_out on success else 0 is * returned and decrypted_out is set to NULL. */ -static size_t -desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out) +MOCK_IMPL(STATIC size_t, +desc_decrypt_superencrypted,(const hs_descriptor_t *desc,char **decrypted_out)) { size_t superencrypted_len = 0; char *superencrypted_plaintext = NULL; @@ -1639,10 +1639,10 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out) * decrypted_out which contains the encrypted layer of the descriptor. * Return the length of decrypted_out on success else 0 is returned and * decrypted_out is set to NULL. */ -static size_t -desc_decrypt_encrypted(const hs_descriptor_t *desc, - const curve25519_secret_key_t *client_auth_sk, - char **decrypted_out) +MOCK_IMPL(STATIC size_t, +desc_decrypt_encrypted,(const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + char **decrypted_out)) { size_t encrypted_len = 0; char *encrypted_plaintext = NULL; @@ -2145,7 +2145,7 @@ desc_decode_plaintext_v3(smartlist_t *tokens, /** Decode the version 3 superencrypted section of the given descriptor desc. * The desc_superencrypted_out will be populated with the decoded data. */ -static hs_desc_decode_status_t +STATIC hs_desc_decode_status_t desc_decode_superencrypted_v3(const hs_descriptor_t *desc, hs_desc_superencrypted_data_t * desc_superencrypted_out) @@ -2259,7 +2259,7 @@ desc_decode_superencrypted_v3(const hs_descriptor_t *desc, /** Decode the version 3 encrypted section of the given descriptor desc. The * desc_encrypted_out will be populated with the decoded data. */ -static hs_desc_decode_status_t +STATIC hs_desc_decode_status_t desc_decode_encrypted_v3(const hs_descriptor_t *desc, const curve25519_secret_key_t *client_auth_sk, hs_desc_encrypted_data_t *desc_encrypted_out) diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h index 7e437faeb8..d959431369 100644 --- a/src/feature/hs/hs_descriptor.h +++ b/src/feature/hs/hs_descriptor.h @@ -339,6 +339,25 @@ MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc, bool is_superencrypted_layer, char **decrypted_out)); +STATIC hs_desc_decode_status_t desc_decode_encrypted_v3( + const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + hs_desc_encrypted_data_t *desc_encrypted_out); + +STATIC hs_desc_decode_status_t +desc_decode_superencrypted_v3(const hs_descriptor_t *desc, + hs_desc_superencrypted_data_t * + desc_superencrypted_out); + +MOCK_DECL(STATIC size_t, desc_decrypt_encrypted,( + const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + char **decrypted_out)); + +MOCK_DECL(STATIC size_t, desc_decrypt_superencrypted,( + const hs_descriptor_t *desc, + char **decrypted_out)); + #endif /* defined(HS_DESCRIPTOR_PRIVATE) */ #endif /* !defined(TOR_HS_DESCRIPTOR_H) */ diff --git a/src/feature/hs/hs_metrics.c b/src/feature/hs/hs_metrics.c index e023eab90c..0f1824c51c 100644 --- a/src/feature/hs/hs_metrics.c +++ b/src/feature/hs/hs_metrics.c @@ -29,18 +29,6 @@ port_to_str(const uint16_t port) return buf; } -/** Return a static buffer pointer that contains a formatted label on the form - * of key=value. - * - * Subsequent call to this function invalidates the previous buffer. */ -static const char * -format_label(const char *key, const char *value) -{ - static char buf[128]; - tor_snprintf(buf, sizeof(buf), "%s=%s", key, value); - return buf; -} - /** Initialize a metrics store for the given service. * * Essentially, this goes over the base_metrics array and adds them all to the @@ -61,12 +49,12 @@ init_store(hs_service_t *service) /* Add labels to the entry. */ metrics_store_entry_add_label(entry, - format_label("onion", service->onion_address)); + metrics_format_label("onion", service->onion_address)); if (base_metrics[i].port_as_label && service->config.ports) { SMARTLIST_FOREACH_BEGIN(service->config.ports, const hs_port_config_t *, p) { metrics_store_entry_add_label(entry, - format_label("port", port_to_str(p->virtual_port))); + metrics_format_label("port", port_to_str(p->virtual_port))); } SMARTLIST_FOREACH_END(p); } } @@ -96,7 +84,7 @@ hs_metrics_update_by_service(const hs_metrics_key_t key, SMARTLIST_FOREACH_BEGIN(entries, metrics_store_entry_t *, entry) { if (port == 0 || metrics_store_entry_has_label(entry, - format_label("port", port_to_str(port)))) { + metrics_format_label("port", port_to_str(port)))) { metrics_store_entry_update(entry, n); break; } diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index 1f18bd71a2..1f1ac4d106 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -43,6 +43,14 @@ #include "feature/dirclient/dir_server_st.h" #include "feature/nodelist/node_st.h" +/** Information about an (HTTP) dirport for a directory authority. */ +struct auth_dirport_t { + /** What is the intended usage for this dirport? One of AUTH_USAGE_* */ + auth_dirport_usage_t usage; + /** What is the correct address/port ? */ + tor_addr_port_t dirport; +}; + /** Global list of a dir_server_t object for each directory * authority. */ static smartlist_t *trusted_dir_servers = NULL; @@ -66,6 +74,11 @@ add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir) /* IPv6 DirPort is not a thing yet for authorities. */ nodelist_add_addr_to_address_set(&dir->ipv6_addr, dir->ipv6_orport, 0); } + if (dir->auth_dirports) { + SMARTLIST_FOREACH_BEGIN(dir->auth_dirports, const auth_dirport_t *, p) { + nodelist_add_addr_to_address_set(&p->dirport.addr, 0, p->dirport.port); + } SMARTLIST_FOREACH_END(p); + } } /** Go over the trusted directory server list and add their address(es) to the @@ -256,7 +269,10 @@ MOCK_IMPL(int, router_digest_is_trusted_dir_type, /** Return true iff the given address matches a trusted directory that matches * at least one bit of type. * - * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. */ + * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. + * + * Only ORPorts' addresses are considered. + */ bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) { @@ -281,6 +297,39 @@ router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) return false; } +/** Return an appropriate usage value describing which authdir port to use + * for a given directory connection purpose. + */ +auth_dirport_usage_t +auth_dirport_usage_for_purpose(int purpose) +{ + switch (purpose) { + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + case DIR_PURPOSE_FETCH_CONSENSUS: + case DIR_PURPOSE_FETCH_CERTIFICATE: + case DIR_PURPOSE_FETCH_MICRODESC: + return AUTH_USAGE_DOWNLOAD; + + case DIR_PURPOSE_UPLOAD_DIR: + return AUTH_USAGE_UPLOAD; + + case DIR_PURPOSE_UPLOAD_VOTE: + case DIR_PURPOSE_UPLOAD_SIGNATURES: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_STATUS_VOTE: + return AUTH_USAGE_VOTING; + + case DIR_PURPOSE_SERVER: + case DIR_PURPOSE_UPLOAD_HSDESC: + case DIR_PURPOSE_FETCH_HSDESC: + case DIR_PURPOSE_HAS_FETCHED_HSDESC: + default: + tor_assert_nonfatal_unreached(); + return AUTH_USAGE_LEGACY; + } +} + /** Create a directory server at <b>address</b>:<b>port</b>, with OR identity * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, * add ourself. If <b>is_authority</b>, this is a directory authority. Return @@ -357,6 +406,7 @@ dir_server_new(int is_authority, ent->fake_status.ipv4_dirport = ent->ipv4_dirport; ent->fake_status.ipv4_orport = ent->ipv4_orport; ent->fake_status.ipv6_orport = ent->ipv6_orport; + ent->fake_status.is_authority = !! is_authority; return ent; } @@ -404,10 +454,98 @@ trusted_dir_server_new(const char *nickname, const char *address, ipv6_addrport, digest, v3_auth_digest, type, weight); + + if (ipv4_dirport) { + tor_addr_port_t p; + memset(&p, 0, sizeof(p)); + tor_addr_copy(&p.addr, &ipv4_addr); + p.port = ipv4_dirport; + trusted_dir_server_add_dirport(result, AUTH_USAGE_LEGACY, &p); + } tor_free(hostname); return result; } +/** + * Add @a dirport as an HTTP DirPort contact point for the directory authority + * @a ds, for use when contacting that authority for the given @a usage. + * + * Multiple ports of the same usage are allowed; if present, then only + * the first one of each address family is currently used. + */ +void +trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport) +{ + tor_assert(ds); + tor_assert(dirport); + + if (BUG(! ds->is_authority)) { + return; + } + + if (ds->auth_dirports == NULL) { + ds->auth_dirports = smartlist_new(); + } + + auth_dirport_t *port = tor_malloc_zero(sizeof(auth_dirport_t)); + port->usage = usage; + tor_addr_port_copy(&port->dirport, dirport); + smartlist_add(ds->auth_dirports, port); +} + +/** + * Helper for trusted_dir_server_get_dirport: only return the exact requested + * usage type. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport_exact(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + tor_assert(ds); + tor_assert_nonfatal(addr_family == AF_INET || addr_family == AF_INET6); + if (ds->auth_dirports == NULL) + return NULL; + + SMARTLIST_FOREACH_BEGIN(ds->auth_dirports, const auth_dirport_t *, port) { + if (port->usage == usage && + tor_addr_family(&port->dirport.addr) == addr_family) { + return &port->dirport; + } + } SMARTLIST_FOREACH_END(port); + + return NULL; +} + +/** + * Return the DirPort of the authority @a ds for with the usage type + * @a usage and address family @a addr_family. If none is found, try + * again with an AUTH_USAGE_LEGACY dirport, if there is one. Return NULL + * if no port can be found. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + const tor_addr_port_t *port; + + while (1) { + port = trusted_dir_server_get_dirport_exact(ds, usage, addr_family); + if (port) + return port; + + // If we tried LEGACY, there is no fallback from this point. + if (usage == AUTH_USAGE_LEGACY) + return NULL; + + // Try again with LEGACY. + usage = AUTH_USAGE_LEGACY; + } +} + /** Return a new dir_server_t for a fallback directory server at * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest * <b>id_digest</b> */ @@ -447,6 +585,10 @@ dir_server_free_(dir_server_t *ds) if (!ds) return; + if (ds->auth_dirports) { + SMARTLIST_FOREACH(ds->auth_dirports, auth_dirport_t *, p, tor_free(p)); + smartlist_free(ds->auth_dirports); + } tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h index f744fecf92..3b4faf07af 100644 --- a/src/feature/nodelist/dirlist.h +++ b/src/feature/nodelist/dirlist.h @@ -11,6 +11,28 @@ #ifndef TOR_DIRLIST_H #define TOR_DIRLIST_H +typedef struct auth_dirport_t auth_dirport_t; +/** + * Different usages for an authority's HTTP directory port. + * + * Historically, only legacy ports existed; proposal 330 added multiple types + * of dirport to better enable authorities to offload work and resist DoS + * attacks. + **/ +typedef enum auth_dirport_usage_t { + /** Flag for an authority's dirport that is intended for misc/legacy + * usage. May be used when no other dirport is available. */ + AUTH_USAGE_LEGACY, + /** Flag for an authority's dirport that is intended for descriptor uploads + * only. */ + AUTH_USAGE_UPLOAD, + /** Flag for an authority's dirport that is intended for voting only */ + AUTH_USAGE_VOTING, + /** Flag for an authority's dirport that is intended for relay downloads + * only. */ + AUTH_USAGE_DOWNLOAD, +} auth_dirport_usage_t; + int get_n_authorities(dirinfo_type_t type); const smartlist_t *router_get_trusted_dir_servers(void); const smartlist_t *router_get_fallback_dir_servers(void); @@ -18,6 +40,8 @@ smartlist_t *router_get_trusted_dir_servers_mutable(void); smartlist_t *router_get_fallback_dir_servers_mutable(void); void mark_all_dirservers_up(smartlist_t *server_list); +auth_dirport_usage_t auth_dirport_usage_for_purpose(int purpose); + dir_server_t *router_get_trusteddirserver_by_digest(const char *d); dir_server_t *router_get_fallback_dirserver_by_digest( const char *digest); @@ -28,6 +52,14 @@ MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, MOCK_DECL(int, router_digest_is_trusted_dir_type, (const char *digest, dirinfo_type_t type)); +const tor_addr_port_t *trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); +const tor_addr_port_t *trusted_dir_server_get_dirport_exact( + const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); + bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type); #define router_addr_is_trusted_dir(d) \ @@ -41,6 +73,9 @@ dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, const tor_addr_port_t *addrport_ipv6, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight); +void trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport); dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, uint16_t dir_port, uint16_t or_port, const tor_addr_port_t *addrport_ipv6, diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index 6db40c0b68..95379a7721 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -87,7 +87,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -95,6 +95,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_fast?" Fast":"", rs->is_possible_guard?" Guard":"", rs->is_hs_dir?" HSDir":"", + rs->is_middle_only?" MiddleOnly":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", rs->is_staledesc?" StaleDesc":"", diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index af808a6ba7..77e2b547f5 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -45,6 +45,8 @@ #include "core/or/channel.h" #include "core/or/channelpadding.h" #include "core/or/circuitpadding.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" #include "core/or/circuitmux.h" #include "core/or/circuitmux_ewma.h" #include "core/or/circuitstats.h" @@ -1664,7 +1666,6 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c, dos_consensus_has_changed(new_c); relay_consensus_has_changed(new_c); hs_dos_consensus_has_changed(new_c); - rep_hist_consensus_has_changed(new_c); } /* Called after a new consensus has been put in the global state. It is safe @@ -1701,6 +1702,11 @@ notify_after_networkstatus_changes(void) channelpadding_new_consensus_params(c); circpad_new_consensus_params(c); router_new_consensus_params(c); + congestion_control_new_consensus_params(c); + flow_control_new_consensus_params(c); + + /* Maintenance of our L2 guard list */ + maintain_layer2_guards(); } /** Copy all the ancillary information (like router download status and so on) diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h index b15e7154c4..df67a47ada 100644 --- a/src/feature/nodelist/node_st.h +++ b/src/feature/nodelist/node_st.h @@ -70,6 +70,8 @@ struct node_t { unsigned int is_exit:1; /**< Do we think this is an OK exit? */ unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked, * or otherwise nasty? */ + /** Is this unsuitable for use as anything besides a middle relay? */ + unsigned int is_middle_only:1; unsigned int is_hs_dir:1; /**< True iff this router is a hidden service * directory according to the authorities. */ diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 121dc8823a..c676e8dfb4 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -2820,6 +2820,7 @@ update_router_have_minimum_dir_info(void) const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); int using_md; + static int be_loud_when_things_work_again = 0; if (!consensus) { if (!networkstatus_get_latest_consensus()) @@ -2875,8 +2876,9 @@ update_router_have_minimum_dir_info(void) if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); - log_info(LD_DIR, - "We now have enough directory information to build circuits."); + tor_log(be_loud_when_things_work_again ? LOG_NOTICE : LOG_INFO, LD_DIR, + "We now have enough directory information to build circuits."); + be_loud_when_things_work_again = 0; } /* If paths have just become unavailable in this update. */ @@ -2885,6 +2887,10 @@ update_router_have_minimum_dir_info(void) tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, "Our directory information is no longer up-to-date " "enough to build circuits: %s", dir_info_status); + if (!quiet) { + /* remember to do a notice-level log when things come back */ + be_loud_when_things_work_again = 1; + } /* a) make us log when we next complete a circuit, so we know when Tor * is back up and usable, and b) disable some activities that Tor diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 565d4596d4..c00f7ffb26 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -1617,6 +1617,13 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, "descriptor for router %s", router_describe(router)); } else { + if (router->purpose == ROUTER_PURPOSE_BRIDGE) { + /* Even if we're not going to keep this descriptor, we need to + * let the bridge descriptor fetch subsystem know that we + * succeeded at getting it -- so we can adjust the retry schedule + * to stop trying for a while. */ + learned_bridge_descriptor(router, from_cache, 0); + } log_info(LD_DIR, "Dropping descriptor that we already have for router %s", router_describe(router)); @@ -2012,6 +2019,30 @@ routerlist_remove_old_routers(void) router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store); } +/* Drop every bridge descriptor in our routerlist. Used by the external + * 'bridgestrap' tool to discard bridge descriptors so that it can then + * do a clean reachability test. */ +void +routerlist_drop_bridge_descriptors(void) +{ + routerinfo_t *router; + int i; + + if (!routerlist) + return; + + for (i = 0; i < smartlist_len(routerlist->routers); ++i) { + router = smartlist_get(routerlist->routers, i); + if (router->purpose == ROUTER_PURPOSE_BRIDGE) { + log_notice(LD_DIR, + "Dropping existing bridge descriptor for %s", + router_describe(router)); + routerlist_remove(routerlist, router, 0, time(NULL)); + i--; + } + } +} + /** We just added a new set of descriptors. Take whatever extra steps * we need. */ void @@ -2023,7 +2054,7 @@ routerlist_descriptors_added(smartlist_t *sl, int from_cache) control_event_descriptors_changed(sl); SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { if (ri->purpose == ROUTER_PURPOSE_BRIDGE) - learned_bridge_descriptor(ri, from_cache); + learned_bridge_descriptor(ri, from_cache, 1); if (ri->needs_retest_if_added) { ri->needs_retest_if_added = 0; dirserv_single_reachability_test(approx_time(), ri); diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index 7dc748c94b..7ba305baf6 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -145,6 +145,7 @@ was_router_added_t router_add_extrainfo_to_routerlist( int from_cache, int from_fetch); void routerlist_descriptors_added(smartlist_t *sl, int from_cache); void routerlist_remove_old_routers(void); +void routerlist_drop_bridge_descriptors(void); int router_load_single_router(const char *s, uint8_t purpose, int cache, const char **msg); int router_load_routers_from_string(const char *s, const char *eos, diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 46ff0bdeac..55b76de581 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -51,6 +51,8 @@ struct routerstatus_t { * choice as an entry guard. */ unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for * an exit node. */ + unsigned int is_middle_only:1; /**< True iff this node is marked as bad + * for anything besides middle positions. */ unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden * service directory. */ unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 6a703f2ab3..c6e0439338 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1539,6 +1539,16 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses, tor_addr_make_unspec(&addr); + /* Note down any DNS errors to the statistics module */ + if (result == DNS_ERR_TIMEOUT) { + /* libevent timed out while resolving a name. However, because libevent + * handles retries and timeouts internally, this means that all attempts of + * libevent timed out. If we wanted to get more granular information about + * individual libevent attempts, we would have to implement our own DNS + * timeout/retry logic */ + rep_hist_note_overload(OVERLOAD_GENERAL); + } + /* Keep track of whether IPv6 is working */ if (type == DNS_IPv6_AAAA) { if (result == DNS_ERR_TIMEOUT) { @@ -1642,7 +1652,7 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses, /* The result can be changed within this function thus why we note the result * at the end. */ - rep_hist_note_dns_query(type, result); + rep_hist_note_dns_error(type, result); tor_free(arg_); } @@ -1662,6 +1672,9 @@ launch_one_resolve(const char *address, uint8_t query_type, addr[0] = (char) query_type; memcpy(addr+1, address, addr_len + 1); + /* Note the query for our statistics. */ + rep_hist_note_dns_request(query_type); + switch (query_type) { case DNS_IPv4_A: req = evdns_base_resolve_ipv4(the_evdns_base, diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am index 84bb1ff35e..8a121cef01 100644 --- a/src/feature/relay/include.am +++ b/src/feature/relay/include.am @@ -15,6 +15,7 @@ MODULE_RELAY_SOURCES = \ src/feature/relay/routermode.c \ src/feature/relay/relay_config.c \ src/feature/relay/relay_handshake.c \ + src/feature/relay/relay_metrics.c \ src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ src/feature/relay/routerkeys.c \ @@ -30,6 +31,7 @@ noinst_HEADERS += \ src/feature/relay/onion_queue.h \ src/feature/relay/relay_config.h \ src/feature/relay/relay_handshake.h \ + src/feature/relay/relay_metrics.h \ src/feature/relay/relay_periodic.h \ src/feature/relay/relay_sys.h \ src/feature/relay/relay_find_addr.h \ diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 85ec0dc74a..c09f4d5b9b 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -164,6 +164,7 @@ onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin) #define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + rep_hist_note_circuit_handshake_dropped(onionskin->handshake_type); if (onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR) { char *m; /* Note this ntor onionskin drop as an overload */ diff --git a/src/feature/relay/relay_metrics.c b/src/feature/relay/relay_metrics.c new file mode 100644 index 0000000000..fc8eb10d1b --- /dev/null +++ b/src/feature/relay/relay_metrics.c @@ -0,0 +1,401 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.c + * @brief Relay metrics exposed through the MetricsPort + **/ + +#define RELAY_METRICS_ENTRY_PRIVATE + +#include "orconfig.h" + +#include "core/or/or.h" +#include "core/or/relay.h" + +#include "lib/malloc/malloc.h" +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_store.h" +#include "lib/log/util_bug.h" + +#include "feature/relay/relay_metrics.h" +#include "feature/stats/rephist.h" + +#include <event2/dns.h> + +/** Declarations of each fill function for metrics defined in base_metrics. */ +static void fill_dns_error_values(void); +static void fill_dns_query_values(void); +static void fill_global_bw_limit_values(void); +static void fill_socket_values(void); +static void fill_onionskins_values(void); +static void fill_oom_values(void); +static void fill_tcp_exhaustion_values(void); + +/** The base metrics that is a static array of metrics added to the metrics + * store. + * + * The key member MUST be also the index of the entry in the array. */ +static const relay_metrics_entry_t base_metrics[] = +{ + { + .key = RELAY_METRICS_NUM_OOM_BYTES, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_oom_bytes_total), + .help = "Total number of bytes the OOM has freed by subsystem", + .fill_fn = fill_oom_values, + }, + { + .key = RELAY_METRICS_NUM_ONIONSKINS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_onionskins_total), + .help = "Total number of onionskins handled", + .fill_fn = fill_onionskins_values, + }, + { + .key = RELAY_METRICS_NUM_SOCKETS, + .type = METRICS_TYPE_GAUGE, + .name = METRICS_NAME(relay_load_socket_total), + .help = "Total number of sockets", + .fill_fn = fill_socket_values, + }, + { + .key = RELAY_METRICS_NUM_GLOBAL_RW_LIMIT, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_global_rate_limit_reached_total), + .help = "Total number of global connection bucket limit reached", + .fill_fn = fill_global_bw_limit_values, + }, + { + .key = RELAY_METRICS_NUM_DNS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_query_total), + .help = "Total number of DNS queries done by this relay", + .fill_fn = fill_dns_query_values, + }, + { + .key = RELAY_METRICS_NUM_DNS_ERRORS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_error_total), + .help = "Total number of DNS errors encountered by this relay", + .fill_fn = fill_dns_error_values, + }, + { + .key = RELAY_METRICS_NUM_TCP_EXHAUSTION, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_tcp_exhaustion_total), + .help = "Total number of times we ran out of TCP ports", + .fill_fn = fill_tcp_exhaustion_values, + }, +}; +static const size_t num_base_metrics = ARRAY_LENGTH(base_metrics); + +/** The only and single store of all the relay metrics. */ +static metrics_store_t *the_store; + +/** Helper function to convert an handshake type into a string. */ +static inline const char * +handshake_type_to_str(const uint16_t type) +{ + switch (type) { + case ONION_HANDSHAKE_TYPE_TAP: + return "tap"; + case ONION_HANDSHAKE_TYPE_FAST: + return "fast"; + case ONION_HANDSHAKE_TYPE_NTOR: + return "ntor"; + default: + // LCOV_EXCL_START + tor_assert_unreached(); + // LCOV_EXCL_STOP + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_tcp_exhaustion_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_TCP_EXHAUSTION]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, rep_hist_get_n_tcp_exhaustion()); +} + +/* NOTE: Disable the record type label until libevent is fixed. */ +#if 0 +/** Helper array containing mapping for the name of the different DNS records + * and their corresponding libevent values. */ +static struct dns_type { + const char *name; + uint8_t type; +} dns_types[] = { + { .name = "A", .type = DNS_IPv4_A }, + { .name = "PTR", .type = DNS_PTR }, + { .name = "AAAA", .type = DNS_IPv6_AAAA }, +}; +static const size_t num_dns_types = ARRAY_LENGTH(dns_types); +#endif + +/** Fill function for the RELAY_METRICS_NUM_DNS_ERRORS metrics. */ +static void +fill_dns_error_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS_ERRORS]; + + /* Helper array to map libeven DNS errors to their names and so we can + * iterate over this array to add all metrics. */ + static struct dns_error { + const char *name; + uint8_t key; + } errors[] = { + { .name = "success", .key = DNS_ERR_NONE }, + { .name = "format", .key = DNS_ERR_FORMAT }, + { .name = "serverfailed", .key = DNS_ERR_SERVERFAILED }, + { .name = "notexist", .key = DNS_ERR_NOTEXIST }, + { .name = "notimpl", .key = DNS_ERR_NOTIMPL }, + { .name = "refused", .key = DNS_ERR_REFUSED }, + { .name = "truncated", .key = DNS_ERR_TRUNCATED }, + { .name = "unknown", .key = DNS_ERR_UNKNOWN }, + { .name = "tor_timeout", .key = DNS_ERR_TIMEOUT }, + { .name = "shutdown", .key = DNS_ERR_SHUTDOWN }, + { .name = "cancel", .key = DNS_ERR_CANCEL }, + { .name = "nodata", .key = DNS_ERR_NODATA }, + }; + static const size_t num_errors = ARRAY_LENGTH(errors); + + /* NOTE: Disable the record type label until libevent is fixed. */ +#if 0 + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + + for (size_t j = 0; j < num_errors; j++) { + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("reason", errors[j].name)); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_error(dns_types[i].type, errors[j].key)); + } + tor_free(record_label); + } +#endif + + /* Put in the DNS errors, unfortunately not per-type for now. */ + for (size_t j = 0; j < num_errors; j++) { + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("reason", errors[j].name)); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_error(0, errors[j].key)); + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_dns_query_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS]; + + /* NOTE: Disable the record type label until libevent is fixed (#40490). */ +#if 0 + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_request(dns_types[i].type)); + tor_free(record_label); + } +#endif + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, rep_hist_get_n_dns_request(0)); +} + +/** Fill function for the RELAY_METRICS_NUM_GLOBAL_RW_LIMIT metrics. */ +static void +fill_global_bw_limit_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_GLOBAL_RW_LIMIT]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "read")); + metrics_store_entry_update(sentry, rep_hist_get_n_read_limit_reached()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "write")); + metrics_store_entry_update(sentry, rep_hist_get_n_write_limit_reached()); +} + +/** Fill function for the RELAY_METRICS_NUM_SOCKETS metrics. */ +static void +fill_socket_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_SOCKETS]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("state", "opened")); + metrics_store_entry_update(sentry, get_n_open_sockets()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, get_max_sockets()); +} + +/** Fill function for the RELAY_METRICS_NUM_ONIONSKINS metrics. */ +static void +fill_onionskins_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_ONIONSKINS]; + + for (uint16_t t = 0; t <= MAX_ONION_HANDSHAKE_TYPE; t++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *type_label = + tor_strdup(metrics_format_label("type", handshake_type_to_str(t))); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "processed")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_n_handshake_assigned(t)); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "dropped")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_n_handshake_dropped(t)); + tor_free(type_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_OOM_BYTES metrics. */ +static void +fill_oom_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_OOM_BYTES]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "cell")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_cell); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "dns")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_dns); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "geoip")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_geoip); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "hsdir")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_hsdir); +} + +/** Reset the global store and fill it with all the metrics from base_metrics + * and their associated values. + * + * To pull this off, every metrics has a "fill" function that is called and in + * charge of adding the metrics to the store, appropriate labels and finally + * updating the value to report. */ +static void +fill_store(void) +{ + /* Reset the current store, we are about to fill it with all the things. */ + metrics_store_reset(the_store); + + /* Call the fill function for each metrics. */ + for (size_t i = 0; i < num_base_metrics; i++) { + if (BUG(!base_metrics[i].fill_fn)) { + continue; + } + base_metrics[i].fill_fn(); + } +} + +/** Return a list of all the relay metrics stores. This is the + * function attached to the .get_metrics() member of the subsys_t. */ +const smartlist_t * +relay_metrics_get_stores(void) +{ + /* We can't have the caller to free the returned list so keep it static, + * simply update it. */ + static smartlist_t *stores_list = NULL; + + /* We dynamically fill the store with all the metrics upon a request. The + * reason for this is because the exposed metrics of a relay are often + * internal counters in the fast path and thus we fetch the value when a + * metrics port request arrives instead of keeping a local metrics store of + * those values. */ + fill_store(); + + if (!stores_list) { + stores_list = smartlist_new(); + smartlist_add(stores_list, the_store); + } + + return stores_list; +} + +/** Initialize the relay metrics. */ +void +relay_metrics_init(void) +{ + if (BUG(the_store)) { + return; + } + the_store = metrics_store_new(); +} + +/** Free the relay metrics. */ +void +relay_metrics_free(void) +{ + if (!the_store) { + return; + } + /* NULL is set with this call. */ + metrics_store_free(the_store); +} diff --git a/src/feature/relay/relay_metrics.h b/src/feature/relay/relay_metrics.h new file mode 100644 index 0000000000..00dfeaa624 --- /dev/null +++ b/src/feature/relay/relay_metrics.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.h + * @brief Header for feature/relay/relay_metrics.c + **/ + +#ifndef TOR_FEATURE_RELAY_RELAY_METRICS_H +#define TOR_FEATURE_RELAY_RELAY_METRICS_H + +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_common.h" + +/** Metrics key for each reported metrics. This key is also used as an index in + * the base_metrics array. */ +typedef enum { + /** Number of OOM invocation. */ + RELAY_METRICS_NUM_OOM_BYTES = 0, + /** Number of onionskines handled. */ + RELAY_METRICS_NUM_ONIONSKINS = 1, + /** Number of sockets. */ + RELAY_METRICS_NUM_SOCKETS = 2, + /** Number of global connection rate limit. */ + RELAY_METRICS_NUM_GLOBAL_RW_LIMIT = 3, + /** Number of DNS queries. */ + RELAY_METRICS_NUM_DNS = 4, + /** Number of DNS query errors. */ + RELAY_METRICS_NUM_DNS_ERRORS = 5, + /** Number of TCP exhaustion reached. */ + RELAY_METRICS_NUM_TCP_EXHAUSTION = 6, +} relay_metrics_key_t; + +/** The metadata of a relay metric. */ +typedef struct relay_metrics_entry_t { + /* Metric key used as a static array index. */ + relay_metrics_key_t key; + /* Metric type. */ + metrics_type_t type; + /* Metrics output name. */ + const char *name; + /* Metrics output help comment. */ + const char *help; + /* Update value function. */ + void (*fill_fn)(void); +} relay_metrics_entry_t; + +/* Init. */ +void relay_metrics_init(void); +void relay_metrics_free(void); + +/* Accessors. */ +const smartlist_t *relay_metrics_get_stores(void); + +#endif /* !defined(TOR_FEATURE_RELAY_RELAY_METRICS_H) */ diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c index ee94590e01..dd9be4e36f 100644 --- a/src/feature/relay/relay_periodic.c +++ b/src/feature/relay/relay_periodic.c @@ -219,7 +219,7 @@ reachability_warnings_callback(time_t now, const or_options_t *options) tor_asprintf(&where4, "%s:%d", address4, me->ipv4_orport); if (!v6_ok) tor_asprintf(&where6, "[%s]:%d", address6, me->ipv6_orport); - const char *opt_and = (!v4_ok && !v6_ok) ? "and" : ""; + const char *opt_and = (!v4_ok && !v6_ok) ? " and " : ""; /* IPv4 reachability test worked but not the IPv6. We will _not_ * publish the descriptor if our IPv6 was configured. We will if it diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c index 25fc0bbd32..9c43734b84 100644 --- a/src/feature/relay/relay_sys.c +++ b/src/feature/relay/relay_sys.c @@ -14,6 +14,7 @@ #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" +#include "feature/relay/relay_metrics.h" #include "feature/relay/onion_queue.h" #include "feature/relay/relay_periodic.h" #include "feature/relay/relay_sys.h" @@ -25,6 +26,7 @@ static int subsys_relay_initialize(void) { + relay_metrics_init(); relay_register_periodic_events(); return 0; } @@ -37,6 +39,7 @@ subsys_relay_shutdown(void) clear_pending_onions(); routerkeys_free_all(); router_free_all(); + relay_metrics_free(); } const struct subsys_fns_t sys_relay = { @@ -46,4 +49,6 @@ const struct subsys_fns_t sys_relay = { .level = RELAY_SUBSYS_LEVEL, .initialize = subsys_relay_initialize, .shutdown = subsys_relay_shutdown, + + .get_metrics = relay_metrics_get_stores, }; diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index df838aa527..8f6a45dfef 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -131,7 +131,11 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, rend_circ = hs_circuitmap_get_rend_circ_relay_side(request); if (!rend_circ) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + /* Once this was a LOG_PROTOCOL_WARN, but it can happen naturally if a + * client gives up on a rendezvous circuit after sending INTRODUCE1, but + * before the onion service sends the RENDEZVOUS1 cell. + */ + log_fn(LOG_DEBUG, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.", hexid); reason = END_CIRC_REASON_TORPROTOCOL; diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index b4b107c3f7..a0fe8597c1 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -1206,11 +1206,11 @@ format_bridge_stats_controller(time_t now) char * format_client_stats_heartbeat(time_t now) { - const int n_hours = 6; + const int n_seconds = get_options()->HeartbeatPeriod; char *out = NULL; int n_clients = 0; clientmap_entry_t **ent; - unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + unsigned cutoff = (unsigned)( (now-n_seconds)/60 ); if (!start_of_bridge_stats_interval) return NULL; /* Not initialized. */ @@ -1226,8 +1226,7 @@ format_client_stats_heartbeat(time_t now) } tor_asprintf(&out, "Heartbeat: " - "In the last %d hours, I have seen %d unique clients.", - n_hours, + "Since last heartbeat message, I have seen %d unique clients.", n_clients); return out; diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index 2bfa14d326..5ff4ef1d2e 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -206,18 +206,33 @@ typedef struct { uint64_t overload_fd_exhausted; } overload_stats_t; +/** Current state of overload stats */ +static overload_stats_t overload_stats; + +/** Counters to count the number of times we've reached an overload for the + * global connection read/write limit. Reported on the MetricsPort. */ +static uint64_t stats_n_read_limit_reached = 0; +static uint64_t stats_n_write_limit_reached = 0; + +/** Total number of times we've reached TCP port exhaustion. */ +static uint64_t stats_n_tcp_exhaustion = 0; + /***** DNS statistics *****/ -/** Represents the statistics of DNS queries seen if it is an Exit. */ +/** Overload DNS statistics. The information in this object is used to assess + * if, due to DNS errors, we should emit a general overload signal or not. + * + * NOTE: This structure is _not_ per DNS query type like the statistics below + * because of a libevent bug + * (https://github.com/libevent/libevent/issues/1219), on error, the type is + * not propagated up back to the user and so we need to keep our own stats for + * the overload signal. */ typedef struct { /** Total number of DNS request seen at an Exit. They might not all end * successfully or might even be lost by tor. This counter is incremented * right before the DNS request is initiated. */ uint64_t stats_n_request; - /** Total number of DNS timeout errors. */ - uint64_t stats_n_error_timeout; - /** When is the next assessment time of the general overload for DNS errors. * Once this time is reached, all stats are reset and this time is set to the * next assessment time. */ @@ -227,121 +242,230 @@ typedef struct { /** Keep track of the DNS requests for the general overload state. */ static overload_dns_stats_t overload_dns_stats; -/* We use a scale here so we can represent percentages with decimal points by - * scaling the value by this factor and so 0.5% becomes a value of 500. - * Default is 1% and thus min and max range is 0 to 100%. */ -#define OVERLOAD_DNS_TIMEOUT_PERCENT_SCALE 1000.0 -#define OVERLOAD_DNS_TIMEOUT_PERCENT_DEFAULT 1000 -#define OVERLOAD_DNS_TIMEOUT_PERCENT_MIN 0 -#define OVERLOAD_DNS_TIMEOUT_PERCENT_MAX 100000 - -/** Consensus parameter: indicate what fraction of DNS timeout errors over the - * total number of DNS requests must be reached before we trigger a general - * overload signal .*/ -static double overload_dns_timeout_fraction = - OVERLOAD_DNS_TIMEOUT_PERCENT_DEFAULT / - OVERLOAD_DNS_TIMEOUT_PERCENT_SCALE / 100.0; - -/* Number of seconds for the assessment period. Default is 10 minutes (600) and - * the min max range is within a 32bit value. */ -#define OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_DEFAULT (10 * 60) -#define OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MIN 0 -#define OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MAX INT32_MAX - -/** Consensus parameter: Period, in seconds, over which we count the number of - * DNS requests and timeout errors. After that period, we assess if we trigger - * an overload or not. */ -static int32_t overload_dns_timeout_period_secs = - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_DEFAULT; +/** Represents the statistics of DNS queries seen if it is an Exit. */ +typedef struct { + /* Total number of DNS errors found in RFC 1035 (from 0 to 5 code). */ + uint64_t stats_n_error_none; /* 0 */ + uint64_t stats_n_error_format; /* 1 */ + uint64_t stats_n_error_serverfailed; /* 2 */ + uint64_t stats_n_error_notexist; /* 3 */ + uint64_t stats_n_error_notimpl; /* 4 */ + uint64_t stats_n_error_refused; /* 5 */ + + /* Total number of DNS errors specific to libevent. */ + uint64_t stats_n_error_truncated; /* 65 */ + uint64_t stats_n_error_unknown; /* 66 */ + uint64_t stats_n_error_tor_timeout; /* 67 */ + uint64_t stats_n_error_shutdown; /* 68 */ + uint64_t stats_n_error_cancel; /* 69 */ + uint64_t stats_n_error_nodata; /* 70 */ + + /* Total number of DNS request seen at an Exit. They might not all end + * successfully or might even be lost by tor. This counter is incremented + * right before the DNS request is initiated. */ + uint64_t stats_n_request; +} dns_stats_t; + +/* This is disabled because of the libevent bug where on error we don't get the + * DNS query type back. Once it is fixed, we can re-enable this. */ +#if 0 +/** DNS statistics store for each DNS record type for which tor supports only + * three at the moment: A, PTR and AAAA. */ +static dns_stats_t dns_A_stats; +static dns_stats_t dns_PTR_stats; +static dns_stats_t dns_AAAA_stats; +#endif -/** Current state of overload stats */ -static overload_stats_t overload_stats; +/** DNS query statistics store. It covers all type of queries. */ +static dns_stats_t dns_all_stats; -/** Return true if this overload happened within the last `n_hours`. */ -static bool -overload_happened_recently(time_t overload_time, int n_hours) +/** Return the point to the DNS statistics store. Ignore the type for now + * because of a libevent problem. */ +static inline dns_stats_t * +get_dns_stats_by_type(const int type) { - /* An overload is relevant if it happened in the last 72 hours */ - if (overload_time > approx_time() - 3600 * n_hours) { - return true; - } - return false; + (void) type; + return &dns_all_stats; } -/** Assess the DNS timeout errors and if we have enough to trigger a general - * overload. */ -static void -overload_general_dns_assessment(void) +#if 0 +/** From a libevent record type, return a pointer to the corresponding DNS + * statistics store. NULL is returned if the type is unhandled. */ +static inline dns_stats_t * +get_dns_stats_by_type(const int type) { - /* Initialize the time. Should be done once. */ - if (overload_dns_stats.next_assessment_time == 0) { - goto reset; + switch (type) { + case DNS_IPv4_A: + return &dns_A_stats; + case DNS_PTR: + return &dns_PTR_stats; + case DNS_IPv6_AAAA: + return &dns_AAAA_stats; + default: + return NULL; } +} +#endif - /* Not the time yet. */ - if (overload_dns_stats.next_assessment_time > approx_time()) { - return; +/** Return the DNS error count for the given libevent DNS type and error code. + * The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_error(int type, uint8_t error) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; } - reset: - /* Reset counters for the next period. */ - overload_dns_stats.stats_n_error_timeout = 0; - overload_dns_stats.stats_n_request = 0; - overload_dns_stats.next_assessment_time = - approx_time() + overload_dns_timeout_period_secs; + switch (error) { + case DNS_ERR_NONE: + return dns_stats->stats_n_error_none; + case DNS_ERR_FORMAT: + return dns_stats->stats_n_error_format; + case DNS_ERR_SERVERFAILED: + return dns_stats->stats_n_error_serverfailed; + case DNS_ERR_NOTEXIST: + return dns_stats->stats_n_error_notexist; + case DNS_ERR_NOTIMPL: + return dns_stats->stats_n_error_notimpl; + case DNS_ERR_REFUSED: + return dns_stats->stats_n_error_refused; + case DNS_ERR_TRUNCATED: + return dns_stats->stats_n_error_truncated; + case DNS_ERR_UNKNOWN: + return dns_stats->stats_n_error_unknown; + case DNS_ERR_TIMEOUT: + return dns_stats->stats_n_error_tor_timeout; + case DNS_ERR_SHUTDOWN: + return dns_stats->stats_n_error_shutdown; + case DNS_ERR_CANCEL: + return dns_stats->stats_n_error_cancel; + case DNS_ERR_NODATA: + return dns_stats->stats_n_error_nodata; + default: + /* Unhandled code sent back by libevent. */ + return 0; + } } -/** Called just before the consensus will be replaced. Update the consensus - * parameters in case they changed. */ -void -rep_hist_consensus_has_changed(const networkstatus_t *ns) +/** Return the total number of DNS request seen for the given libevent DNS + * record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_request(int type) { - overload_dns_timeout_fraction = - networkstatus_get_param(ns, "overload_dns_timeout_scale_percent", - OVERLOAD_DNS_TIMEOUT_PERCENT_DEFAULT, - OVERLOAD_DNS_TIMEOUT_PERCENT_MIN, - OVERLOAD_DNS_TIMEOUT_PERCENT_MAX) / - OVERLOAD_DNS_TIMEOUT_PERCENT_SCALE / 100.0; - - overload_dns_timeout_period_secs = - networkstatus_get_param(ns, "overload_dns_timeout_period_secs", - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_DEFAULT, - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MIN, - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MAX); + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; + } + return dns_stats->stats_n_request; } /** Note a DNS error for the given given libevent DNS record type and error * code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. * - * IMPORTANT: Libevent is _not_ returning the type in case of an error and so - * if error is anything but DNS_ERR_NONE, the type is not usable and set to 0. + * NOTE: Libevent is _not_ returning the type in case of an error and so if + * error is anything but DNS_ERR_NONE, the type is not usable and set to 0. * * See: https://gitlab.torproject.org/tpo/core/tor/-/issues/40490 */ void -rep_hist_note_dns_query(int type, uint8_t error) +rep_hist_note_dns_error(int type, uint8_t error) { - (void) type; + overload_dns_stats.stats_n_request++; - /* Assess if we need to trigger a general overload with regards to the DNS - * errors or not. */ - overload_general_dns_assessment(); + /* Again, the libevent bug (see function comment), for an error that is + * anything but DNS_ERR_NONE, the type is always 0 which means that we don't + * have a DNS stat object for it so this code will do nothing until libevent + * is fixed. */ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + /* Unsupported DNS query type. */ + if (!dns_stats) { + return; + } - /* We only care about timeouts for the moment. */ switch (error) { + case DNS_ERR_NONE: + dns_stats->stats_n_error_none++; + break; + case DNS_ERR_FORMAT: + dns_stats->stats_n_error_format++; + break; + case DNS_ERR_SERVERFAILED: + dns_stats->stats_n_error_serverfailed++; + break; + case DNS_ERR_NOTEXIST: + dns_stats->stats_n_error_notexist++; + break; + case DNS_ERR_NOTIMPL: + dns_stats->stats_n_error_notimpl++; + break; + case DNS_ERR_REFUSED: + dns_stats->stats_n_error_refused++; + break; + case DNS_ERR_TRUNCATED: + dns_stats->stats_n_error_truncated++; + break; + case DNS_ERR_UNKNOWN: + dns_stats->stats_n_error_unknown++; + break; case DNS_ERR_TIMEOUT: - overload_dns_stats.stats_n_error_timeout++; + dns_stats->stats_n_error_tor_timeout++; + break; + case DNS_ERR_SHUTDOWN: + dns_stats->stats_n_error_shutdown++; + break; + case DNS_ERR_CANCEL: + dns_stats->stats_n_error_cancel++; + break; + case DNS_ERR_NODATA: + dns_stats->stats_n_error_nodata++; break; default: + /* Unhandled code sent back by libevent. */ break; } +} - /* Increment total number of requests. */ - overload_dns_stats.stats_n_request++; +/** Note a DNS request for the given given libevent DNS record type. */ +void +rep_hist_note_dns_request(int type) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return; + } + dns_stats->stats_n_request++; +} + +/***** END of DNS statistics *****/ + +/** Return true if this overload happened within the last `n_hours`. */ +static bool +overload_happened_recently(time_t overload_time, int n_hours) +{ + /* An overload is relevant if it happened in the last 72 hours */ + if (overload_time > approx_time() - 3600 * n_hours) { + return true; + } + return false; } /* The current version of the overload stats version */ #define OVERLOAD_STATS_VERSION 1 +/** Return the stats_n_read_limit_reached counter. */ +uint64_t +rep_hist_get_n_read_limit_reached(void) +{ + return stats_n_read_limit_reached; +} + +/** Return the stats_n_write_limit_reached counter. */ +uint64_t +rep_hist_get_n_write_limit_reached(void) +{ + return stats_n_write_limit_reached; +} + /** Returns an allocated string for server descriptor for publising information * on whether we are overloaded or not. */ char * @@ -420,6 +544,7 @@ rep_hist_note_overload(overload_type_t overload) SET_TO_START_OF_HOUR(overload_stats.overload_general_time); break; case OVERLOAD_READ: { + stats_n_read_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_read_counted + 60) { /* Count once a minute */ overload_stats.overload_read_count++; @@ -428,6 +553,7 @@ rep_hist_note_overload(overload_type_t overload) break; } case OVERLOAD_WRITE: { + stats_n_write_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_write_counted + 60) { /* Count once a minute */ overload_stats.overload_write_count++; @@ -442,6 +568,22 @@ rep_hist_note_overload(overload_type_t overload) } } +/** Note down that we've reached a TCP port exhaustion. This triggers an + * overload general event. */ +void +rep_hist_note_tcp_exhaustion(void) +{ + stats_n_tcp_exhaustion++; + rep_hist_note_overload(OVERLOAD_GENERAL); +} + +/** Return the total number of TCP exhaustion times we've reached. */ +uint64_t +rep_hist_get_n_tcp_exhaustion(void) +{ + return stats_n_tcp_exhaustion; +} + /** Return the or_history_t for the OR with identity digest <b>id</b>, * creating it if necessary. */ static or_history_t * @@ -641,7 +783,7 @@ rep_hist_downrate_old_runs(time_t now) return stability_last_downrated + STABILITY_INTERVAL; /* Okay, we should downrate the data. By how much? */ - while (stability_last_downrated + STABILITY_INTERVAL < now) { + while (stability_last_downrated + STABILITY_INTERVAL <= now) { stability_last_downrated += STABILITY_INTERVAL; alpha *= STABILITY_ALPHA; } @@ -1908,11 +2050,18 @@ rep_hist_note_desc_served(const char * desc) /** Internal statistics to track how many requests of each type of * handshake we've received, and how many we've assigned to cpuworkers. * Useful for seeing trends in cpu load. + * + * They are reset at every heartbeat. * @{ */ STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ +/** Counters keeping the same stats as above but for the entire duration of the + * process (not reset). */ +static uint64_t stats_n_onionskin_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +static uint64_t stats_n_onionskin_dropped[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; + /** A new onionskin (using the <b>type</b> handshake) has arrived. */ void rep_hist_note_circuit_handshake_requested(uint16_t type) @@ -1926,8 +2075,20 @@ rep_hist_note_circuit_handshake_requested(uint16_t type) void rep_hist_note_circuit_handshake_assigned(uint16_t type) { - if (type <= MAX_ONION_HANDSHAKE_TYPE) + if (type <= MAX_ONION_HANDSHAKE_TYPE) { onion_handshakes_assigned[type]++; + stats_n_onionskin_assigned[type]++; + } +} + +/** We've just drop an onionskin (using the <b>type</b> handshake) due to being + * overloaded. */ +void +rep_hist_note_circuit_handshake_dropped(uint16_t type) +{ + if (type <= MAX_ONION_HANDSHAKE_TYPE) { + stats_n_onionskin_dropped[type]++; + } } /** Get the circuit handshake value that is requested. */ @@ -1950,6 +2111,26 @@ rep_hist_get_circuit_handshake_assigned, (uint16_t type)) return onion_handshakes_assigned[type]; } +/** Get the total number of circuit handshake value that is assigned. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_n_handshake_assigned, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return stats_n_onionskin_assigned[type]; +} + +/** Get the total number of circuit handshake value that is dropped. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_n_handshake_dropped, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return stats_n_onionskin_dropped[type]; +} + /** Log our onionskin statistics since the last time we were called. */ void rep_hist_log_circuit_handshake_stats(time_t now) diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index 749b4996a8..7f414de4c8 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -58,11 +58,17 @@ time_t rep_hist_desc_stats_write(time_t now); void rep_hist_note_circuit_handshake_requested(uint16_t type); void rep_hist_note_circuit_handshake_assigned(uint16_t type); +void rep_hist_note_circuit_handshake_dropped(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); MOCK_DECL(int, rep_hist_get_circuit_handshake_requested, (uint16_t type)); MOCK_DECL(int, rep_hist_get_circuit_handshake_assigned, (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_n_handshake_assigned, + (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_n_handshake_dropped, + (uint16_t type)); + void rep_hist_hs_stats_init(time_t now); void rep_hist_hs_stats_term(void); time_t rep_hist_hs_stats_write(time_t now, bool is_v3); @@ -72,14 +78,16 @@ void rep_hist_seen_new_rp_cell(bool is_v2); char *rep_hist_get_hs_v3_stats_string(void); void rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key); -void rep_hist_note_dns_query(int type, uint8_t error); - void rep_hist_free_all(void); void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here); void rep_hist_log_link_protocol_counts(void); -void rep_hist_consensus_has_changed(const networkstatus_t *ns); + +uint64_t rep_hist_get_n_dns_error(int type, uint8_t error); +uint64_t rep_hist_get_n_dns_request(int type); +void rep_hist_note_dns_request(int type); +void rep_hist_note_dns_error(int type, uint8_t error); extern uint64_t rephist_total_alloc; extern uint32_t rephist_total_num; @@ -162,6 +170,12 @@ void rep_hist_note_overload(overload_type_t overload); char *rep_hist_get_overload_general_line(void); char *rep_hist_get_overload_stats_lines(void); +void rep_hist_note_tcp_exhaustion(void); +uint64_t rep_hist_get_n_tcp_exhaustion(void); + +uint64_t rep_hist_get_n_read_limit_reached(void); +uint64_t rep_hist_get_n_write_limit_reached(void); + #ifdef TOR_UNIT_TESTS struct hs_v2_stats_t; const struct hs_v2_stats_t *rep_hist_get_hs_v2_stats(void); diff --git a/src/include.am b/src/include.am index 0826da7548..36d323e6eb 100644 --- a/src/include.am +++ b/src/include.am @@ -85,7 +85,6 @@ include src/app/main/include.am include src/core/include.am include src/app/include.am -include src/rust/include.am include src/test/include.am include src/tools/include.am include src/win32/include.am diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index 50bfedffba..991b33d9e7 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -15,6 +15,15 @@ #include "orconfig.h" #include <inttypes.h> +#if defined(__MINGW32__) || defined(__MINGW64__) +#define MINGW_ANY +#endif + +#ifdef MINGW_ANY +/* We need this for __MINGW_PRINTF_FORMAT, alas. */ +#include <stdio.h> +#endif + #if defined(__has_feature) # if __has_feature(address_sanitizer) /* Some of the fancy glibc strcmp() macros include references to memory that @@ -36,16 +45,30 @@ #error "It seems that you encode characters in something other than ASCII." #endif +/* Use the right magic attribute on mingw, which might be printf, gnu_printf, + * or ms_printf, depending on how we're set up to build. + */ +#ifdef __MINGW_PRINTF_FORMAT +#define PRINTF_FORMAT_ATTR __MINGW_PRINTF_FORMAT +#else +#define PRINTF_FORMAT_ATTR printf +#endif +#ifdef __MINGW_SCANF_FORMAT +#define SCANF_FORMAT_ATTR __MINGW_SCANF_FORMAT +#else +#define SCANF_FORMAT_ATTR scanf +#endif + /* GCC can check printf and scanf types on arbitrary functions. */ #ifdef __GNUC__ #define CHECK_PRINTF(formatIdx, firstArg) \ - __attribute__ ((format(printf, formatIdx, firstArg))) + __attribute__ ((format(PRINTF_FORMAT_ATTR, formatIdx, firstArg))) #else #define CHECK_PRINTF(formatIdx, firstArg) #endif /* defined(__GNUC__) */ #ifdef __GNUC__ #define CHECK_SCANF(formatIdx, firstArg) \ - __attribute__ ((format(scanf, formatIdx, firstArg))) + __attribute__ ((format(SCANF_FORMAT_ATTR, formatIdx, firstArg))) #else #define CHECK_SCANF(formatIdx, firstArg) #endif /* defined(__GNUC__) */ @@ -191,10 +214,6 @@ #define OP_EQ == #define OP_NE != -#if defined(__MINGW32__) || defined(__MINGW64__) -#define MINGW_ANY -#endif - /** Macro: yield a pointer to the field at position <b>off</b> within the * structure <b>st</b>. Example: * <pre> diff --git a/src/lib/metrics/metrics_common.c b/src/lib/metrics/metrics_common.c index 5941a4d892..f3f7e22d88 100644 --- a/src/lib/metrics/metrics_common.c +++ b/src/lib/metrics/metrics_common.c @@ -11,6 +11,7 @@ #include "orconfig.h" #include "lib/log/util_bug.h" +#include "lib/string/printf.h" #include "lib/metrics/metrics_common.h" @@ -27,3 +28,15 @@ metrics_type_to_str(const metrics_type_t type) tor_assert_unreached(); } } + +/** Return a static buffer pointer that contains a formatted label on the form + * of key=value. + * + * Subsequent call to this function invalidates the previous buffer. */ +const char * +metrics_format_label(const char *key, const char *value) +{ + static char buf[128]; + tor_snprintf(buf, sizeof(buf), "%s=\"%s\"", key, value); + return buf; +} diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h index 59aa9c0e90..3644ad3d50 100644 --- a/src/lib/metrics/metrics_common.h +++ b/src/lib/metrics/metrics_common.h @@ -42,4 +42,7 @@ typedef struct metrics_gauge_t { const char *metrics_type_to_str(const metrics_type_t type); +/* Helpers. */ +const char *metrics_format_label(const char *key, const char *value); + #endif /* !defined(TOR_LIB_METRICS_METRICS_COMMON_H) */ diff --git a/src/lib/metrics/metrics_store.c b/src/lib/metrics/metrics_store.c index 4cab5245f3..33b1780438 100644 --- a/src/lib/metrics/metrics_store.c +++ b/src/lib/metrics/metrics_store.c @@ -34,7 +34,8 @@ struct metrics_store_t { }; /** Function pointer to the format function of a specific driver. */ -typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *); +typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *, + bool no_comment); /** Helper: Free a single entry in a metrics_store_t taking a void pointer * parameter. */ @@ -47,6 +48,8 @@ metrics_store_free_void(void *p) smartlist_free(list); } +#include <stdio.h> + /** Put the given store output in the buffer data and use the format function * given in fmt to get it for each entry. */ static void @@ -57,8 +60,11 @@ get_output(const metrics_store_t *store, buf_t *data, fmt_driver_fn_t fmt) tor_assert(fmt); STRMAP_FOREACH(store->entries, key, const smartlist_t *, entries) { + /* Indicate that we've formatted the coment already for the entries. */ + bool comment_formatted = false; SMARTLIST_FOREACH_BEGIN(entries, const metrics_store_entry_t *, entry) { - fmt(entry, data); + fmt(entry, data, comment_formatted); + comment_formatted = true; } SMARTLIST_FOREACH_END(entry); } STRMAP_FOREACH_END; } @@ -138,3 +144,14 @@ metrics_store_get_output(const metrics_format_t fmt, // LCOV_EXCL_STOP } } + +/** Reset a store as in free its content. */ +void +metrics_store_reset(metrics_store_t *store) +{ + if (store == NULL) { + return; + } + strmap_free(store->entries, metrics_store_free_void); + store->entries = strmap_new(); +} diff --git a/src/lib/metrics/metrics_store.h b/src/lib/metrics/metrics_store.h index 42bc56e8fd..d85f484bd6 100644 --- a/src/lib/metrics/metrics_store.h +++ b/src/lib/metrics/metrics_store.h @@ -28,6 +28,7 @@ metrics_store_t *metrics_store_new(void); metrics_store_entry_t *metrics_store_add(metrics_store_t *store, metrics_type_t type, const char *name, const char *help); +void metrics_store_reset(metrics_store_t *store); /* Accessors. */ smartlist_t *metrics_store_get_all(const metrics_store_t *store, diff --git a/src/lib/metrics/prometheus.c b/src/lib/metrics/prometheus.c index 65241ed6c1..aac23ac92e 100644 --- a/src/lib/metrics/prometheus.c +++ b/src/lib/metrics/prometheus.c @@ -42,14 +42,17 @@ format_labels(smartlist_t *labels) /** Format the given entry in to the buffer data. */ void -prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data) +prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data, + bool no_comment) { tor_assert(entry); tor_assert(data); - buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); - buf_add_printf(data, "# TYPE %s %s\n", entry->name, - metrics_type_to_str(entry->type)); + if (!no_comment) { + buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); + buf_add_printf(data, "# TYPE %s %s\n", entry->name, + metrics_type_to_str(entry->type)); + } buf_add_printf(data, "%s%s %" PRIi64 "\n", entry->name, format_labels(entry->labels), metrics_store_entry_get_value(entry)); diff --git a/src/lib/metrics/prometheus.h b/src/lib/metrics/prometheus.h index 19770e7911..faa7681daa 100644 --- a/src/lib/metrics/prometheus.h +++ b/src/lib/metrics/prometheus.h @@ -13,6 +13,6 @@ #include "lib/metrics/metrics_store_entry.h" void prometheus_format_store_entry(const metrics_store_entry_t *entry, - buf_t *data); + buf_t *data, bool no_comment); #endif /* !defined(TOR_LIB_METRICS_PROMETHEUS_H) */ diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 26b155bc4c..085eb8c458 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -2005,20 +2005,15 @@ parse_port_range(const char *port, uint16_t *port_min_out, char *endptr = NULL; port_min = (int)tor_parse_long(port, 10, 0, 65535, &ok, &endptr); if (!ok) { - log_warn(LD_GENERAL, - "Malformed port %s on address range; rejecting.", - escaped(port)); - return -1; - } else if (endptr && *endptr == '-') { + goto malformed_port; + } else if (endptr && *endptr != '\0') { + if (*endptr != '-') + goto malformed_port; port = endptr+1; endptr = NULL; port_max = (int)tor_parse_long(port, 10, 1, 65535, &ok, &endptr); - if (!ok) { - log_warn(LD_GENERAL, - "Malformed port %s on address range; rejecting.", - escaped(port)); - return -1; - } + if (!ok) + goto malformed_port; } else { port_max = port_min; } @@ -2037,6 +2032,11 @@ parse_port_range(const char *port, uint16_t *port_min_out, *port_max_out = (uint16_t) port_max; return 0; + malformed_port: + log_warn(LD_GENERAL, + "Malformed port %s on address range; rejecting.", + escaped(port)); + return -1; } /** Given a host-order <b>addr</b>, call tor_inet_ntop() on it diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 02222e5a1c..a15f99ad76 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -58,6 +58,10 @@ #include <linux/futex.h> #include <sys/file.h> +#ifdef ENABLE_FRAGILE_HARDENING +#include <sys/ptrace.h> +#endif + #include <stdarg.h> #include <seccomp.h> #include <signal.h> @@ -148,7 +152,11 @@ static sandbox_cfg_t *filter_dynamic = NULL; static int filter_nopar_gen[] = { SCMP_SYS(access), SCMP_SYS(brk), +#ifdef __NR_clock_gettime64 + SCMP_SYS(clock_gettime64), +#else SCMP_SYS(clock_gettime), +#endif SCMP_SYS(close), SCMP_SYS(clone), SCMP_SYS(dup), @@ -191,6 +199,9 @@ static int filter_nopar_gen[] = { SCMP_SYS(getgid32), #endif SCMP_SYS(getpid), +#ifdef ENABLE_FRAGILE_HARDENING + SCMP_SYS(getppid), +#endif #ifdef __NR_getrlimit SCMP_SYS(getrlimit), #endif @@ -241,6 +252,9 @@ static int filter_nopar_gen[] = { SCMP_SYS(sigreturn), #endif SCMP_SYS(stat), +#if defined(__i386__) && defined(__NR_statx) + SCMP_SYS(statx), +#endif SCMP_SYS(uname), SCMP_SYS(wait4), SCMP_SYS(write), @@ -339,6 +353,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return rc; } +#ifdef __NR_time /** * Function responsible for setting up the time syscall for * the seccomp filter sandbox. @@ -347,13 +362,11 @@ static int sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter) { (void) filter; -#ifdef __NR_time + return seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(time), SCMP_CMP(0, SCMP_CMP_EQ, 0)); -#else - return 0; -#endif /* defined(__NR_time) */ } +#endif /* defined(__NR_time) */ /** * Function responsible for setting up the accept4 syscall for @@ -532,6 +545,24 @@ sb_open(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int use_openat = libc_uses_openat_for_open(); +#ifdef ENABLE_FRAGILE_HARDENING + /* AddressSanitizer uses the "open" syscall to access information about the + * running process via the filesystem, so that call must be allowed without + * restriction or the sanitizer will be unable to execute normally when the + * process terminates. */ + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, SCMP_SYS(open)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + "libseccomp error %d", rc); + return rc; + } + + /* If glibc also uses only the "open" syscall to open files on this system + * there is no need to consider any additional rules. */ + if (!use_openat) + return 0; +#endif + // for each dynamic parameter filters for (elem = filter; elem != NULL; elem = elem->next) { smp_param_t *param = elem->param; @@ -575,6 +606,32 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#ifdef __i386__ +static int +sb_chown32(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && param->syscall + == SCMP_SYS(chown32)) { + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown32), + SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add chown32 syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} +#else static int sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter) { @@ -599,6 +656,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#endif /* defined(__i386__) */ /** * Function responsible for setting up the rename syscall for @@ -687,6 +745,34 @@ sb_opendir(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#ifdef ENABLE_FRAGILE_HARDENING +/** + * Function responsible for setting up the ptrace syscall for + * the seccomp filter sandbox. + */ +static int +sb_ptrace(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + pid_t pid = getpid(); + (void) filter; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ptrace), + SCMP_CMP(0, SCMP_CMP_EQ, PTRACE_ATTACH), + SCMP_CMP(1, SCMP_CMP_EQ, pid)); + if (rc) + return rc; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ptrace), + SCMP_CMP(0, SCMP_CMP_EQ, PTRACE_GETREGS), + SCMP_CMP(1, SCMP_CMP_EQ, pid)); + if (rc) + return rc; + + return 0; +} +#endif + /** * Function responsible for setting up the socket syscall for * the seccomp filter sandbox. @@ -1009,6 +1095,18 @@ sb_prctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int rc = 0; (void) filter; +#ifdef ENABLE_FRAGILE_HARDENING + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), + SCMP_CMP(0, SCMP_CMP_EQ, PR_GET_DUMPABLE)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), + SCMP_CMP(0, SCMP_CMP_EQ, PR_SET_PTRACER)); + if (rc) + return rc; +#endif + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), SCMP_CMP(0, SCMP_CMP_EQ, PR_SET_DUMPABLE)); if (rc) @@ -1053,6 +1151,13 @@ sb_rt_sigprocmask(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int rc = 0; (void) filter; +#ifdef ENABLE_FRAGILE_HARDENING + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigprocmask), + SCMP_CMP(0, SCMP_CMP_EQ, SIG_BLOCK)); + if (rc) + return rc; +#endif + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigprocmask), SCMP_CMP(0, SCMP_CMP_EQ, SIG_UNBLOCK)); if (rc) @@ -1192,16 +1297,25 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) static sandbox_filter_func_t filter_func[] = { sb_rt_sigaction, sb_rt_sigprocmask, +#ifdef __NR_time sb_time, +#endif sb_accept4, #ifdef __NR_mmap2 sb_mmap2, #endif +#ifdef __i386__ + sb_chown32, +#else sb_chown, +#endif sb_chmod, sb_open, sb_openat, sb_opendir, +#ifdef ENABLE_FRAGILE_HARDENING + sb_ptrace, +#endif sb_rename, #ifdef __NR_fcntl64 sb_fcntl64, @@ -1468,6 +1582,12 @@ new_element(int syscall, char *value) return new_element2(syscall, value, NULL); } +#ifdef __i386__ +#define SCMP_chown SCMP_SYS(chown32) +#else +#define SCMP_chown SCMP_SYS(chown) +#endif + #ifdef __NR_stat64 #define SCMP_stat SCMP_SYS(stat64) #else @@ -1518,7 +1638,7 @@ sandbox_cfg_allow_chown_filename(sandbox_cfg_t **cfg, char *file) { sandbox_cfg_t *elem = NULL; - elem = new_element(SCMP_SYS(chown), file); + elem = new_element(SCMP_chown, file); elem->next = *cfg; *cfg = elem; diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c index 62758093a7..bd35b76d1b 100644 --- a/src/lib/string/printf.c +++ b/src/lib/string/printf.c @@ -8,9 +8,9 @@ * \brief Compatibility wrappers around snprintf and its friends **/ +#include "lib/cc/torint.h" #include "lib/string/printf.h" #include "lib/err/torerr.h" -#include "lib/cc/torint.h" #include "lib/malloc/malloc.h" #include <stdlib.h> @@ -45,7 +45,7 @@ tor_vsnprintf(char *str, size_t size, const char *format, va_list args) return -1; /* no place for the NUL */ if (size > SIZE_T_CEILING) return -1; -#ifdef _WIN32 +#if defined(_WIN32) && !defined(HAVE_VSNPRINTF) r = _vsnprintf(str, size, format, args); #else r = vsnprintf(str, size, format, args); diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in deleted file mode 100644 index 6eddc75459..0000000000 --- a/src/rust/.cargo/config.in +++ /dev/null @@ -1,12 +0,0 @@ -[source] - -@RUST_DL@ [source.crates-io] -@RUST_DL@ registry = 'https://github.com/rust-lang/crates.io-index' -@RUST_DL@ replace-with = 'vendored-sources' - -@RUST_DL@ [source.vendored-sources] -@RUST_DL@ directory = '@TOR_RUST_DEPENDENCIES@' - -[build] -@RUST_WARN@ rustflags = [ "-D", "warnings" ] -@RUST_TARGET_PROP@ diff --git a/src/rust/.rustfmt.toml b/src/rust/.rustfmt.toml deleted file mode 100644 index 4ff839dcf3..0000000000 --- a/src/rust/.rustfmt.toml +++ /dev/null @@ -1,12 +0,0 @@ -max_width = 100 -hard_tabs = false -tab_spaces = 4 -newline_style = "Unix" -#use_small_heuristics = "Default" -reorder_imports = true -reorder_modules = true -remove_nested_parens = true -merge_derives = true -use_try_shorthand = false -use_field_init_shorthand = false -force_explicit_abi = true diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock deleted file mode 100644 index e2f24b0af7..0000000000 --- a/src/rust/Cargo.lock +++ /dev/null @@ -1,122 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "crypto" -version = "0.0.1" -dependencies = [ - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "external 0.0.1", - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", - "smartlist 0.0.1", - "tor_allocate 0.0.1", - "tor_log 0.1.0", -] - -[[package]] -name = "digest" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "external" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "smartlist 0.0.1", - "tor_allocate 0.0.1", -] - -[[package]] -name = "generic-array" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "libc" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "protover" -version = "0.0.1" -dependencies = [ - "external 0.0.1", - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "smartlist 0.0.1", - "tor_allocate 0.0.1", - "tor_log 0.1.0", - "tor_util 0.0.1", -] - -[[package]] -name = "rand" -version = "0.5.0-pre.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand_core" -version = "0.2.0-pre.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "smartlist" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tor_allocate" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tor_log" -version = "0.1.0" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "tor_allocate 0.0.1", -] - -[[package]] -name = "tor_rust" -version = "0.1.0" -dependencies = [ - "protover 0.0.1", - "tor_util 0.0.1", -] - -[[package]] -name = "tor_util" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "tor_allocate 0.0.1", - "tor_log 0.1.0", -] - -[[package]] -name = "typenum" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" -"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" -"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff" -"checksum rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3795e4701d9628a63a84d0289e66279883b40df165fca7caed7b87122447032a" -"checksum rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7255ffbdb188d5be1a69b6f9f3cf187de4207430b9e79ed5b76458a6b20de9a" -"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd" diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml deleted file mode 100644 index de8693ea33..0000000000 --- a/src/rust/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workspace] -members = [ - "crypto", - "external", - "protover", - "smartlist", - "tor_allocate", - "tor_log", - "tor_rust", - "tor_util", -] - -# Can remove panic="abort" when this issue is fixed: -# https://github.com/rust-lang/rust/issues/52652 -[profile.dev] -panic = "abort" - -[profile.release] -debug = true -panic = "abort" - -[profile.test] -panic = "abort" - -[profile.bench] -panic = "abort" diff --git a/src/rust/build.rs b/src/rust/build.rs deleted file mode 100644 index 5626b35f75..0000000000 --- a/src/rust/build.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Build script for Rust modules in Tor. -//! -//! We need to use this because some of our Rust tests need to use some -//! of our C modules, which need to link some external libraries. -//! -//! This script works by looking at a "config.rust" file generated by our -//! configure script, and then building a set of options for cargo to pass to -//! the compiler. - -use std::collections::HashMap; -use std::env; -use std::fs::File; -use std::io; -use std::io::prelude::*; -use std::path::PathBuf; - -/// Wrapper around a key-value map. -struct Config(HashMap<String, String>); - -/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR -/// location provided by cargo and recursing up the directory tree. Note that -/// we need to look in the OUT_DIR, since autoconf will place generated files -/// in the build directory. -fn find_cfg() -> io::Result<String> { - let mut path = PathBuf::from(env::var("OUT_DIR").unwrap()); - loop { - path.push("config.rust"); - if path.exists() { - return Ok(path.to_str().unwrap().to_owned()); - } - path.pop(); // remove config.rust - if !path.pop() { - // can't remove last part of directory - return Err(io::Error::new(io::ErrorKind::NotFound, "No config.rust")); - } - } -} - -impl Config { - /// Find the config.rust file and try to parse it. - /// - /// The file format is a series of lines of the form KEY=VAL, with - /// any blank lines and lines starting with # ignored. - fn load() -> io::Result<Config> { - let path = find_cfg()?; - let f = File::open(&path)?; - let reader = io::BufReader::new(f); - let mut map = HashMap::new(); - for line in reader.lines() { - let s = line?; - if s.trim().starts_with("#") || s.trim() == "" { - continue; - } - let idx = match s.find("=") { - None => { - return Err(io::Error::new(io::ErrorKind::InvalidData, "missing =")); - } - Some(x) => x, - }; - let (var, eq_val) = s.split_at(idx); - let val = &eq_val[1..]; - map.insert(var.to_owned(), val.to_owned()); - } - Ok(Config(map)) - } - - /// Return a reference to the value whose key is 'key'. - /// - /// Panics if 'key' is not found in the configuration. - fn get(&self, key: &str) -> &str { - self.0.get(key).unwrap() - } - - /// Add a dependency on a static C library that is part of Tor, by name. - fn component(&self, s: &str) { - println!("cargo:rustc-link-lib=static={}", s); - } - - /// Add a dependency on a native library that is not part of Tor, by name. - fn dependency(&self, s: &str) { - println!("cargo:rustc-link-lib={}", s); - } - - /// Add a link path, relative to Tor's build directory. - fn link_relpath(&self, s: &str) { - let builddir = self.get("BUILDDIR"); - println!("cargo:rustc-link-search=native={}/{}", builddir, s); - } - - /// Add an absolute link path. - fn link_path(&self, s: &str) { - println!("cargo:rustc-link-search=native={}", s); - } - - /// Parse the CFLAGS in s, looking for -l and -L items, and adding - /// rust configuration as appropriate. - fn from_cflags(&self, s: &str) { - let mut next_is_lib = false; - let mut next_is_path = false; - for ent in self.get(s).split_whitespace() { - if next_is_lib { - self.dependency(ent); - next_is_lib = false; - } else if next_is_path { - self.link_path(ent); - next_is_path = false; - } else if ent == "-l" { - next_is_lib = true; - } else if ent == "-L" { - next_is_path = true; - } else if ent.starts_with("-L") { - self.link_path(&ent[2..]); - } else if ent.starts_with("-l") { - self.dependency(&ent[2..]); - } - } - } -} - -pub fn main() { - let cfg = Config::load().unwrap(); - let package = env::var("CARGO_PKG_NAME").unwrap(); - - match package.as_ref() { - "crypto" => { - // Right now, I'm having a separate configuration for each Rust - // package, since I'm hoping we can trim them down. Once we have a - // second Rust package that needs to use this build script, let's - // extract some of this stuff into a module. - // - // This is a ridiculous amount of code to be pulling in just - // to test our crypto library: modularity would be our - // friend here. - cfg.from_cflags("TOR_LDFLAGS_zlib"); - cfg.from_cflags("TOR_LDFLAGS_openssl"); - cfg.from_cflags("TOR_LDFLAGS_libevent"); - - cfg.link_relpath("src/lib"); - cfg.link_relpath("src/ext/keccak-tiny"); - cfg.link_relpath("src/ext/ed25519/ref10"); - cfg.link_relpath("src/ext/ed25519/donna"); - cfg.link_relpath("src/trunnel"); - - // Note that we can't pull in "libtor-testing", or else we - // will have dependencies on all the other rust packages that - // tor uses. We must be careful with factoring and dependencies - // moving forward! - cfg.component("tor-crypt-ops-testing"); - cfg.component("tor-sandbox-testing"); - cfg.component("tor-encoding-testing"); - cfg.component("tor-fs-testing"); - cfg.component("tor-net-testing"); - cfg.component("tor-buf-testing"); - cfg.component("tor-time-testing"); - cfg.component("tor-thread-testing"); - cfg.component("tor-memarea-testing"); - cfg.component("tor-log-testing"); - cfg.component("tor-lock-testing"); - cfg.component("tor-fdio-testing"); - cfg.component("tor-container-testing"); - cfg.component("tor-smartlist-core-testing"); - cfg.component("tor-string-testing"); - cfg.component("tor-malloc"); - cfg.component("tor-wallclock"); - cfg.component("tor-err-testing"); - cfg.component("tor-version-testing"); - cfg.component("tor-intmath-testing"); - cfg.component("tor-ctime-testing"); - cfg.component("curve25519_donna"); - cfg.component("keccak-tiny"); - cfg.component("ed25519_ref10"); - cfg.component("ed25519_donna"); - cfg.component("or-trunnel-testing"); - - cfg.from_cflags("TOR_ZLIB_LIBS"); - cfg.from_cflags("TOR_LIB_MATH"); - cfg.from_cflags("NSS_LIBS"); - cfg.from_cflags("TOR_OPENSSL_LIBS"); - cfg.from_cflags("TOR_LIBEVENT_LIBS"); - cfg.from_cflags("TOR_LIB_WS32"); - cfg.from_cflags("TOR_LIB_GDI"); - cfg.from_cflags("TOR_LIB_USERENV"); - cfg.from_cflags("CURVE25519_LIBS"); - cfg.from_cflags("TOR_LZMA_LIBS"); - cfg.from_cflags("TOR_ZSTD_LIBS"); - cfg.from_cflags("LIBS"); - } - _ => { - panic!("No configuration in build.rs for package {}", package); - } - } -} diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml deleted file mode 100644 index a7ff7f78d9..0000000000 --- a/src/rust/crypto/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -authors = ["The Tor Project", - "Isis Lovecruft <isis@torproject.org>"] -name = "crypto" -version = "0.0.1" -publish = false -build = "../build.rs" - -[lib] -name = "crypto" -path = "lib.rs" - -[dependencies] -libc = "=0.2.39" -digest = "=0.7.2" -rand_core = { version = "=0.2.0-pre.0", default-features = false } - -external = { path = "../external" } -smartlist = { path = "../smartlist" } -tor_allocate = { path = "../tor_allocate" } -tor_log = { path = "../tor_log" } - -[dev-dependencies] -rand = { version = "=0.5.0-pre.2", default-features = false } -rand_core = { version = "=0.2.0-pre.0", default-features = false } - -[features] -# If this feature is enabled, test code which calls Tor C code from Rust will -# execute with `cargo test`. Due to numerous linker issues (#25386), this is -# currently disabled by default. -test-c-from-rust = [] - -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs deleted file mode 100644 index 58343b9ca7..0000000000 --- a/src/rust/crypto/digests/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Hash Digests and eXtendible Output Functions (XOFs) - -pub mod sha2; diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs deleted file mode 100644 index 91e8b2b3c9..0000000000 --- a/src/rust/crypto/digests/sha2.rs +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Hash Digests and eXtendible Output Functions (XOFs) - -pub use digest::Digest; - -use digest::generic_array::typenum::U32; -use digest::generic_array::typenum::U64; -use digest::generic_array::GenericArray; -use digest::BlockInput; -use digest::FixedOutput; -use digest::Input; - -use external::crypto_digest::get_256_bit_digest; -use external::crypto_digest::get_512_bit_digest; -use external::crypto_digest::CryptoDigest; -use external::crypto_digest::DigestAlgorithm; - -pub use external::crypto_digest::DIGEST256_LEN; -pub use external::crypto_digest::DIGEST512_LEN; - -/// The block size for both SHA-256 and SHA-512 digests is 512 bits/64 bytes. -/// -/// Unfortunately, we have to use the generic_array crate currently to express -/// this at compile time. Later, in the future, when Rust implements const -/// generics, we'll be able to remove this dependency (actually, it will get -/// removed from the digest crate, which is currently `pub use`ing it). -type BlockSize = U64; - -/// A SHA2-256 digest. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_dup` -#[derive(Clone)] -pub struct Sha256 { - engine: CryptoDigest, -} - -/// Construct a new, default instance of a `Sha256` hash digest function. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha256, Digest}; -/// -/// let mut hasher: Sha256 = Sha256::default(); -/// ``` -/// -/// # Returns -/// -/// A new `Sha256` digest. -impl Default for Sha256 { - fn default() -> Sha256 { - Sha256 { - engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_256)), - } - } -} - -impl BlockInput for Sha256 { - type BlockSize = BlockSize; -} - -/// Input `msg` into the digest. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha256, Digest}; -/// -/// let mut hasher: Sha256 = Sha256::default(); -/// -/// hasher.input(b"foo"); -/// hasher.input(b"bar"); -/// ``` -impl Input for Sha256 { - fn process(&mut self, msg: &[u8]) { - self.engine.add_bytes(&msg); - } -} - -/// Retrieve the output hash from everything which has been fed into this -/// `Sha256` digest thus far. -/// -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest in external::crypto_digest. -impl FixedOutput for Sha256 { - type OutputSize = U32; - - fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { - let buffer: [u8; DIGEST256_LEN] = get_256_bit_digest(self.engine); - - GenericArray::from(buffer) - } -} - -/// A SHA2-512 digest. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_dup` -#[derive(Clone)] -pub struct Sha512 { - engine: CryptoDigest, -} - -/// Construct a new, default instance of a `Sha512` hash digest function. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha512, Digest}; -/// -/// let mut hasher: Sha512 = Sha512::default(); -/// ``` -/// -/// # Returns -/// -/// A new `Sha512` digest. -impl Default for Sha512 { - fn default() -> Sha512 { - Sha512 { - engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_512)), - } - } -} - -impl BlockInput for Sha512 { - type BlockSize = BlockSize; -} - -/// Input `msg` into the digest. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha512, Digest}; -/// -/// let mut hasher: Sha512 = Sha512::default(); -/// -/// hasher.input(b"foo"); -/// hasher.input(b"bar"); -/// ``` -impl Input for Sha512 { - fn process(&mut self, msg: &[u8]) { - self.engine.add_bytes(&msg); - } -} - -/// Retrieve the output hash from everything which has been fed into this -/// `Sha512` digest thus far. -/// -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest in external::crypto_digest. -impl FixedOutput for Sha512 { - type OutputSize = U64; - - fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { - let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine); - - GenericArray::clone_from_slice(&buffer) - } -} - -#[cfg(test)] -mod test { - #[cfg(feature = "test-c-from-rust")] - use digest::Digest; - - #[cfg(feature = "test-c-from-rust")] - use super::*; - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha256_default() { - let _: Sha256 = Sha256::default(); - } - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha256_digest() { - let mut h: Sha256 = Sha256::new(); - let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; - let expected = [ - 151, 223, 53, 136, 181, 163, 242, 75, 171, 195, 133, 27, 55, 47, 11, 167, 26, 157, 205, - 222, 212, 59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157, - ]; - - h.input(b"foo"); - h.input(b"bar"); - h.input(b"baz"); - - result.copy_from_slice(h.fixed_result().as_slice()); - - println!("{:?}", &result[..]); - - assert_eq!(result, expected); - } - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha512_default() { - let _: Sha512 = Sha512::default(); - } - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha512_digest() { - let mut h: Sha512 = Sha512::new(); - let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; - - let expected = [ - 203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167, 153, 217, 233, 8, 190, 69, 231, - 103, 245, 209, 71, 212, 116, 73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20, - 122, 221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38, 179, 237, 61, 126, - 246, 240, 103, 202, 153, 24, 90, - ]; - - h.input(b"foo"); - h.input(b"bar"); - h.input(b"baz"); - - result.copy_from_slice(h.fixed_result().as_slice()); - - println!("{:?}", &result[..]); - - assert_eq!(&result[..], &expected[..]); - } -} diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs deleted file mode 100644 index 866ea93547..0000000000 --- a/src/rust/crypto/lib.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Common cryptographic functions and utilities. -//! -//! # Hash Digests and eXtendable Output Functions (XOFs) -//! -//! The `digests` module contains submodules for specific hash digests -//! and extendable output functions. -//! -//! ```rust,no_run -//! use crypto::digests::sha2::*; -//! -//! let mut hasher: Sha256 = Sha256::default(); -//! let mut result: [u8; 32] = [0u8; 32]; -//! -//! hasher.input(b"foo"); -//! hasher.input(b"bar"); -//! hasher.input(b"baz"); -//! -//! result.copy_from_slice(hasher.result().as_slice()); -//! -//! assert!(result == [b'X'; DIGEST256_LEN]); -//! ``` - -// XXX: add missing docs -//#![deny(missing_docs)] - -// External crates from cargo or TOR_RUST_DEPENDENCIES. -extern crate digest; -extern crate libc; -extern crate rand_core; - -// External dependencies for tests. -#[cfg(test)] -extern crate rand as rand_crate; - -// Our local crates. -extern crate external; -#[cfg(not(test))] -#[macro_use] -extern crate tor_log; - -pub mod digests; // Unfortunately named "digests" plural to avoid name conflict with the digest crate -pub mod rand; diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs deleted file mode 100644 index da8b3bd8a5..0000000000 --- a/src/rust/crypto/rand/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -// Internal dependencies -pub mod rng; diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs deleted file mode 100644 index 644a5c20b1..0000000000 --- a/src/rust/crypto/rand/rng.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Wrappers for Tor's random number generators to provide implementations of -//! `rand_core` traits. - -// This is the real implementation, in use in production, which calls into our C -// wrappers in /src/common/crypto_rand.c, which call into OpenSSL, system -// libraries, and make syscalls. -#[cfg(not(test))] -mod internal { - use std::u64; - - use rand_core::impls::next_u32_via_fill; - use rand_core::impls::next_u64_via_fill; - use rand_core::CryptoRng; - use rand_core::Error; - use rand_core::RngCore; - - use external::c_tor_crypto_rand; - use external::c_tor_crypto_seed_rng; - use external::c_tor_crypto_strongest_rand; - - use tor_log::LogDomain; - use tor_log::LogSeverity; - - /// Largest strong entropy request permitted. - // - // C_RUST_COUPLED: `MAX_STRONGEST_RAND_SIZE` /src/common/crypto_rand.c - const MAX_STRONGEST_RAND_SIZE: usize = 256; - - /// A wrapper around OpenSSL's RNG. - pub struct TorRng { - // This private, zero-length field forces the struct to be treated the - // same as its opaque C counterpart. - _unused: [u8; 0], - } - - /// Mark `TorRng` as being suitable for cryptographic purposes. - impl CryptoRng for TorRng {} - - impl TorRng { - // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c - #[allow(dead_code)] - pub fn new() -> Self { - if !c_tor_crypto_seed_rng() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::General, - "TorRng::from_seed()", - "The RNG could not be seeded!" - ); - } - // XXX also log success at info level —isis - TorRng { _unused: [0u8; 0] } - } - } - - impl RngCore for TorRng { - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u32(&mut self) -> u32 { - next_u32_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u64(&mut self) -> u64 { - next_u64_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn fill_bytes(&mut self, dest: &mut [u8]) { - c_tor_crypto_rand(dest); - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { - Ok(self.fill_bytes(dest)) - } - } - - /// A CSPRNG which hashes together randomness from OpenSSL's RNG and entropy - /// obtained from the operating system. - pub struct TorStrongestRng { - // This private, zero-length field forces the struct to be treated the - // same as its opaque C counterpart. - _unused: [u8; 0], - } - - /// Mark `TorRng` as being suitable for cryptographic purposes. - impl CryptoRng for TorStrongestRng {} - - impl TorStrongestRng { - // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c - #[allow(dead_code)] - pub fn new() -> Self { - if !c_tor_crypto_seed_rng() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::General, - "TorStrongestRng::from_seed()", - "The RNG could not be seeded!" - ); - } - // XXX also log success at info level —isis - TorStrongestRng { _unused: [0u8; 0] } - } - } - - impl RngCore for TorStrongestRng { - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u32(&mut self) -> u32 { - next_u32_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u64(&mut self) -> u64 { - next_u64_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn fill_bytes(&mut self, dest: &mut [u8]) { - debug_assert!(dest.len() <= MAX_STRONGEST_RAND_SIZE); - - c_tor_crypto_strongest_rand(dest); - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { - Ok(self.fill_bytes(dest)) - } - } -} - -// For testing, we expose a pure-Rust implementation. -#[cfg(test)] -mod internal { - // It doesn't matter if we pretend ChaCha is a CSPRNG in tests. - pub use rand_crate::ChaChaRng as TorRng; - pub use rand_crate::ChaChaRng as TorStrongestRng; -} - -// Finally, expose the public functionality of whichever appropriate internal -// module. -pub use self::internal::*; diff --git a/src/rust/external/Cargo.toml b/src/rust/external/Cargo.toml deleted file mode 100644 index 5f443645bb..0000000000 --- a/src/rust/external/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "external" - -[dependencies] -libc = "=0.2.39" -smartlist = { path = "../smartlist" } -tor_allocate = { path = "../tor_allocate" } - -[lib] -name = "external" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs deleted file mode 100644 index 873f75e7a3..0000000000 --- a/src/rust/external/crypto_digest.rs +++ /dev/null @@ -1,454 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Bindings to external digest and XOF functions which live within -//! src/common/crypto_digest.[ch]. -//! -//! We wrap our C implementations in src/common/crypto_digest.[ch] with more -//! Rusty types and interfaces in src/rust/crypto/digest/. - -use std::process::abort; - -use libc::c_char; -use libc::c_int; -use libc::size_t; -use libc::uint8_t; - -use smartlist::Stringlist; - -/// Length of the output of our message digest. -pub const DIGEST_LEN: usize = 20; - -/// Length of the output of our second (improved) message digests. (For now -/// this is just sha256, but it could be any other 256-bit digest.) -pub const DIGEST256_LEN: usize = 32; - -/// Length of the output of our 64-bit optimized message digests (SHA512). -pub const DIGEST512_LEN: usize = 64; - -/// Length of a sha1 message digest when encoded in base32 with trailing = signs -/// removed. -pub const BASE32_DIGEST_LEN: usize = 32; - -/// Length of a sha1 message digest when encoded in base64 with trailing = signs -/// removed. -pub const BASE64_DIGEST_LEN: usize = 27; - -/// Length of a sha256 message digest when encoded in base64 with trailing = -/// signs removed. -pub const BASE64_DIGEST256_LEN: usize = 43; - -/// Length of a sha512 message digest when encoded in base64 with trailing = -/// signs removed. -pub const BASE64_DIGEST512_LEN: usize = 86; - -/// Length of hex encoding of SHA1 digest, not including final NUL. -pub const HEX_DIGEST_LEN: usize = 40; - -/// Length of hex encoding of SHA256 digest, not including final NUL. -pub const HEX_DIGEST256_LEN: usize = 64; - -/// Length of hex encoding of SHA512 digest, not including final NUL. -pub const HEX_DIGEST512_LEN: usize = 128; - -/// Our C code uses an enum to declare the digest algorithm types which we know -/// about. However, because enums are implementation-defined in C, we can -/// neither work with them directly nor translate them into Rust enums. -/// Instead, we represent them as a u8 (under the assumption that we'll never -/// support more than 256 hash functions). -#[allow(non_camel_case_types)] -type digest_algorithm_t = u8; - -const DIGEST_SHA1: digest_algorithm_t = 0; -const DIGEST_SHA256: digest_algorithm_t = 1; -const DIGEST_SHA512: digest_algorithm_t = 2; -const DIGEST_SHA3_256: digest_algorithm_t = 3; -const DIGEST_SHA3_512: digest_algorithm_t = 4; - -/// The number of hash digests we produce for a `common_digests_t`. -/// -/// We can't access these from Rust, because their definitions in C require -/// introspecting the `digest_algorithm_t` typedef, which is an enum, so we have -/// to redefine them here. -const N_COMMON_DIGEST_ALGORITHMS: usize = DIGEST_SHA256 as usize + 1; - -/// A digest function. -#[repr(C)] -#[derive(Debug, Copy, Clone)] -#[allow(non_camel_case_types)] -struct crypto_digest_t { - // This private, zero-length field forces the struct to be treated the same - // as its opaque C counterpart. - _unused: [u8; 0], -} - -/// An eXtendible Output Function (XOF). -#[repr(C)] -#[derive(Debug, Copy, Clone)] -#[allow(non_camel_case_types)] -struct crypto_xof_t { - // This private, zero-length field forces the struct to be treated the same - // as its opaque C counterpart. - _unused: [u8; 0], -} - -/// A set of all the digests we commonly compute, taken on a single -/// string. Any digests that are shorter than 512 bits are right-padded -/// with 0 bits. -/// -/// Note that this representation wastes 44 bytes for the SHA1 case, so -/// don't use it for anything where we need to allocate a whole bunch at -/// once. -#[repr(C)] -#[derive(Debug, Copy, Clone)] -#[allow(non_camel_case_types)] -struct common_digests_t { - pub d: [[c_char; N_COMMON_DIGEST_ALGORITHMS]; DIGEST256_LEN], -} - -/// A `smartlist_t` is just an alias for the `#[repr(C)]` type `Stringlist`, to -/// make it more clear that we're working with a smartlist which is owned by C. -#[allow(non_camel_case_types)] -// BINDGEN_GENERATED: This type isn't actually bindgen generated, but the code -// below it which uses it is. As such, this comes up as "dead code" as well. -#[allow(dead_code)] -type smartlist_t = Stringlist; - -/// All of the external functions from `src/common/crypto_digest.h`. -/// -/// These are kept private because they should be wrapped with Rust to make their usage safer. -// -// BINDGEN_GENERATED: These definitions were generated with bindgen and cleaned -// up manually. As such, there are more bindings than are likely necessary or -// which are in use. -#[allow(dead_code)] -extern "C" { - fn crypto_digest(digest: *mut c_char, m: *const c_char, len: size_t) -> c_int; - fn crypto_digest256( - digest: *mut c_char, - m: *const c_char, - len: size_t, - algorithm: digest_algorithm_t, - ) -> c_int; - fn crypto_digest512( - digest: *mut c_char, - m: *const c_char, - len: size_t, - algorithm: digest_algorithm_t, - ) -> c_int; - fn crypto_common_digests(ds_out: *mut common_digests_t, m: *const c_char, len: size_t) - -> c_int; - fn crypto_digest_smartlist_prefix( - digest_out: *mut c_char, - len_out: size_t, - prepend: *const c_char, - lst: *const smartlist_t, - append: *const c_char, - alg: digest_algorithm_t, - ); - fn crypto_digest_smartlist( - digest_out: *mut c_char, - len_out: size_t, - lst: *const smartlist_t, - append: *const c_char, - alg: digest_algorithm_t, - ); - fn crypto_digest_algorithm_get_name(alg: digest_algorithm_t) -> *const c_char; - fn crypto_digest_algorithm_get_length(alg: digest_algorithm_t) -> size_t; - fn crypto_digest_algorithm_parse_name(name: *const c_char) -> c_int; - fn crypto_digest_new() -> *mut crypto_digest_t; - fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; - fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; - fn crypto_digest_free_(digest: *mut crypto_digest_t); - fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t); - fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t); - fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t; - fn crypto_digest_assign(into: *mut crypto_digest_t, from: *const crypto_digest_t); - fn crypto_hmac_sha256( - hmac_out: *mut c_char, - key: *const c_char, - key_len: size_t, - msg: *const c_char, - msg_len: size_t, - ); - fn crypto_mac_sha3_256( - mac_out: *mut uint8_t, - len_out: size_t, - key: *const uint8_t, - key_len: size_t, - msg: *const uint8_t, - msg_len: size_t, - ); - fn crypto_xof_new() -> *mut crypto_xof_t; - fn crypto_xof_add_bytes(xof: *mut crypto_xof_t, data: *const uint8_t, len: size_t); - fn crypto_xof_squeeze_bytes(xof: *mut crypto_xof_t, out: *mut uint8_t, len: size_t); - fn crypto_xof_free(xof: *mut crypto_xof_t); -} - -/// A wrapper around a `digest_algorithm_t`. -pub enum DigestAlgorithm { - SHA2_256, - SHA2_512, - SHA3_256, - SHA3_512, -} - -impl From<DigestAlgorithm> for digest_algorithm_t { - fn from(digest: DigestAlgorithm) -> digest_algorithm_t { - match digest { - DigestAlgorithm::SHA2_256 => DIGEST_SHA256, - DigestAlgorithm::SHA2_512 => DIGEST_SHA512, - DigestAlgorithm::SHA3_256 => DIGEST_SHA3_256, - DigestAlgorithm::SHA3_512 => DIGEST_SHA3_512, - } - } -} - -/// A wrapper around a mutable pointer to a `crypto_digest_t`. -pub struct CryptoDigest(*mut crypto_digest_t); - -/// Explicitly copy the state of a `CryptoDigest` hash digest context. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_dup` -impl Clone for CryptoDigest { - fn clone(&self) -> CryptoDigest { - let digest: *mut crypto_digest_t; - - unsafe { - digest = crypto_digest_dup(self.0 as *const crypto_digest_t); - } - - // See the note in the implementation of CryptoDigest for the - // reasoning for `abort()` here. - if digest.is_null() { - abort(); - } - - CryptoDigest(digest) - } -} - -impl CryptoDigest { - /// A wrapper to call one of the C functions `crypto_digest_new`, - /// `crypto_digest256_new`, or `crypto_digest512_new`. - /// - /// # Warnings - /// - /// This function will `abort()` the entire process in an "abnormal" fashion, - /// i.e. not unwinding this or any other thread's stack, running any - /// destructors, or calling any panic/exit hooks) if `tor_malloc()` (called in - /// `crypto_digest256_new()`) is unable to allocate memory. - /// - /// # Returns - /// - /// A new `CryptoDigest`, which is a wrapper around a opaque representation - /// of a `crypto_digest_t`. The underlying `crypto_digest_t` _MUST_ only - /// ever be handled via a raw pointer, and never introspected. - /// - /// # C_RUST_COUPLED - /// - /// * `crypto_digest_new` - /// * `crypto_digest256_new` - /// * `crypto_digest512_new` - /// * `tor_malloc` (called by `crypto_digest256_new`, but we make - /// assumptions about its behaviour and return values here) - pub fn new(algorithm: Option<DigestAlgorithm>) -> CryptoDigest { - let digest: *mut crypto_digest_t; - - if algorithm.is_none() { - unsafe { - digest = crypto_digest_new(); - } - } else { - let algo: digest_algorithm_t = algorithm.unwrap().into(); // can't fail because it's Some - - unsafe { - // XXX This is a pretty awkward API to use from Rust... - digest = match algo { - DIGEST_SHA1 => crypto_digest_new(), - DIGEST_SHA256 => crypto_digest256_new(DIGEST_SHA256), - DIGEST_SHA3_256 => crypto_digest256_new(DIGEST_SHA3_256), - DIGEST_SHA512 => crypto_digest512_new(DIGEST_SHA512), - DIGEST_SHA3_512 => crypto_digest512_new(DIGEST_SHA3_512), - _ => abort(), - } - } - } - - // In our C code, `crypto_digest*_new()` allocates memory with - // `tor_malloc()`. In `tor_malloc()`, if the underlying malloc - // implementation fails to allocate the requested memory and returns a - // NULL pointer, we call `exit(1)`. In the case that this `exit(1)` is - // called within a worker, be that a process or a thread, the inline - // comments within `tor_malloc()` mention "that's ok, since the parent - // will run out of memory soon anyway". However, if it takes long - // enough for the worker to die, and it manages to return a NULL pointer - // to our Rust code, our Rust is now in an irreparably broken state and - // may exhibit undefined behaviour. An even worse scenario, if/when we - // have parent/child processes/threads controlled by Rust, would be that - // the UB contagion in Rust manages to spread to other children before - // the entire process (hopefully terminates). - // - // However, following the assumptions made in `tor_malloc()` that - // calling `exit(1)` in a child is okay because the parent will - // eventually run into the same errors, and also to stymie any UB - // contagion in the meantime, we call abort!() here to terminate the - // entire program immediately. - if digest.is_null() { - abort(); - } - - CryptoDigest(digest) - } - - /// A wrapper to call the C function `crypto_digest_add_bytes`. - /// - /// # Inputs - /// - /// * `bytes`: a byte slice of bytes to be added into this digest. - /// - /// # C_RUST_COUPLED - /// - /// * `crypto_digest_add_bytes` - pub fn add_bytes(&self, bytes: &[u8]) { - unsafe { - crypto_digest_add_bytes( - self.0 as *mut crypto_digest_t, - bytes.as_ptr() as *const c_char, - bytes.len() as size_t, - ) - } - } -} - -impl Drop for CryptoDigest { - fn drop(&mut self) { - unsafe { - crypto_digest_free_(self.0 as *mut crypto_digest_t); - } - } -} - -/// Get the 256-bit digest output of a `crypto_digest_t`. -/// -/// # Inputs -/// -/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA256` or a -/// `DIGEST_SHA3_256`. -/// -/// # Warning -/// -/// Calling this function with a `CryptoDigest` which is neither SHA2-256 or -/// SHA3-256 is a programming error. Since we cannot introspect the opaque -/// struct from Rust, however, there is no way for us to check that the correct -/// one is being passed in. That is up to you, dear programmer. If you mess -/// up, you will get a incorrectly-sized hash digest in return, and it will be -/// your fault. Don't do that. -/// -/// # Returns -/// -/// A 256-bit hash digest, as a `[u8; 32]`. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_get_digest` -/// * `DIGEST256_LEN` -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest w.r.t. output array size. -pub fn get_256_bit_digest(digest: CryptoDigest) -> [u8; DIGEST256_LEN] { - let mut buffer: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; - - unsafe { - crypto_digest_get_digest( - digest.0, - buffer.as_mut_ptr() as *mut c_char, - DIGEST256_LEN as size_t, - ); - - if buffer.as_ptr().is_null() { - abort(); - } - } - buffer -} - -/// Get the 512-bit digest output of a `crypto_digest_t`. -/// -/// # Inputs -/// -/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA512` or a -/// `DIGEST_SHA3_512`. -/// -/// # Warning -/// -/// Calling this function with a `CryptoDigest` which is neither SHA2-512 or -/// SHA3-512 is a programming error. Since we cannot introspect the opaque -/// struct from Rust, however, there is no way for us to check that the correct -/// one is being passed in. That is up to you, dear programmer. If you mess -/// up, you will get a incorrectly-sized hash digest in return, and it will be -/// your fault. Don't do that. -/// -/// # Returns -/// -/// A 512-bit hash digest, as a `[u8; 64]`. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_get_digest` -/// * `DIGEST512_LEN` -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest w.r.t. output array size. -pub fn get_512_bit_digest(digest: CryptoDigest) -> [u8; DIGEST512_LEN] { - let mut buffer: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; - - unsafe { - crypto_digest_get_digest( - digest.0, - buffer.as_mut_ptr() as *mut c_char, - DIGEST512_LEN as size_t, - ); - - if buffer.as_ptr().is_null() { - abort(); - } - } - buffer -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_layout_common_digests_t() { - assert_eq!( - ::std::mem::size_of::<common_digests_t>(), - 64usize, - concat!("Size of: ", stringify!(common_digests_t)) - ); - assert_eq!( - ::std::mem::align_of::<common_digests_t>(), - 1usize, - concat!("Alignment of ", stringify!(common_digests_t)) - ); - } - - #[test] - fn test_layout_crypto_digest_t() { - assert_eq!( - ::std::mem::size_of::<crypto_digest_t>(), - 0usize, - concat!("Size of: ", stringify!(crypto_digest_t)) - ); - assert_eq!( - ::std::mem::align_of::<crypto_digest_t>(), - 1usize, - concat!("Alignment of ", stringify!(crypto_digest_t)) - ); - } -} diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs deleted file mode 100644 index 703382093c..0000000000 --- a/src/rust/external/crypto_rand.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Bindings to external (P)RNG interfaces and utilities in -//! src/common/crypto_rand.[ch]. -//! -//! We wrap our C implementations in src/common/crypto_rand.[ch] here in order -//! to provide wrappers with native Rust types, and then provide more Rusty -//! types and and trait implementations in src/rust/crypto/rand/. - -use std::time::Duration; - -use libc::c_double; -use libc::c_int; -use libc::size_t; -use libc::time_t; -use libc::uint8_t; - -extern "C" { - fn crypto_seed_rng() -> c_int; - fn crypto_rand(out: *mut uint8_t, out_len: size_t); - fn crypto_strongest_rand(out: *mut uint8_t, out_len: size_t); - fn crypto_rand_time_range(min: time_t, max: time_t) -> time_t; - fn crypto_rand_double() -> c_double; -} - -/// Seed OpenSSL's random number generator with bytes from the operating -/// system. -/// -/// # Returns -/// -/// `true` on success; `false` on failure. -pub fn c_tor_crypto_seed_rng() -> bool { - let ret: c_int; - - unsafe { - ret = crypto_seed_rng(); - } - match ret { - 0 => return true, - _ => return false, - } -} - -/// Fill the bytes of `dest` with random data. -pub fn c_tor_crypto_rand(dest: &mut [u8]) { - unsafe { - crypto_rand(dest.as_mut_ptr(), dest.len() as size_t); - } -} - -/// Fill the bytes of `dest` with "strong" random data by hashing -/// together randomness obtained from OpenSSL's RNG and the operating -/// system. -pub fn c_tor_crypto_strongest_rand(dest: &mut [u8]) { - // We'll let the C side panic if the len is larger than - // MAX_STRONGEST_RAND_SIZE, rather than potentially panicking here. A - // paranoid caller should assert on the length of dest *before* calling this - // function. - unsafe { - crypto_strongest_rand(dest.as_mut_ptr(), dest.len() as size_t); - } -} - -/// Get a random time, in seconds since the Unix Epoch. -/// -/// # Returns -/// -/// A `std::time::Duration` of seconds since the Unix Epoch. -pub fn c_tor_crypto_rand_time_range(min: &Duration, max: &Duration) -> Duration { - let ret: time_t; - - unsafe { - ret = crypto_rand_time_range(min.as_secs() as time_t, max.as_secs() as time_t); - } - - Duration::from_secs(ret as u64) -} - -/// Return a pseudorandom 64-bit float, chosen uniformly from the range [0.0, 1.0). -pub fn c_tor_crypto_rand_double() -> f64 { - unsafe { crypto_rand_double() } -} diff --git a/src/rust/external/external.rs b/src/rust/external/external.rs deleted file mode 100644 index 0d324c8820..0000000000 --- a/src/rust/external/external.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -use libc::{c_char, c_int}; -use std::ffi::CString; - -extern "C" { - fn tor_version_as_new_as(platform: *const c_char, cutoff: *const c_char) -> c_int; -} - -/// Wrap calls to tor_version_as_new_as, defined in routerparse.c -pub fn c_tor_version_as_new_as(platform: &str, cutoff: &str) -> bool { - // CHK: These functions should log a warning if an error occurs. This - // can be added when integration with tor's logger is added to rust - let c_platform = match CString::new(platform) { - Ok(n) => n, - Err(_) => return false, - }; - - let c_cutoff = match CString::new(cutoff) { - Ok(n) => n, - Err(_) => return false, - }; - - let result: c_int = unsafe { tor_version_as_new_as(c_platform.as_ptr(), c_cutoff.as_ptr()) }; - - result == 1 -} - -extern "C" { - fn tor_is_using_nss() -> c_int; -} - -/// Return true if Tor was built to use NSS. -pub fn c_tor_is_using_nss() -> bool { - 0 != unsafe { tor_is_using_nss() } -} diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs deleted file mode 100644 index 2f50610a4d..0000000000 --- a/src/rust/external/lib.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Copyright (c) 2016-2019, The Tor Project, Inc. */ -//! See LICENSE for licensing information */ - -//! Interface for external calls to tor C ABI -//! -//! The purpose of this module is to provide a clean interface for when Rust -//! modules need to interact with functionality in tor C code rather than each -//! module implementing this functionality repeatedly. - -extern crate libc; -extern crate tor_allocate; -extern crate smartlist; - -pub mod crypto_digest; -mod crypto_rand; -mod external; - -pub use crypto_rand::*; -pub use external::*; diff --git a/src/rust/include.am b/src/rust/include.am deleted file mode 100644 index 5e5b0b3faf..0000000000 --- a/src/rust/include.am +++ /dev/null @@ -1,41 +0,0 @@ -include src/rust/tor_rust/include.am - -EXTRA_DIST +=\ - src/rust/build.rs \ - src/rust/Cargo.toml \ - src/rust/Cargo.lock \ - src/rust/.cargo/config.in \ - src/rust/crypto/Cargo.toml \ - src/rust/crypto/lib.rs \ - src/rust/crypto/digests/mod.rs \ - src/rust/crypto/digests/sha2.rs \ - src/rust/crypto/rand/mod.rs \ - src/rust/crypto/rand/rng.rs \ - src/rust/external/Cargo.toml \ - src/rust/external/crypto_digest.rs \ - src/rust/external/crypto_rand.rs \ - src/rust/external/external.rs \ - src/rust/external/lib.rs \ - src/rust/protover/Cargo.toml \ - src/rust/protover/errors.rs \ - src/rust/protover/protoset.rs \ - src/rust/protover/ffi.rs \ - src/rust/protover/lib.rs \ - src/rust/protover/protover.rs \ - src/rust/protover/tests/protover.rs \ - src/rust/smartlist/Cargo.toml \ - src/rust/smartlist/lib.rs \ - src/rust/smartlist/smartlist.rs \ - src/rust/tor_allocate/Cargo.toml \ - src/rust/tor_allocate/lib.rs \ - src/rust/tor_allocate/tor_allocate.rs \ - src/rust/tor_log/Cargo.toml \ - src/rust/tor_log/lib.rs \ - src/rust/tor_log/tor_log.rs \ - src/rust/tor_rust/Cargo.toml \ - src/rust/tor_rust/include.am \ - src/rust/tor_rust/lib.rs \ - src/rust/tor_util/Cargo.toml \ - src/rust/tor_util/ffi.rs \ - src/rust/tor_util/lib.rs \ - src/rust/tor_util/strings.rs diff --git a/src/rust/protover/Cargo.toml b/src/rust/protover/Cargo.toml deleted file mode 100644 index 84a7c71c1a..0000000000 --- a/src/rust/protover/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "protover" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] - -[dependencies] -libc = "=0.2.39" - -[dependencies.smartlist] -path = "../smartlist" - -[dependencies.external] -path = "../external" - -[dependencies.tor_util] -path = "../tor_util" - -[dependencies.tor_allocate] -path = "../tor_allocate" - -[dependencies.tor_log] -path = "../tor_log" - -[lib] -name = "protover" -path = "lib.rs" diff --git a/src/rust/protover/errors.rs b/src/rust/protover/errors.rs deleted file mode 100644 index 04397ac4fe..0000000000 --- a/src/rust/protover/errors.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Various errors which may occur during protocol version parsing. - -use std::fmt; -use std::fmt::Display; - -/// All errors which may occur during protover parsing routines. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -#[allow(missing_docs)] // See Display impl for error descriptions -pub enum ProtoverError { - Overlap, - LowGreaterThanHigh, - Unparseable, - ExceedsMax, - ExceedsExpansionLimit, - UnknownProtocol, - ExceedsNameLimit, - InvalidProtocol, -} - -/// Descriptive error messages for `ProtoverError` variants. -impl Display for ProtoverError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - ProtoverError::Overlap => write!( - f, - "Two or more (low, high) protover ranges would overlap once expanded." - ), - ProtoverError::LowGreaterThanHigh => write!( - f, - "The low in a (low, high) protover range was greater than high." - ), - ProtoverError::Unparseable => write!(f, "The protover string was unparseable."), - ProtoverError::ExceedsMax => write!( - f, - "The high in a (low, high) protover range exceeds 63." - ), - ProtoverError::ExceedsExpansionLimit => write!( - f, - "The protover string would exceed the maximum expansion limit." - ), - ProtoverError::UnknownProtocol => write!( - f, - "A protocol in the protover string we attempted to parse is unknown." - ), - ProtoverError::ExceedsNameLimit => { - write!(f, "An unrecognised protocol name was too long.") - } - ProtoverError::InvalidProtocol => { - write!(f, "A protocol name includes invalid characters.") - } - } - } -} diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs deleted file mode 100644 index 2bf8d3a987..0000000000 --- a/src/rust/protover/ffi.rs +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! FFI functions, only to be called from C. -//! -//! Equivalent C versions of this api are in `protover.c` - -use libc::{c_char, c_int, uint32_t}; -use std::ffi::CStr; - -use smartlist::*; -use tor_allocate::allocate_and_copy_string; - -use errors::ProtoverError; -use protover::*; - -/// Translate C enums to Rust Proto enums, using the integer value of the C -/// enum to map to its associated Rust enum. -/// -/// C_RUST_COUPLED: protover.h `protocol_type_t` -fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> { - match c_proto { - 0 => Ok(Protocol::Link), - 1 => Ok(Protocol::LinkAuth), - 2 => Ok(Protocol::Relay), - 3 => Ok(Protocol::DirCache), - 4 => Ok(Protocol::HSDir), - 5 => Ok(Protocol::HSIntro), - 6 => Ok(Protocol::HSRend), - 7 => Ok(Protocol::Desc), - 8 => Ok(Protocol::Microdesc), - 9 => Ok(Protocol::Cons), - 10 => Ok(Protocol::Padding), - 11 => Ok(Protocol::FlowCtrl), - _ => Err(ProtoverError::UnknownProtocol), - } -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::all_supported -#[no_mangle] -pub extern "C" fn protover_all_supported( - c_relay_version: *const c_char, - missing_out: *mut *mut c_char, -) -> c_int { - if c_relay_version.is_null() { - return 1; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_relay_version) }; - - let relay_version = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 1, - }; - - let relay_proto_entry: UnvalidatedProtoEntry = - match UnvalidatedProtoEntry::from_str_any_len(relay_version) { - Ok(n) => n, - Err(_) => return 1, - }; - - if let Some(unsupported) = relay_proto_entry.all_supported() { - if missing_out.is_null() { - return 0; - } - let ptr = allocate_and_copy_string(&unsupported.to_string()); - unsafe { *missing_out = ptr }; - - return 0; - } - - 1 -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::list_supports_protocol -#[no_mangle] -pub extern "C" fn protocol_list_supports_protocol( - c_protocol_list: *const c_char, - c_protocol: uint32_t, - version: uint32_t, -) -> c_int { - if c_protocol_list.is_null() { - return 0; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) }; - - let protocol_list = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 0, - }; - let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() { - Ok(n) => n, - Err(_) => return 0, - }; - let protocol: UnknownProtocol = match translate_to_rust(c_protocol) { - Ok(n) => n.into(), - Err(_) => return 0, - }; - if proto_entry.supports_protocol(&protocol, &version) { - 1 - } else { - 0 - } -} - -#[no_mangle] -pub extern "C" fn protover_contains_long_protocol_names_(c_protocol_list: *const c_char) -> c_int { - if c_protocol_list.is_null() { - return 1; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) }; - - let protocol_list = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 1, - }; - - match protocol_list.parse::<UnvalidatedProtoEntry>() { - Ok(_) => 0, - Err(_) => 1, - } -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::list_supports_protocol_or_later -#[no_mangle] -pub extern "C" fn protocol_list_supports_protocol_or_later( - c_protocol_list: *const c_char, - c_protocol: uint32_t, - version: uint32_t, -) -> c_int { - if c_protocol_list.is_null() { - return 0; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) }; - - let protocol_list = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 0, - }; - - let protocol = match translate_to_rust(c_protocol) { - Ok(n) => n, - Err(_) => return 0, - }; - - let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() { - Ok(n) => n, - Err(_) => return 0, - }; - - if proto_entry.supports_protocol_or_later(&protocol.into(), &version) { - return 1; - } - 0 -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::get_supported_protocols -#[no_mangle] -pub extern "C" fn protover_get_supported_protocols() -> *const c_char { - let supported: &'static CStr; - - supported = get_supported_protocols_cstr(); - supported.as_ptr() -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::compute_vote -// -// Why is the threshold a signed integer? —isis -#[no_mangle] -pub extern "C" fn protover_compute_vote(list: *const Stringlist, threshold: c_int) -> *mut c_char { - if list.is_null() { - return allocate_and_copy_string(""); - } - - // Dereference of raw pointer requires an unsafe block. The pointer is - // checked above to ensure it is not null. - let data: Vec<String> = unsafe { (*list).get_list() }; - let hold: usize = threshold as usize; - let mut proto_entries: Vec<UnvalidatedProtoEntry> = Vec::new(); - - for datum in data { - let entry: UnvalidatedProtoEntry = match datum.parse() { - Ok(n) => n, - Err(_) => continue, - }; - proto_entries.push(entry); - } - let vote: UnvalidatedProtoEntry = ProtoverVote::compute(&proto_entries, &hold); - - allocate_and_copy_string(&vote.to_string()) -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::is_supported_here -#[no_mangle] -pub extern "C" fn protover_is_supported_here(c_protocol: uint32_t, version: uint32_t) -> c_int { - let protocol = match translate_to_rust(c_protocol) { - Ok(n) => n, - Err(_) => return 0, - }; - - let is_supported = is_supported_here(&protocol, &version); - - return if is_supported { 1 } else { 0 }; -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::compute_for_old_tor -#[no_mangle] -pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const c_char { - let supported: &'static CStr; - let empty: &'static CStr; - - empty = cstr!(""); - - if version.is_null() { - return empty.as_ptr(); - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(version) }; - - let version = match c_str.to_str() { - Ok(n) => n, - Err(_) => return empty.as_ptr(), - }; - - supported = compute_for_old_tor_cstr(&version); - supported.as_ptr() -} diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs deleted file mode 100644 index 35c4106ae5..0000000000 --- a/src/rust/protover/lib.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Copyright (c) 2016-2019, The Tor Project, Inc. */ -//! See LICENSE for licensing information */ - -//! Versioning information for different pieces of the Tor protocol. -//! -//! The below description is taken from src/rust/protover.c, which is currently -//! enabled by default. We are in the process of experimenting with Rust in -//! tor, and this protover module is implemented to help achieve this goal. -//! -//! Starting in version 0.2.9.3-alpha, Tor places separate version numbers on -//! each of the different components of its protocol. Relays use these numbers -//! to advertise what versions of the protocols they can support, and clients -//! use them to find what they can ask a given relay to do. Authorities vote -//! on the supported protocol versions for each relay, and also vote on the -//! which protocols you should have to support in order to be on the Tor -//! network. All Tor instances use these required/recommended protocol versions -//! to tell what level of support for recent protocols each relay has, and -//! to decide whether they should be running given their current protocols. -//! -//! The main advantage of these protocol versions numbers over using Tor -//! version numbers is that they allow different implementations of the Tor -//! protocols to develop independently, without having to claim compatibility -//! with specific versions of Tor. - -// XXX: add missing docs -//#![deny(missing_docs)] - -extern crate external; -extern crate libc; -extern crate smartlist; -extern crate tor_allocate; -#[macro_use] -extern crate tor_util; - -pub mod errors; -pub mod ffi; -pub mod protoset; -mod protover; - -pub use protover::*; diff --git a/src/rust/protover/protoset.rs b/src/rust/protover/protoset.rs deleted file mode 100644 index 0ab94457c5..0000000000 --- a/src/rust/protover/protoset.rs +++ /dev/null @@ -1,697 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Sets for lazily storing ordered, non-overlapping ranges of integers. - -use std::cmp; -use std::iter; -use std::slice; -use std::str::FromStr; -use std::u32; - -use errors::ProtoverError; - -/// A single version number. -pub type Version = u32; - -/// A `ProtoSet` stores an ordered `Vec<T>` of `(low, high)` pairs of ranges of -/// non-overlapping protocol versions. -/// -/// # Examples -/// -/// ``` -/// use std::str::FromStr; -/// -/// use protover::errors::ProtoverError; -/// use protover::protoset::ProtoSet; -/// use protover::protoset::Version; -/// -/// # fn do_test() -> Result<ProtoSet, ProtoverError> { -/// let protoset: ProtoSet = ProtoSet::from_str("3-5,8")?; -/// -/// // We could also equivalently call: -/// let protoset: ProtoSet = "3-5,8".parse()?; -/// -/// assert!(protoset.contains(&4)); -/// assert!(!protoset.contains(&7)); -/// -/// let expanded: Vec<Version> = protoset.clone().into(); -/// -/// assert_eq!(&expanded[..], &[3, 4, 5, 8]); -/// -/// let contracted: String = protoset.clone().to_string(); -/// -/// assert_eq!(contracted, "3-5,8".to_string()); -/// # Ok(protoset) -/// # } -/// # fn main() { do_test(); } // wrap the test so we can use the ? operator -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct ProtoSet { - pub(crate) pairs: Vec<(Version, Version)>, -} - -impl Default for ProtoSet { - fn default() -> Self { - let pairs: Vec<(Version, Version)> = Vec::new(); - - ProtoSet { pairs } - } -} - -impl<'a> ProtoSet { - /// Create a new `ProtoSet` from a slice of `(low, high)` pairs. - /// - /// # Inputs - /// - /// We do not assume the input pairs are deduplicated or ordered. - pub fn from_slice(low_high_pairs: &'a [(Version, Version)]) -> Result<Self, ProtoverError> { - let mut pairs: Vec<(Version, Version)> = Vec::with_capacity(low_high_pairs.len()); - - for &(low, high) in low_high_pairs { - pairs.push((low, high)); - } - // Sort the pairs without reallocation and remove all duplicate pairs. - pairs.sort_unstable(); - pairs.dedup(); - - ProtoSet { pairs }.is_ok() - } -} - -/// Expand this `ProtoSet` to a `Vec` of all its `Version`s. -/// -/// # Examples -/// -/// ``` -/// use std::str::FromStr; -/// use protover::protoset::ProtoSet; -/// use protover::protoset::Version; -/// # use protover::errors::ProtoverError; -/// -/// # fn do_test() -> Result<Vec<Version>, ProtoverError> { -/// let protoset: ProtoSet = ProtoSet::from_str("3-5,21")?; -/// let versions: Vec<Version> = protoset.into(); -/// -/// assert_eq!(&versions[..], &[3, 4, 5, 21]); -/// # -/// # Ok(versions) -/// # } -/// # fn main() { do_test(); } // wrap the test so we can use the ? operator -/// ``` -impl Into<Vec<Version>> for ProtoSet { - fn into(self) -> Vec<Version> { - let mut versions: Vec<Version> = Vec::new(); - - for &(low, high) in self.iter() { - versions.extend(low..high + 1); - } - versions - } -} - -impl ProtoSet { - /// Get an iterator over the `(low, high)` `pairs` in this `ProtoSet`. - pub fn iter(&self) -> slice::Iter<(Version, Version)> { - self.pairs.iter() - } - - /// Expand this `ProtoSet` into a `Vec` of all its `Version`s. - /// - /// # Examples - /// - /// ``` - /// # use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<bool, ProtoverError> { - /// let protoset: ProtoSet = "3-5,9".parse()?; - /// - /// assert_eq!(protoset.expand(), vec![3, 4, 5, 9]); - /// - /// let protoset: ProtoSet = "1,3,5-7".parse()?; - /// - /// assert_eq!(protoset.expand(), vec![1, 3, 5, 6, 7]); - /// # - /// # Ok(true) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - pub fn expand(self) -> Vec<Version> { - self.into() - } - - pub fn len(&self) -> usize { - let mut length: usize = 0; - - for &(low, high) in self.iter() { - length += (high as usize - low as usize) + 1; - } - - length - } - - /// Check that this `ProtoSet` is well-formed. - /// - /// This is automatically called in `ProtoSet::from_str()`. - /// - /// # Errors - /// - /// * `ProtoverError::LowGreaterThanHigh`: if its `pairs` were not - /// well-formed, i.e. a `low` in a `(low, high)` was higher than the - /// previous `high`, - /// * `ProtoverError::Overlap`: if one or more of the `pairs` are - /// overlapping, - /// * `ProtoverError::ExceedsMax`: if the number of versions when expanded - /// would exceed `MAX_PROTOCOLS_TO_EXPAND`, and - /// - /// # Returns - /// - /// A `Result` whose `Ok` is this `Protoset`, and whose `Err` is one of the - /// errors enumerated in the Errors section above. - fn is_ok(self) -> Result<ProtoSet, ProtoverError> { - let mut last_high: Version = 0; - - for &(low, high) in self.iter() { - if low == u32::MAX || high == u32::MAX { - return Err(ProtoverError::ExceedsMax); - } - if low <= last_high { - return Err(ProtoverError::Overlap); - } else if low > high { - return Err(ProtoverError::LowGreaterThanHigh); - } - last_high = high; - } - - Ok(self) - } - - /// Determine if this `ProtoSet` contains no `Version`s. - /// - /// # Returns - /// - /// * `true` if this `ProtoSet`'s length is zero, and - /// * `false` otherwise. - /// - /// # Examples - /// - /// ``` - /// use protover::protoset::ProtoSet; - /// - /// let protoset: ProtoSet = ProtoSet::default(); - /// - /// assert!(protoset.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.pairs.len() == 0 - } - - /// Determine if `version` is included within this `ProtoSet`. - /// - /// # Inputs - /// - /// * `version`: a `Version`. - /// - /// # Returns - /// - /// `true` if the `version` is contained within this set; `false` otherwise. - /// - /// # Examples - /// - /// ``` - /// # use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<ProtoSet, ProtoverError> { - /// let protoset: ProtoSet = ProtoSet::from_slice(&[(0, 5), (7, 9), (13, 14)])?; - /// - /// assert!(protoset.contains(&5)); - /// assert!(!protoset.contains(&10)); - /// # - /// # Ok(protoset) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - pub fn contains(&self, version: &Version) -> bool { - for &(low, high) in self.iter() { - if low <= *version && *version <= high { - return true; - } - } - false - } - - /// Returns all the `Version`s in `self` which are not also in the `other` - /// `ProtoSet`. - /// - /// # Examples - /// - /// ``` - /// # use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<bool, ProtoverError> { - /// let protoset: ProtoSet = "1,3-6,10-12,15-16".parse()?; - /// let other: ProtoSet = "2,5-7,9-11,14-20".parse()?; - /// - /// let subset: ProtoSet = protoset.and_not_in(&other); - /// - /// assert_eq!(subset.expand(), vec![1, 3, 4, 12]); - /// # - /// # Ok(true) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - pub fn and_not_in(&self, other: &Self) -> Self { - if self.is_empty() || other.is_empty() { - return self.clone(); - } - - let pairs = self.iter().flat_map(|&(lo, hi)| { - let the_end = (hi + 1, hi + 1); // special case to mark the end of the range. - let excluded_ranges = other - .iter() - .cloned() // have to be owned tuples, to match iter::once(the_end). - .skip_while(move|&(_, hi2)| hi2 < lo) // skip the non-overlapping ranges. - .take_while(move|&(lo2, _)| lo2 <= hi) // take all the overlapping ones. - .chain(iter::once(the_end)); - - let mut nextlo = lo; - excluded_ranges.filter_map(move |(excluded_lo, excluded_hi)| { - let pair = if nextlo < excluded_lo { - Some((nextlo, excluded_lo - 1)) - } else { - None - }; - nextlo = cmp::min(excluded_hi, u32::MAX - 1) + 1; - pair - }) - }); - - let pairs = pairs.collect(); - ProtoSet::is_ok(ProtoSet { pairs }).expect("should be already sorted") - } -} - -/// Largest allowed protocol version. -/// C_RUST_COUPLED: protover.c `MAX_PROTOCOL_VERSION` -const MAX_PROTOCOL_VERSION: Version = 63; - -impl FromStr for ProtoSet { - type Err = ProtoverError; - - /// Parse the unique version numbers supported by a subprotocol from a string. - /// - /// # Inputs - /// - /// * `version_string`, a string comprised of "[0-9,-]" - /// - /// # Returns - /// - /// A `Result` whose `Ok` value is a `ProtoSet` holding all of the unique - /// version numbers. - /// - /// The returned `Result`'s `Err` value is an `ProtoverError` appropriate to - /// the error. - /// - /// # Errors - /// - /// This function will error if: - /// - /// * the `version_string` is an equals (`"="`) sign, - /// * the expansion of a version range produces an error (see - /// `expand_version_range`), - /// * any single version number is not parseable as an `u32` in radix 10, or - /// * there are greater than 2^16 version numbers to expand. - /// - /// # Examples - /// - /// ``` - /// use std::str::FromStr; - /// - /// use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<ProtoSet, ProtoverError> { - /// let protoset: ProtoSet = ProtoSet::from_str("2-5,8")?; - /// - /// assert!(protoset.contains(&5)); - /// assert!(!protoset.contains(&10)); - /// - /// // We can also equivalently call `ProtoSet::from_str` by doing (all - /// // implementations of `FromStr` can be called this way, this one isn't - /// // special): - /// let protoset: ProtoSet = "4-6,12".parse()?; - /// - /// // Calling it (either way) can take really large ranges (up to `u32::MAX`): - /// let protoset: ProtoSet = "1-70000".parse()?; - /// let protoset: ProtoSet = "1-4294967296".parse()?; - /// - /// // There are lots of ways to get an `Err` from this function. Here are - /// // a few: - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("=")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("not_an_int")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("3-")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-,4")); - /// - /// // An empty string is, however, legal, and results in an - /// // empty `ProtoSet`: - /// assert_eq!(Ok(ProtoSet::default()), ProtoSet::from_str("")); - /// # - /// # Ok(protoset) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - fn from_str(version_string: &str) -> Result<Self, Self::Err> { - // If we were passed in an empty string, then return an empty ProtoSet. - if version_string.is_empty() { - return Ok(Self::default()); - } - - let mut pairs: Vec<(Version, Version)> = Vec::new(); - let pieces: ::std::str::Split<char> = version_string.split(','); - - for p in pieces { - let (lo,hi) = if p.contains('-') { - let mut pair = p.splitn(2, '-'); - - let low = pair.next().ok_or(ProtoverError::Unparseable)?; - let high = pair.next().ok_or(ProtoverError::Unparseable)?; - - let lo: Version = low.parse().or(Err(ProtoverError::Unparseable))?; - let hi: Version = high.parse().or(Err(ProtoverError::Unparseable))?; - - (lo,hi) - } else { - let v: u32 = p.parse().or(Err(ProtoverError::Unparseable))?; - - (v, v) - }; - - if lo > MAX_PROTOCOL_VERSION || hi > MAX_PROTOCOL_VERSION { - return Err(ProtoverError::ExceedsMax); - } - pairs.push((lo, hi)); - } - - ProtoSet::from_slice(&pairs[..]) - } -} - -impl ToString for ProtoSet { - /// Contracts a `ProtoSet` of versions into a string. - /// - /// # Returns - /// - /// A `String` representation of this `ProtoSet` in ascending order. - fn to_string(&self) -> String { - let mut final_output: Vec<String> = Vec::new(); - - for &(lo, hi) in self.iter() { - if lo != hi { - debug_assert!(lo < hi); - final_output.push(format!("{}-{}", lo, hi)); - } else { - final_output.push(format!("{}", lo)); - } - } - final_output.join(",") - } -} - -/// Checks to see if there is a continuous range of integers, starting at the -/// first in the list. Returns the last integer in the range if a range exists. -/// -/// # Inputs -/// -/// `list`, an ordered vector of `u32` integers of "[0-9,-]" representing the -/// supported versions for a single protocol. -/// -/// # Returns -/// -/// A `bool` indicating whether the list contains a range, starting at the first -/// in the list, a`Version` of the last integer in the range, and a `usize` of -/// the index of that version. -/// -/// For example, if given vec![1, 2, 3, 5], find_range will return true, -/// as there is a continuous range, and 3, which is the last number in the -/// continuous range, and 2 which is the index of 3. -fn find_range(list: &Vec<Version>) -> (bool, Version, usize) { - if list.len() == 0 { - return (false, 0, 0); - } - - let mut index: usize = 0; - let mut iterable = list.iter().peekable(); - let mut range_end = match iterable.next() { - Some(n) => *n, - None => return (false, 0, 0), - }; - - let mut has_range = false; - - while iterable.peek().is_some() { - let n = *iterable.next().unwrap(); - if n != range_end + 1 { - break; - } - - has_range = true; - range_end = n; - index += 1; - } - - (has_range, range_end, index) -} - -impl From<Vec<Version>> for ProtoSet { - fn from(mut v: Vec<Version>) -> ProtoSet { - let mut version_pairs: Vec<(Version, Version)> = Vec::new(); - - v.sort_unstable(); - v.dedup(); - - 'vector: while !v.is_empty() { - let (has_range, end, index): (bool, Version, usize) = find_range(&v); - - if has_range { - let first: Version = match v.first() { - Some(x) => *x, - None => continue, - }; - let last: Version = match v.get(index) { - Some(x) => *x, - None => continue, - }; - debug_assert!(last == end, format!("last = {}, end = {}", last, end)); - - version_pairs.push((first, last)); - v = v.split_off(index + 1); - - if v.len() == 0 { - break 'vector; - } - } else { - let last: Version = match v.get(index) { - Some(x) => *x, - None => continue, - }; - version_pairs.push((last, last)); - v.remove(index); - } - } - ProtoSet::from_slice(&version_pairs[..]).unwrap_or(ProtoSet::default()) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_find_range() { - assert_eq!((false, 0, 0), find_range(&vec![])); - assert_eq!((false, 1, 0), find_range(&vec![1])); - assert_eq!((true, 2, 1), find_range(&vec![1, 2])); - assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3])); - assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3, 5])); - } - - macro_rules! assert_contains_each { - ($protoset:expr, $versions:expr) => { - for version in $versions { - assert!($protoset.contains(version)); - } - }; - } - - macro_rules! test_protoset_contains_versions { - ($list:expr, $str:expr) => { - let versions: &[Version] = $list; - let protoset: Result<ProtoSet, ProtoverError> = ProtoSet::from_str($str); - - assert!(protoset.is_ok()); - let p = protoset.unwrap(); - assert_contains_each!(p, versions); - }; - } - - #[test] - fn test_versions_from_str() { - test_protoset_contains_versions!(&[], ""); - test_protoset_contains_versions!(&[1], "1"); - test_protoset_contains_versions!(&[1, 2], "1,2"); - test_protoset_contains_versions!(&[1, 2, 3], "1-3"); - test_protoset_contains_versions!(&[1, 2, 5], "1-2,5"); - test_protoset_contains_versions!(&[1, 3, 4, 5], "1,3-5"); - test_protoset_contains_versions!(&[42, 55, 56, 57, 58], "42,55-58"); - } - - #[test] - fn test_versions_from_str_ab() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("a,b")); - } - - #[test] - fn test_versions_from_str_negative_1() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1")); - } - - #[test] - fn test_versions_from_str_commas() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str(",")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,,2")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2,")); - } - - #[test] - fn test_versions_from_str_hyphens() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("--1")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1-2")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1--2")); - } - - #[test] - fn test_versions_from_str_triple() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-2-3")); - } - - #[test] - fn test_versions_from_str_1exclam() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,!")); - } - - #[test] - fn test_versions_from_str_percent_equal() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("%=")); - } - - #[test] - fn test_versions_from_str_whitespace() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2\n")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1\r,2")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,\t2")); - } - - #[test] - fn test_versions_from_str_overlap() { - assert_eq!(Err(ProtoverError::Overlap), ProtoSet::from_str("1-3,2-4")); - } - - #[test] - fn test_versions_from_slice_overlap() { - assert_eq!( - Err(ProtoverError::Overlap), - ProtoSet::from_slice(&[(1, 3), (2, 4)]) - ); - } - - #[test] - fn test_versions_from_str_max() { - assert_eq!( - Err(ProtoverError::ExceedsMax), - ProtoSet::from_str("4294967295") - ); - } - - #[test] - fn test_versions_from_slice_max() { - assert_eq!( - Err(ProtoverError::ExceedsMax), - ProtoSet::from_slice(&[(4294967295, 4294967295)]) - ); - } - - #[test] - fn test_protoset_contains() { - let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 5), (7, 9), (13, 14)]).unwrap(); - - for x in 1..6 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - for x in 7..10 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - for x in 13..15 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - - for x in [6, 10, 11, 12, 15, 42, 43, 44, 45, 1234584].iter() { - assert!(!protoset.contains(&x), format!("should not contain {}", x)); - } - } - - #[test] - fn test_protoset_contains_1_3() { - let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 3)]).unwrap(); - - for x in 1..4 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - } - - macro_rules! assert_protoset_from_vec_contains_all { - ($($x:expr),*) => ( - let vec: Vec<Version> = vec!($($x),*); - let protoset: ProtoSet = vec.clone().into(); - - for x in vec.iter() { - assert!(protoset.contains(&x)); - } - ) - } - - #[test] - fn test_protoset_from_vec_123() { - assert_protoset_from_vec_contains_all!(1, 2, 3); - } - - #[test] - fn test_protoset_from_vec_1_315() { - assert_protoset_from_vec_contains_all!(1, 2, 3, 15); - } - - #[test] - fn test_protoset_from_vec_unordered() { - let v: Vec<Version> = vec![2, 3, 8, 4, 3, 9, 7, 2]; - let ps: ProtoSet = v.into(); - - assert_eq!(ps.to_string(), "2-4,7-9"); - } - - #[test] - fn test_protoset_into_vec() { - let ps: ProtoSet = "1-13,42".parse().unwrap(); - let v: Vec<Version> = ps.into(); - - assert!(v.contains(&7)); - assert!(v.contains(&42)); - } -} - -#[cfg(all(test, feature = "bench"))] -mod bench { - use super::*; -} diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs deleted file mode 100644 index da87509ffa..0000000000 --- a/src/rust/protover/protover.rs +++ /dev/null @@ -1,984 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -use std::collections::hash_map; -use std::collections::HashMap; -use std::ffi::CStr; -use std::fmt; -use std::str; -use std::str::FromStr; -use std::string::String; - -use external::c_tor_version_as_new_as; - -use errors::ProtoverError; -use protoset::ProtoSet; -use protoset::Version; - -/// The first version of Tor that included "proto" entries in its descriptors. -/// Authorities should use this to decide whether to guess proto lines. -/// -/// C_RUST_COUPLED: -/// protover.h `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS` -const FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS: &'static str = "0.2.9.3-alpha"; - -/// The maximum number of subprotocol version numbers we will attempt to expand -/// before concluding that someone is trying to DoS us -/// -/// C_RUST_COUPLED: protover.c `MAX_PROTOCOLS_TO_EXPAND` -const MAX_PROTOCOLS_TO_EXPAND: usize = 1 << 16; - -/// The maximum size an `UnknownProtocol`'s name may be. -pub(crate) const MAX_PROTOCOL_NAME_LENGTH: usize = 100; - -/// Known subprotocols in Tor. Indicates which subprotocol a relay supports. -/// -/// C_RUST_COUPLED: protover.h `protocol_type_t` -#[derive(Clone, Hash, Eq, PartialEq, Debug)] -pub enum Protocol { - Cons, - Desc, - DirCache, - HSDir, - HSIntro, - HSRend, - Link, - LinkAuth, - Microdesc, - Relay, - Padding, - FlowCtrl, -} - -impl fmt::Display for Protocol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -/// Translates a string representation of a protocol into a Proto type. -/// Error if the string is an unrecognized protocol name. -/// -/// C_RUST_COUPLED: protover.c `PROTOCOL_NAMES` -impl FromStr for Protocol { - type Err = ProtoverError; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "Cons" => Ok(Protocol::Cons), - "Desc" => Ok(Protocol::Desc), - "DirCache" => Ok(Protocol::DirCache), - "HSDir" => Ok(Protocol::HSDir), - "HSIntro" => Ok(Protocol::HSIntro), - "HSRend" => Ok(Protocol::HSRend), - "Link" => Ok(Protocol::Link), - "LinkAuth" => Ok(Protocol::LinkAuth), - "Microdesc" => Ok(Protocol::Microdesc), - "Relay" => Ok(Protocol::Relay), - "Padding" => Ok(Protocol::Padding), - "FlowCtrl" => Ok(Protocol::FlowCtrl), - _ => Err(ProtoverError::UnknownProtocol), - } - } -} - -/// A protocol string which is not one of the `Protocols` we currently know -/// about. -#[derive(Clone, Debug, Hash, Eq, PartialEq)] -pub struct UnknownProtocol(String); - -impl fmt::Display for UnknownProtocol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -fn is_valid_proto(s: &str) -> bool { - s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') -} - -impl FromStr for UnknownProtocol { - type Err = ProtoverError; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - if !is_valid_proto(s) { - Err(ProtoverError::InvalidProtocol) - } else if s.len() <= MAX_PROTOCOL_NAME_LENGTH { - Ok(UnknownProtocol(s.to_string())) - } else { - Err(ProtoverError::ExceedsNameLimit) - } - } -} - -impl UnknownProtocol { - /// Create an `UnknownProtocol`, ignoring whether or not it - /// exceeds MAX_PROTOCOL_NAME_LENGTH. - fn from_str_any_len(s: &str) -> Result<Self, ProtoverError> { - if !is_valid_proto(s) { - return Err(ProtoverError::InvalidProtocol); - } - Ok(UnknownProtocol(s.to_string())) - } -} - -impl From<Protocol> for UnknownProtocol { - fn from(p: Protocol) -> UnknownProtocol { - UnknownProtocol(p.to_string()) - } -} - -#[cfg(feature = "test_linking_hack")] -fn have_linkauth_v1() -> bool { - true -} - -#[cfg(not(feature = "test_linking_hack"))] -fn have_linkauth_v1() -> bool { - use external::c_tor_is_using_nss; - !c_tor_is_using_nss() -} - -/// Get a CStr representation of current supported protocols, for -/// passing to C, or for converting to a `&str` for Rust. -/// -/// # Returns -/// -/// An `&'static CStr` whose value is the existing protocols supported by tor. -/// Returned data is in the format as follows: -/// -/// "HSDir=1-1 LinkAuth=1" -/// -/// # Note -/// -/// Rust code can use the `&'static CStr` as a normal `&'a str` by -/// calling `protover::get_supported_protocols`. -/// -// C_RUST_COUPLED: protover.c `protover_get_supported_protocols` -pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { - if !have_linkauth_v1() { - cstr!( - "Cons=1-2 \ - Desc=1-2 \ - DirCache=2 \ - FlowCtrl=1 \ - HSDir=1-2 \ - HSIntro=3-5 \ - HSRend=1-2 \ - Link=1-5 \ - LinkAuth=3 \ - Microdesc=1-2 \ - Padding=2 \ - Relay=1-3" - ) - } else { - cstr!( - "Cons=1-2 \ - Desc=1-2 \ - DirCache=2 \ - FlowCtrl=1 \ - HSDir=1-2 \ - HSIntro=3-5 \ - HSRend=1-2 \ - Link=1-5 \ - LinkAuth=1,3 \ - Microdesc=1-2 \ - Padding=2 \ - Relay=1-3" - ) - } -} - -/// A map of protocol names to the versions of them which are supported. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ProtoEntry(HashMap<Protocol, ProtoSet>); - -impl Default for ProtoEntry { - fn default() -> ProtoEntry { - ProtoEntry(HashMap::new()) - } -} - -impl ProtoEntry { - /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`. - pub fn iter(&self) -> hash_map::Iter<Protocol, ProtoSet> { - self.0.iter() - } - - /// Translate the supported tor versions from a string into a - /// ProtoEntry, which is useful when looking up a specific - /// subprotocol. - pub fn supported() -> Result<Self, ProtoverError> { - let supported_cstr: &'static CStr = get_supported_protocols_cstr(); - let supported: &str = supported_cstr.to_str().unwrap_or(""); - - supported.parse() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn get(&self, protocol: &Protocol) -> Option<&ProtoSet> { - self.0.get(protocol) - } - - pub fn insert(&mut self, key: Protocol, value: ProtoSet) { - self.0.insert(key, value); - } - - pub fn remove(&mut self, key: &Protocol) -> Option<ProtoSet> { - self.0.remove(key) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl FromStr for ProtoEntry { - type Err = ProtoverError; - - /// Parse a string of subprotocol types and their version numbers. - /// - /// # Inputs - /// - /// * A `protocol_entry` string, comprised of a keywords, an "=" sign, and - /// one or more version numbers, each separated by a space. For example, - /// `"Cons=3-4 HSDir=1"`. - /// - /// # Returns - /// - /// A `Result` whose `Ok` value is a `ProtoEntry`. - /// Otherwise, the `Err` value of this `Result` is a `ProtoverError`. - fn from_str(protocol_entry: &str) -> Result<ProtoEntry, ProtoverError> { - let mut proto_entry: ProtoEntry = ProtoEntry::default(); - - if protocol_entry.is_empty() { - return Ok(proto_entry); - } - - let entries = protocol_entry.split(' '); - - for entry in entries { - let mut parts = entry.splitn(2, '='); - - let proto = match parts.next() { - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - - let vers = match parts.next() { - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - let versions: ProtoSet = vers.parse()?; - let proto_name: Protocol = proto.parse()?; - - proto_entry.insert(proto_name, versions); - - if proto_entry.len() > MAX_PROTOCOLS_TO_EXPAND { - return Err(ProtoverError::ExceedsMax); - } - } - Ok(proto_entry) - } -} - -/// Generate an implementation of `ToString` for either a `ProtoEntry` or an -/// `UnvalidatedProtoEntry`. -macro_rules! impl_to_string_for_proto_entry { - ($t:ty) => { - impl ToString for $t { - fn to_string(&self) -> String { - let mut parts: Vec<String> = Vec::new(); - - for (protocol, versions) in self.iter() { - parts.push(format!("{}={}", protocol.to_string(), versions.to_string())); - } - parts.sort_unstable(); - parts.join(" ") - } - } - }; -} - -impl_to_string_for_proto_entry!(ProtoEntry); -impl_to_string_for_proto_entry!(UnvalidatedProtoEntry); - -/// A `ProtoEntry`, but whose `Protocols` can be any `UnknownProtocol`, not just -/// the supported ones enumerated in `Protocols`. The protocol versions are -/// validated, however. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct UnvalidatedProtoEntry(HashMap<UnknownProtocol, ProtoSet>); - -impl Default for UnvalidatedProtoEntry { - fn default() -> UnvalidatedProtoEntry { - UnvalidatedProtoEntry(HashMap::new()) - } -} - -impl UnvalidatedProtoEntry { - /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`. - pub fn iter(&self) -> hash_map::Iter<UnknownProtocol, ProtoSet> { - self.0.iter() - } - - pub fn get(&self, protocol: &UnknownProtocol) -> Option<&ProtoSet> { - self.0.get(protocol) - } - - pub fn insert(&mut self, key: UnknownProtocol, value: ProtoSet) { - self.0.insert(key, value); - } - - pub fn remove(&mut self, key: &UnknownProtocol) -> Option<ProtoSet> { - self.0.remove(key) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - let mut total: usize = 0; - - for (_, versions) in self.iter() { - total += versions.len(); - } - total - } - - /// Determine if we support every protocol a client supports, and if not, - /// determine which protocols we do not have support for. - /// - /// # Returns - /// - /// Optionally, return parameters which the client supports but which we do not. - /// - /// # Examples - /// ``` - /// use protover::UnvalidatedProtoEntry; - /// - /// let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap(); - /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - /// assert_eq!(true, unsupported.is_none()); - /// - /// let protocols: UnvalidatedProtoEntry = "Link=1-2 Wombat=9".parse().unwrap(); - /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - /// assert_eq!(true, unsupported.is_some()); - /// assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); - /// ``` - pub fn all_supported(&self) -> Option<UnvalidatedProtoEntry> { - let mut unsupported: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - let supported: ProtoEntry = match ProtoEntry::supported() { - Ok(x) => x, - Err(_) => return None, - }; - - for (protocol, versions) in self.iter() { - let is_supported: Result<Protocol, ProtoverError> = protocol.0.parse(); - let supported_protocol: Protocol; - - // If the protocol wasn't even in the enum, then we definitely don't - // know about it and don't support any of its versions. - if is_supported.is_err() { - if !versions.is_empty() { - unsupported.insert(protocol.clone(), versions.clone()); - } - continue; - } else { - supported_protocol = is_supported.unwrap(); - } - - let maybe_supported_versions: Option<&ProtoSet> = supported.get(&supported_protocol); - let supported_versions: &ProtoSet; - - // If the protocol wasn't in the map, then we don't know about it - // and don't support any of its versions. Add its versions to the - // map (if it has versions). - if maybe_supported_versions.is_none() { - if !versions.is_empty() { - unsupported.insert(protocol.clone(), versions.clone()); - } - continue; - } else { - supported_versions = maybe_supported_versions.unwrap(); - } - let unsupported_versions = versions.and_not_in(supported_versions); - - if !unsupported_versions.is_empty() { - unsupported.insert(protocol.clone(), unsupported_versions); - } - } - - if unsupported.is_empty() { - return None; - } - Some(unsupported) - } - - /// Determine if we have support for some protocol and version. - /// - /// # Inputs - /// - /// * `proto`, an `UnknownProtocol` to test support for - /// * `vers`, a `Version` which we will go on to determine whether the - /// specified protocol supports. - /// - /// # Return - /// - /// Returns `true` iff this `UnvalidatedProtoEntry` includes support for the - /// indicated protocol and version, and `false` otherwise. - /// - /// # Examples - /// - /// ``` - /// # use std::str::FromStr; - /// use protover::*; - /// # use protover::errors::ProtoverError; - /// - /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> { - /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=1 Doggo=3-5".parse()?; - /// assert_eq!(true, proto.supports_protocol(&Protocol::Cons.into(), &1)); - /// assert_eq!(false, proto.supports_protocol(&Protocol::Cons.into(), &5)); - /// assert_eq!(true, proto.supports_protocol(&UnknownProtocol::from_str("Doggo")?, &4)); - /// # Ok(proto) - /// # } fn main () { do_test(); } - /// ``` - pub fn supports_protocol(&self, proto: &UnknownProtocol, vers: &Version) -> bool { - let supported_versions: &ProtoSet = match self.get(proto) { - Some(n) => n, - None => return false, - }; - supported_versions.contains(&vers) - } - - /// As `UnvalidatedProtoEntry::supports_protocol()`, but also returns `true` - /// if any later version of the protocol is supported. - /// - /// # Examples - /// ``` - /// use protover::*; - /// # use protover::errors::ProtoverError; - /// - /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> { - /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=5".parse()?; - /// - /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &5)); - /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &4)); - /// assert_eq!(false, proto.supports_protocol_or_later(&Protocol::Cons.into(), &6)); - /// # Ok(proto) - /// # } fn main () { do_test(); } - /// ``` - pub fn supports_protocol_or_later(&self, proto: &UnknownProtocol, vers: &Version) -> bool { - let supported_versions: &ProtoSet = match self.get(&proto) { - Some(n) => n, - None => return false, - }; - supported_versions.iter().any(|v| v.1 >= *vers) - } - - /// Split a string containing (potentially) several protocols and their - /// versions into a `Vec` of tuples of string in `(protocol, versions)` - /// form. - /// - /// # Inputs - /// - /// A &str in the form `"Link=3-4 Cons=5"`. - /// - /// # Returns - /// - /// A `Result` whose `Ok` variant is a `Vec<(&str, &str)>` of `(protocol, - /// versions)`, or whose `Err` variant is a `ProtoverError`. - /// - /// # Errors - /// - /// This will error with a `ProtoverError::Unparseable` if any of the - /// following are true: - /// - /// * If a protocol name is an empty string, e.g. `"Cons=1,3 =3-5"`. - /// * If an entry has no equals sign, e.g. `"Cons=1,3 Desc"`. - /// * If there is leading or trailing whitespace, e.g. `" Cons=1,3 Link=3"`. - /// * If there is any other extra whitespice, e.g. `"Cons=1,3 Link=3"`. - fn parse_protocol_and_version_str<'a>( - protocol_string: &'a str, - ) -> Result<Vec<(&'a str, &'a str)>, ProtoverError> { - let mut protovers: Vec<(&str, &str)> = Vec::new(); - - if protocol_string.is_empty() { - return Ok(protovers); - } - - for subproto in protocol_string.split(' ') { - let mut parts = subproto.splitn(2, '='); - - let name = match parts.next() { - Some("") => return Err(ProtoverError::Unparseable), - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - let vers = match parts.next() { - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - protovers.push((name, vers)); - } - Ok(protovers) - } -} - -impl FromStr for UnvalidatedProtoEntry { - type Err = ProtoverError; - - /// Parses a protocol list without validating the protocol names. - /// - /// # Inputs - /// - /// * `protocol_string`, a string comprised of keys and values, both which are - /// strings. The keys are the protocol names while values are a string - /// representation of the supported versions. - /// - /// The input is _not_ expected to be a subset of the Protocol types - /// - /// # Returns - /// - /// A `Result` whose `Ok` value is an `UnvalidatedProtoEntry`. - /// - /// The returned `Result`'s `Err` value is an `ProtoverError`. - /// - /// # Errors - /// - /// This function will error if: - /// - /// * The protocol string does not follow the "protocol_name=version_list" - /// expected format, or - /// * If the version string is malformed. See `impl FromStr for ProtoSet`. - fn from_str(protocol_string: &str) -> Result<UnvalidatedProtoEntry, ProtoverError> { - let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - let parts: Vec<(&str, &str)> = - UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?; - - for &(name, vers) in parts.iter() { - let versions = ProtoSet::from_str(vers)?; - let protocol = UnknownProtocol::from_str(name)?; - - parsed.insert(protocol, versions); - } - Ok(parsed) - } -} - -impl UnvalidatedProtoEntry { - /// Create an `UnknownProtocol`, ignoring whether or not it - /// exceeds MAX_PROTOCOL_NAME_LENGTH. - pub(crate) fn from_str_any_len( - protocol_string: &str, - ) -> Result<UnvalidatedProtoEntry, ProtoverError> { - let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - let parts: Vec<(&str, &str)> = - UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?; - - for &(name, vers) in parts.iter() { - let versions = ProtoSet::from_str(vers)?; - let protocol = UnknownProtocol::from_str_any_len(name)?; - - parsed.insert(protocol, versions); - } - Ok(parsed) - } -} - -/// Pretend a `ProtoEntry` is actually an `UnvalidatedProtoEntry`. -impl From<ProtoEntry> for UnvalidatedProtoEntry { - fn from(proto_entry: ProtoEntry) -> UnvalidatedProtoEntry { - let mut unvalidated: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - - for (protocol, versions) in proto_entry.iter() { - unvalidated.insert(UnknownProtocol::from(protocol.clone()), versions.clone()); - } - unvalidated - } -} - -/// A mapping of protocols to a count of how many times each of their `Version`s -/// were voted for or supported. -/// -/// # Warning -/// -/// The "protocols" are *not* guaranteed to be known/supported `Protocol`s, in -/// order to allow new subprotocols to be introduced even if Directory -/// Authorities don't yet know of them. -pub struct ProtoverVote(HashMap<UnknownProtocol, HashMap<Version, usize>>); - -impl Default for ProtoverVote { - fn default() -> ProtoverVote { - ProtoverVote(HashMap::new()) - } -} - -impl IntoIterator for ProtoverVote { - type Item = (UnknownProtocol, HashMap<Version, usize>); - type IntoIter = hash_map::IntoIter<UnknownProtocol, HashMap<Version, usize>>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl ProtoverVote { - pub fn entry( - &mut self, - key: UnknownProtocol, - ) -> hash_map::Entry<UnknownProtocol, HashMap<Version, usize>> { - self.0.entry(key) - } - - /// Protocol voting implementation. - /// - /// Given a slice of `UnvalidatedProtoEntry`s and a vote `threshold`, return - /// a new `UnvalidatedProtoEntry` encoding all of the protocols that are - /// listed by at least `threshold` of the inputs. - /// - /// # Examples - /// - /// ``` - /// use protover::ProtoverVote; - /// use protover::UnvalidatedProtoEntry; - /// - /// let protos: &[UnvalidatedProtoEntry] = &["Link=3-4".parse().unwrap(), - /// "Link=3".parse().unwrap()]; - /// let vote = ProtoverVote::compute(protos, &2); - /// assert_eq!("Link=3", vote.to_string()); - /// ``` - // C_RUST_COUPLED: protover.c protover_compute_vote - pub fn compute( - proto_entries: &[UnvalidatedProtoEntry], - threshold: &usize, - ) -> UnvalidatedProtoEntry { - let mut all_count: ProtoverVote = ProtoverVote::default(); - let mut final_output: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - - if proto_entries.is_empty() { - return final_output; - } - - // parse and collect all of the protos and their versions and collect them - for vote in proto_entries { - // C_RUST_DIFFERS: This doesn't actually differ, bu this check on - // the total is here to make it match. Because the C version calls - // expand_protocol_list() which checks if there would be too many - // subprotocols *or* individual version numbers, i.e. more than - // MAX_PROTOCOLS_TO_EXPAND, and does this *per vote*, we need to - // match it's behaviour and ensure we're not allowing more than it - // would. - if vote.len() > MAX_PROTOCOLS_TO_EXPAND { - continue; - } - - for (protocol, versions) in vote.iter() { - let supported_vers: &mut HashMap<Version, usize> = - all_count.entry(protocol.clone()).or_insert(HashMap::new()); - - for version in versions.clone().expand() { - let counter: &mut usize = supported_vers.entry(version).or_insert(0); - *counter += 1; - } - } - } - - for (protocol, mut versions) in all_count { - // Go through and remove versions that are less than the threshold - versions.retain(|_, count| *count as usize >= *threshold); - - if versions.len() > 0 { - let voted_versions: Vec<Version> = versions.keys().cloned().collect(); - let voted_protoset: ProtoSet = ProtoSet::from(voted_versions); - - final_output.insert(protocol, voted_protoset); - } - } - final_output - } -} - -/// Returns a boolean indicating whether the given protocol and version is -/// supported in any of the existing Tor protocols -/// -/// # Examples -/// ``` -/// use protover::is_supported_here; -/// use protover::Protocol; -/// -/// let is_supported = is_supported_here(&Protocol::Link, &10); -/// assert_eq!(false, is_supported); -/// -/// let is_supported = is_supported_here(&Protocol::Link, &1); -/// assert_eq!(true, is_supported); -/// ``` -pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool { - let currently_supported: ProtoEntry = match ProtoEntry::supported() { - Ok(result) => result, - Err(_) => return false, - }; - let supported_versions = match currently_supported.get(proto) { - Some(n) => n, - None => return false, - }; - supported_versions.contains(vers) -} - -/// Since older versions of Tor cannot infer their own subprotocols, -/// determine which subprotocols are supported by older Tor versions. -/// -/// # Inputs -/// -/// * `version`, a string comprised of "[0-9a-z.-]" -/// -/// # Returns -/// -/// A `&'static CStr` encoding a list of protocol names and supported -/// versions. The string takes the following format: -/// -/// "HSDir=1-1 LinkAuth=1" -/// -/// This function returns the protocols that are supported by the version input, -/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS` -/// (but not older than 0.2.4.19). For newer tors (or older than 0.2.4.19), it -/// returns an empty string. -/// -/// # Note -/// -/// This function is meant to be called for/within FFI code. If you'd -/// like to use this code in Rust, please see `compute_for_old_tor()`. -// -// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` -pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static CStr { - let empty: &'static CStr = cstr!(""); - - if c_tor_version_as_new_as(version, FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS) { - return empty; - } - if c_tor_version_as_new_as(version, "0.2.9.1-alpha") { - return cstr!( - "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2" - ); - } - if c_tor_version_as_new_as(version, "0.2.7.5") { - return cstr!( - "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2" - ); - } - if c_tor_version_as_new_as(version, "0.2.4.19") { - return cstr!( - "Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2" - ); - } - empty -} - -/// Since older versions of Tor cannot infer their own subprotocols, -/// determine which subprotocols are supported by older Tor versions. -/// -/// # Inputs -/// -/// * `version`, a string comprised of "[0-9a-z.-]" -/// -/// # Returns -/// -/// A `Result` whose `Ok` value is an `&'static str` encoding a list of protocol -/// names and supported versions. The string takes the following format: -/// -/// "HSDir=1-1 LinkAuth=1" -/// -/// This function returns the protocols that are supported by the version input, -/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`. -/// (but not older than 0.2.4.19). For newer tors (or older than 0.2.4.19), its -/// `Ok` `Result` contains an empty string. -/// -/// Otherwise, its `Err` contains a `ProtoverError::Unparseable` if the -/// `version` string was invalid utf-8. -/// -/// # Note -/// -/// This function is meant to be called for/within non-FFI Rust code. -// -// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` -pub fn compute_for_old_tor(version: &str) -> Result<&'static str, ProtoverError> { - // .to_str() fails with a Utf8Error if it couldn't validate the - // utf-8, so convert that here into an Unparseable ProtoverError. - compute_for_old_tor_cstr(version) - .to_str() - .or(Err(ProtoverError::Unparseable)) -} - -#[cfg(test)] -mod test { - use std::str::FromStr; - use std::string::ToString; - - use super::*; - - macro_rules! parse_proto { - ($e:expr) => {{ - let proto: Result<UnknownProtocol, _> = $e.parse(); - let proto2 = UnknownProtocol::from_str_any_len($e); - assert_eq!(proto, proto2); - proto - }}; - } - - #[test] - fn test_protocol_from_str() { - assert!(parse_proto!("Cons").is_ok()); - assert!(parse_proto!("123").is_ok()); - assert!(parse_proto!("1-2-3").is_ok()); - - let err = Err(ProtoverError::InvalidProtocol); - assert_eq!(err, parse_proto!("a_b_c")); - assert_eq!(err, parse_proto!("a b")); - assert_eq!(err, parse_proto!("a,")); - assert_eq!(err, parse_proto!("b.")); - assert_eq!(err, parse_proto!("é")); - } - - macro_rules! assert_protoentry_is_parseable { - ($e:expr) => { - let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse(); - - assert!(protoentry.is_ok(), format!("{:?}", protoentry.err())); - }; - } - - macro_rules! assert_protoentry_is_unparseable { - ($e:expr) => { - let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse(); - - assert!(protoentry.is_err()); - }; - } - - #[test] - fn test_protoentry_from_str_multiple_protocols_multiple_versions() { - assert_protoentry_is_parseable!("Cons=3-4 Link=1,3-5"); - } - - #[test] - fn test_protoentry_from_str_empty() { - assert_protoentry_is_parseable!(""); - assert!(UnvalidatedProtoEntry::from_str("").is_ok()); - } - - #[test] - fn test_protoentry_from_str_single_protocol_single_version() { - assert_protoentry_is_parseable!("HSDir=1"); - } - - #[test] - fn test_protoentry_from_str_unknown_protocol() { - assert_protoentry_is_unparseable!("Ducks=5-7,8"); - } - - #[test] - fn test_protoentry_from_str_allowed_number_of_versions() { - assert_protoentry_is_parseable!("Desc=1-63"); - } - - #[test] - fn test_protoentry_from_str_too_many_versions() { - assert_protoentry_is_unparseable!("Desc=1-64"); - } - - #[test] - fn test_protoentry_all_supported_single_protocol_single_version() { - let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); - } - - #[test] - fn test_protoentry_all_supported_multiple_protocol_multiple_versions() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); - } - - #[test] - fn test_protoentry_all_supported_three_values() { - let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); - } - - #[test] - fn test_protoentry_all_supported_unknown_protocol() { - let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); - } - - #[test] - fn test_protoentry_all_supported_unsupported_high_version() { - let protocols: UnvalidatedProtoEntry = "HSDir=12-60".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("HSDir=12-60", &unsupported.unwrap().to_string()); - } - - #[test] - fn test_protoentry_all_supported_unsupported_low_version() { - let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("HSIntro=2", &unsupported.unwrap().to_string()); - } - - #[test] - fn test_contract_protocol_list() { - let mut versions = ""; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1-2"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1,3"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1-4"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1,3,5-7"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1-3,50"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - } -} diff --git a/src/rust/protover/tests/protover.rs b/src/rust/protover/tests/protover.rs deleted file mode 100644 index a6305ac39a..0000000000 --- a/src/rust/protover/tests/protover.rs +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -extern crate protover; - -use protover::errors::ProtoverError; -use protover::ProtoEntry; -use protover::ProtoverVote; -use protover::UnvalidatedProtoEntry; - -#[test] -fn parse_protocol_with_single_proto_and_single_version() { - let _: ProtoEntry = "Cons=1".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_multiple_versions() { - let _: ProtoEntry = "Cons=1-2".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_different_single_protocol_and_single_version() { - let _: ProtoEntry = "HSDir=1".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_supported_version() { - let _: ProtoEntry = "Desc=2".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_two_protocols_and_single_version() { - let _: ProtoEntry = "Cons=1 HSDir=1".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_two_sequential_versions() { - let _: ProtoEntry = "Desc=1-2".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_protocol_range() { - let _: ProtoEntry = "Link=1-4".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_protocol_set() { - let _: ProtoEntry = "Link=3-4 Desc=2".parse().unwrap(); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_protocol_set() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_two_values() { - let protocols: UnvalidatedProtoEntry = "Microdesc=1-2 Relay=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_one_value() { - let protocols: UnvalidatedProtoEntry = "Microdesc=1-2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_three_values() { - let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_unsupported_protocol() { - let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_unsupported_versions() { - let protocols: UnvalidatedProtoEntry = "Link=3-63".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Link=6-63", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_unsupported_low_version() { - let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("HSIntro=2", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_unsupported_high_version() { - let protocols: UnvalidatedProtoEntry = "Cons=1-2,60".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Cons=60", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_mix_of_supported_and_unsupproted() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Wombat=9".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); -} - -#[test] -fn protover_string_supports_protocol_returns_true_for_single_supported() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap(); - let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &1); - assert_eq!(true, is_supported); -} - -#[test] -fn protover_string_supports_protocol_returns_false_for_single_unsupported() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap(); - let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2); - assert_eq!(false, is_supported); -} - -#[test] -fn protover_string_supports_protocol_returns_false_for_unsupported() { - let protocols: UnvalidatedProtoEntry = "Link=3-4".parse().unwrap(); - let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2); - assert_eq!(false, is_supported); -} - -#[test] -#[should_panic] -fn parse_protocol_with_unexpected_characters() { - let _: UnvalidatedProtoEntry = "Cons=*-%".parse().unwrap(); -} - -#[test] -fn protover_compute_vote_returns_empty_for_empty_string() { - let protocols: &[UnvalidatedProtoEntry] = &["".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_single_protocol_for_matching() { - let protocols: &[UnvalidatedProtoEntry] = &["Cons=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_two_protocols_for_two_matching() { - let protocols: &[UnvalidatedProtoEntry] = &["Link=1 Cons=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1 Link=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_one_protocol_when_one_out_of_two_matches() { - let protocols: &[UnvalidatedProtoEntry] = - &["Cons=1 Link=2".parse().unwrap(), "Cons=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &2); - assert_eq!("Cons=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_protocols_that_it_doesnt_currently_support() { - let protocols: &[UnvalidatedProtoEntry] = - &["Foo=1 Cons=2".parse().unwrap(), "Bar=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Bar=1 Cons=2 Foo=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_matching_for_mix() { - let protocols: &[UnvalidatedProtoEntry] = &["Link=1-10,50 Cons=1,3-7,8".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1,3-8 Link=1-10,50", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_matching_for_longer_mix() { - let protocols: &[UnvalidatedProtoEntry] = &[ - "Desc=1-10,50 Cons=1,3-7,8".parse().unwrap(), - "Link=12-45,8 Cons=2-6,8 Desc=9".parse().unwrap(), - ]; - - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1-8 Desc=1-10,50 Link=8,12-45", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_matching_for_longer_mix_with_threshold_two() { - let protocols: &[UnvalidatedProtoEntry] = &[ - "Desc=1-10,50 Cons=1,3-7,8".parse().unwrap(), - "Link=8,12-45 Cons=2-6,8 Desc=9".parse().unwrap(), - ]; - - let listed = ProtoverVote::compute(protocols, &2); - assert_eq!("Cons=3-6,8 Desc=9", listed.to_string()); -} - -#[test] -fn protover_compute_vote_handles_duplicated_versions() { - let protocols: &[UnvalidatedProtoEntry] = - &["Cons=1".parse().unwrap(), "Cons=1".parse().unwrap()]; - assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string()); - - let protocols: &[UnvalidatedProtoEntry] = - &["Cons=1-2".parse().unwrap(), "Cons=1-2".parse().unwrap()]; - assert_eq!("Cons=1-2", ProtoverVote::compute(protocols, &2).to_string()); -} - -#[test] -fn protover_compute_vote_handles_invalid_proto_entries() { - let protocols: &[UnvalidatedProtoEntry] = &[ - "Cons=1".parse().unwrap(), - "Cons=1".parse().unwrap(), - "Dinosaur=1".parse().unwrap(), - ]; - assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string()); -} - -#[test] -fn parse_protocol_with_single_protocol_and_two_nonsequential_versions() { - let _: ProtoEntry = "Desc=1,2".parse().unwrap(); -} - -#[test] -fn protover_is_supported_here_returns_true_for_supported_protocol() { - assert_eq!( - true, - protover::is_supported_here(&protover::Protocol::Cons, &1) - ); -} - -#[test] -fn protover_is_supported_here_returns_false_for_unsupported_protocol() { - assert_eq!( - false, - protover::is_supported_here(&protover::Protocol::Cons, &5) - ); -} - -#[test] -fn protocol_all_supported_with_single_proto_and_single_version() { - let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_multiple_versions() { - let protocol: UnvalidatedProtoEntry = "Cons=1-2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_different_single_protocol_and_single_version() { - let protocol: UnvalidatedProtoEntry = "HSDir=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_supported_version() { - let protocol: UnvalidatedProtoEntry = "Desc=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_two_protocols_and_single_version() { - let protocols: UnvalidatedProtoEntry = "Cons=1 HSDir=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_two_nonsequential_versions() { - let protocol: UnvalidatedProtoEntry = "Desc=1,2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_two_sequential_versions() { - let protocol: UnvalidatedProtoEntry = "Desc=1-2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_protocol_range() { - let protocol: UnvalidatedProtoEntry = "Link=1-4".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protover_all_supported_should_exclude_versions_we_actually_do_support() { - let proto: UnvalidatedProtoEntry = "Link=3-63".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=6-63".to_string()); -} - -#[test] -fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex1() { - let proto: UnvalidatedProtoEntry = "Link=1-3,30-63".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=30-63".to_string()); -} - -#[test] -fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex2() { - let proto: UnvalidatedProtoEntry = "Link=1-3,5-12".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=6-12".to_string()); -} - -#[test] -fn protover_all_supported_should_exclude_some_versions_and_entire_protocols() { - let proto: UnvalidatedProtoEntry = "Link=1-3,5-12 Quokka=50-51".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=6-12 Quokka=50-51".to_string()); -} - -#[test] -// C_RUST_DIFFERS: The C will return true (e.g. saying "yes, that's supported") -// but set the msg to NULL (??? seems maybe potentially bad). The Rust will -// simply return a None. -fn protover_all_supported_should_return_empty_string_for_weird_thing() { - let proto: UnvalidatedProtoEntry = "Fribble=".parse().unwrap(); - let result: Option<UnvalidatedProtoEntry> = proto.all_supported(); - - assert!(result.is_none()); -} - -#[test] -fn protover_unvalidatedprotoentry_should_err_entirely_unparseable_things() { - let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Fribble".parse(); - - assert_eq!(Err(ProtoverError::Unparseable), proto); -} - -#[test] -fn protover_all_supported_over_maximum_limit() { - let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Sleen=1-4294967295".parse(); - - assert_eq!(Err(ProtoverError::ExceedsMax), proto); -} diff --git a/src/rust/smartlist/Cargo.toml b/src/rust/smartlist/Cargo.toml deleted file mode 100644 index a5afe7bf74..0000000000 --- a/src/rust/smartlist/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "smartlist" - -[dependencies] -libc = "0.2.39" - -[lib] -name = "smartlist" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/smartlist/lib.rs b/src/rust/smartlist/lib.rs deleted file mode 100644 index 23301f88c3..0000000000 --- a/src/rust/smartlist/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -extern crate libc; - -mod smartlist; - -pub use smartlist::*; - -// When testing we may be compiled with sanitizers which are incompatible with -// Rust's default allocator, jemalloc (unsure why at this time). Most crates -// link to `tor_allocate` which switches by default to a non-jemalloc allocator, -// but we don't already depend on `tor_allocate` so make sure that while testing -// we don't use jemalloc. (but rather malloc/free) -#[global_allocator] -#[cfg(test)] -static A: std::alloc::System = std::alloc::System; diff --git a/src/rust/smartlist/smartlist.rs b/src/rust/smartlist/smartlist.rs deleted file mode 100644 index d8f8083dff..0000000000 --- a/src/rust/smartlist/smartlist.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -use libc::{c_char, c_int}; -use std::ffi::CStr; -use std::slice; - -/// Smartlists are a type used in C code in tor to define a collection of a -/// generic type, which has a capacity and a number used. Each Smartlist -/// defines how to extract the list of values from the underlying C structure -/// -/// Implementations are required to have a C representation, as this module -/// serves purely to translate smartlists as defined in tor to vectors in Rust. -pub trait Smartlist<T> { - fn get_list(&self) -> Vec<T>; -} - -#[repr(C)] -pub struct Stringlist { - pub list: *const *const c_char, - pub num_used: c_int, - pub capacity: c_int, -} - -impl Smartlist<String> for Stringlist { - fn get_list(&self) -> Vec<String> { - let empty: Vec<String> = Vec::new(); - let mut rust_list: Vec<String> = Vec::new(); - - if self.list.is_null() || self.num_used == 0 { - return empty; - } - - // unsafe, as we need to extract the smartlist list into a vector of - // pointers, and then transform each element into a Rust string. - let elems: &[*const c_char] = - unsafe { slice::from_raw_parts(self.list, self.num_used as usize) }; - - for elem in elems.iter() { - if elem.is_null() { - continue; - } - - // unsafe, as we need to create a cstring from the referenced - // element - let c_string = unsafe { CStr::from_ptr(*elem) }; - - let r_string = match c_string.to_str() { - Ok(n) => n, - Err(_) => return empty, - }; - - rust_list.push(String::from(r_string)); - } - - rust_list - } -} - -// TODO: CHK: this module maybe should be tested from a test in C with a -// smartlist as defined in tor. -#[cfg(test)] -mod test { - #[test] - fn test_get_list_of_strings() { - extern crate libc; - - use libc::c_char; - use std::ffi::CString; - - use super::Smartlist; - use super::Stringlist; - - { - // test to verify that null pointers are gracefully handled - use std::ptr; - - let sl = Stringlist { - list: ptr::null(), - num_used: 0, - capacity: 0, - }; - - let data = sl.get_list(); - assert_eq!(0, data.len()); - } - - { - let args = vec![String::from("a"), String::from("b")]; - - // for each string, transform it into a CString - let c_strings: Vec<_> = args - .iter() - .map(|arg| CString::new(arg.as_str()).unwrap()) - .collect(); - - // then, collect a pointer for each CString - let p_args: Vec<_> = c_strings.iter().map(|arg| arg.as_ptr()).collect(); - - let p: *const *const c_char = p_args.as_ptr(); - - // This is the representation that we expect when receiving a - // smartlist at the Rust/C FFI layer. - let sl = Stringlist { - list: p, - num_used: 2, - capacity: 2, - }; - - let data = sl.get_list(); - assert_eq!("a", &data[0]); - assert_eq!("b", &data[1]); - } - } -} diff --git a/src/rust/tor_allocate/Cargo.toml b/src/rust/tor_allocate/Cargo.toml deleted file mode 100644 index 06ac605f17..0000000000 --- a/src/rust/tor_allocate/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "tor_allocate" - -[dependencies] -libc = "=0.2.39" - -[lib] -name = "tor_allocate" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/tor_allocate/lib.rs b/src/rust/tor_allocate/lib.rs deleted file mode 100644 index fff8a08006..0000000000 --- a/src/rust/tor_allocate/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! Allocation helper functions that allow data to be allocated in Rust -//! using tor's specified allocator. In doing so, this can be later freed -//! from C. -//! -//! This is currently a temporary solution, we will later use tor's allocator -//! by default for any allocation that occurs in Rust. However, as this will -//! stabalize in 2018, we can use this as a temporary measure. - -extern crate libc; - -use std::alloc::System; - -mod tor_allocate; -pub use tor_allocate::*; - -#[global_allocator] -static A: System = System; diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs deleted file mode 100644 index 7b35e2451f..0000000000 --- a/src/rust/tor_allocate/tor_allocate.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ -// No-op defined purely for testing at the module level -use libc::c_char; - -use libc::c_void; -#[cfg(not(feature = "testing"))] -use std::{mem, ptr, slice}; - -// Define a no-op implementation for testing Rust modules without linking to C -#[cfg(feature = "testing")] -pub fn allocate_and_copy_string(s: &str) -> *mut c_char { - use std::ffi::CString; - CString::new(s).unwrap().into_raw() -} - -// Defined only for tests, used for testing purposes, so that we don't need -// to link to tor C files. Uses the system allocator -#[cfg(test)] -unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void { - use libc::malloc; - malloc(size) -} - -#[cfg(all(not(test), not(feature = "testing")))] -extern "C" { - fn tor_malloc_(size: usize) -> *mut c_void; -} - -/// Allocate memory using tor_malloc_ and copy an existing string into the -/// allocated buffer, returning a pointer that can later be called in C. -/// -/// # Inputs -/// -/// * `src`, a reference to a String. -/// -/// # Returns -/// -/// A `*mut c_char` that should be freed by tor_free in C -/// -#[cfg(not(feature = "testing"))] -pub fn allocate_and_copy_string(src: &str) -> *mut c_char { - let bytes: &[u8] = src.as_bytes(); - - let size = mem::size_of_val::<[u8]>(bytes); - let size_one_byte = mem::size_of::<u8>(); - - // handle integer overflow when adding one to the calculated length - let size_with_null_byte = match size.checked_add(size_one_byte) { - Some(n) => n, - None => return ptr::null_mut(), - }; - - let dest = unsafe { tor_malloc_(size_with_null_byte) as *mut u8 }; - - if dest.is_null() { - return ptr::null_mut(); - } - - unsafe { ptr::copy_nonoverlapping(bytes.as_ptr(), dest, size) }; - - // set the last byte as null, using the ability to index into a slice - // rather than doing pointer arithmetic - let slice = unsafe { slice::from_raw_parts_mut(dest, size_with_null_byte) }; - slice[size] = 0; // add a null terminator - - dest as *mut c_char -} - -#[cfg(test)] -mod test { - - #[test] - fn test_allocate_and_copy_string_with_empty() { - use libc::{c_void, free}; - use std::ffi::CStr; - - use tor_allocate::allocate_and_copy_string; - - let allocated_empty = allocate_and_copy_string(""); - - let allocated_empty_rust = unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() }; - - assert_eq!("", allocated_empty_rust); - - unsafe { free(allocated_empty as *mut c_void) }; - } - - #[test] - fn test_allocate_and_copy_string_with_not_empty_string() { - use libc::{c_void, free}; - use std::ffi::CStr; - - use tor_allocate::allocate_and_copy_string; - - let allocated_empty = allocate_and_copy_string("foo bar biz"); - - let allocated_empty_rust = unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() }; - - assert_eq!("foo bar biz", allocated_empty_rust); - - unsafe { free(allocated_empty as *mut c_void) }; - } -} diff --git a/src/rust/tor_log/Cargo.toml b/src/rust/tor_log/Cargo.toml deleted file mode 100644 index 14d9ae803a..0000000000 --- a/src/rust/tor_log/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "tor_log" -version = "0.1.0" -authors = ["The Tor Project"] - -[lib] -name = "tor_log" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] - -[dependencies] -libc = "0.2.39" - -[dependencies.tor_allocate] -path = "../tor_allocate" diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs deleted file mode 100644 index 4aa658e35b..0000000000 --- a/src/rust/tor_log/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Copyright (c) 2016-2019, The Tor Project, Inc. */ -//! See LICENSE for licensing information */ - -//! Logging wrapper for Rust to utilize Tor's logger, found at -//! src/common/log.c and src/common/torlog.h -//! -//! Exposes different interfaces depending on whether we are running in test -//! or non-test mode. When testing, we use a no-op implementation, -//! otherwise we link directly to C. - -extern crate libc; -extern crate tor_allocate; - -mod tor_log; - -pub use tor_log::*; diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs deleted file mode 100644 index 391cb32ab3..0000000000 --- a/src/rust/tor_log/tor_log.rs +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -// Note that these functions are untested due to the fact that there are no -// return variables to test and they are calling into a C API. - -/// The related domain which the logging message is relevant. For example, -/// log messages relevant to networking would use LogDomain::LdNet, whereas -/// general messages can use LdGeneral. -#[derive(Eq, PartialEq)] -pub enum LogDomain { - Net, - General, -} - -/// The severity level at which to log messages. -#[derive(Eq, PartialEq)] -pub enum LogSeverity { - Notice, - Warn, -} - -/// Main entry point for Rust modules to log messages. -/// -/// # Inputs -/// -/// * A `severity` of type LogSeverity, which defines the level of severity the -/// message will be logged. -/// * A `domain` of type LogDomain, which defines the domain the log message -/// will be associated with. -/// * A `function` of type &str, which defines the name of the function where -/// the message is being logged. There is a current RFC for a macro that -/// defines function names. When it is, we should use it. See -/// https://github.com/rust-lang/rfcs/pull/1719 -/// * A `message` of type &str, which is the log message itself. -#[macro_export] -macro_rules! tor_log_msg { - ($severity: path, - $domain: path, - $function: expr, - $($message:tt)*) => - { - { - let msg = format!($($message)*); - $crate::tor_log_msg_impl($severity, $domain, $function, msg) - } - }; -} - -#[inline] -pub fn tor_log_msg_impl(severity: LogSeverity, domain: LogDomain, function: &str, message: String) { - use std::ffi::CString; - - /// Default function name to log in case of errors when converting - /// a function name to a CString - const ERR_LOG_FUNCTION: &str = "tor_log_msg"; - - /// Default message to log in case of errors when converting a log - /// message to a CString - const ERR_LOG_MSG: &str = "Unable to log message from Rust \ - module due to error when converting to CString"; - - let func = match CString::new(function) { - Ok(n) => n, - Err(_) => CString::new(ERR_LOG_FUNCTION).unwrap(), - }; - - let msg = match CString::new(message) { - Ok(n) => n, - Err(_) => CString::new(ERR_LOG_MSG).unwrap(), - }; - - // Bind to a local variable to preserve ownership. This is essential so - // that ownership is guaranteed until these local variables go out of scope - let func_ptr = func.as_ptr(); - let msg_ptr = msg.as_ptr(); - - let c_severity = unsafe { log::translate_severity(severity) }; - let c_domain = unsafe { log::translate_domain(domain) }; - - unsafe { log::tor_log_string(c_severity, c_domain, func_ptr, msg_ptr) } -} - -/// This implementation is used when compiling for actual use, as opposed to -/// testing. -#[cfg(not(test))] -pub mod log { - use super::LogDomain; - use super::LogSeverity; - use libc::{c_char, c_int}; - - /// Severity log types. These mirror definitions in src/lib/log/log.h - /// C_RUST_COUPLED: src/lib/log/log.c, log domain types - extern "C" { - static LOG_WARN_: c_int; - static LOG_NOTICE_: c_int; - } - - /// Domain log types. These mirror definitions in src/lib/log/log.h - /// C_RUST_COUPLED: src/lib/log/log.c, log severity types - extern "C" { - static LD_NET_: u64; - static LD_GENERAL_: u64; - } - - /// Translate Rust definitions of log domain levels to C. This exposes a 1:1 - /// mapping between types. - #[inline] - pub unsafe fn translate_domain(domain: LogDomain) -> u64 { - match domain { - LogDomain::Net => LD_NET_, - LogDomain::General => LD_GENERAL_, - } - } - - /// Translate Rust definitions of log severity levels to C. This exposes a - /// 1:1 mapping between types. - #[inline] - pub unsafe fn translate_severity(severity: LogSeverity) -> c_int { - match severity { - LogSeverity::Warn => LOG_WARN_, - LogSeverity::Notice => LOG_NOTICE_, - } - } - - /// The main entry point into Tor's logger. When in non-test mode, this - /// will link directly with `tor_log_string` in torlog.c - extern "C" { - pub fn tor_log_string( - severity: c_int, - domain: u64, - function: *const c_char, - string: *const c_char, - ); - } -} - -/// This module exposes no-op functionality for testing other Rust modules -/// without linking to C. -#[cfg(test)] -pub mod log { - use super::LogDomain; - use super::LogSeverity; - use libc::{c_char, c_int}; - - pub static mut LAST_LOGGED_FUNCTION: *mut String = 0 as *mut String; - pub static mut LAST_LOGGED_MESSAGE: *mut String = 0 as *mut String; - - pub unsafe fn tor_log_string( - _severity: c_int, - _domain: u32, - function: *const c_char, - message: *const c_char, - ) { - use std::ffi::CStr; - - let f = CStr::from_ptr(function); - let fct = match f.to_str() { - Ok(n) => n, - Err(_) => "", - }; - LAST_LOGGED_FUNCTION = Box::into_raw(Box::new(String::from(fct))); - - let m = CStr::from_ptr(message); - let msg = match m.to_str() { - Ok(n) => n, - Err(_) => "", - }; - LAST_LOGGED_MESSAGE = Box::into_raw(Box::new(String::from(msg))); - } - - pub unsafe fn translate_domain(_domain: LogDomain) -> u32 { - 1 - } - - pub unsafe fn translate_severity(_severity: LogSeverity) -> c_int { - 1 - } -} - -#[cfg(test)] -mod test { - use tor_log::log::{LAST_LOGGED_FUNCTION, LAST_LOGGED_MESSAGE}; - use tor_log::*; - - #[test] - fn test_get_log_message() { - { - fn test_macro() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::Net, - "test_macro", - "test log message {}", - "a", - ); - } - - test_macro(); - - let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; - assert_eq!("test_macro", *function); - - let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; - assert_eq!("test log message a", *message); - } - - // test multiple inputs into the log message - { - fn test_macro() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::Net, - "next_test_macro", - "test log message {} {} {} {} {}", - 1, - 2, - 3, - 4, - 5 - ); - } - - test_macro(); - - let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; - assert_eq!("next_test_macro", *function); - - let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; - assert_eq!("test log message 1 2 3 4 5", *message); - } - - // test how a long log message will be formatted - { - fn test_macro() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::Net, - "test_macro", - "{}", - "All the world's a stage, and all the men and women \ - merely players: they have their exits and their \ - entrances; and one man in his time plays many parts, his \ - acts being seven ages." - ); - } - - test_macro(); - - let expected_string = "All the world's a \ - stage, and all the men \ - and women merely players: \ - they have their exits and \ - their entrances; and one man \ - in his time plays many parts, \ - his acts being seven ages."; - - let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; - assert_eq!("test_macro", *function); - - let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; - assert_eq!(expected_string, *message); - } - } -} diff --git a/src/rust/tor_rust/Cargo.toml b/src/rust/tor_rust/Cargo.toml deleted file mode 100644 index 35c629882e..0000000000 --- a/src/rust/tor_rust/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -authors = ["The Tor Project"] -name = "tor_rust" -version = "0.1.0" - -[lib] -name = "tor_rust" -path = "lib.rs" -crate_type = ["staticlib"] - -[dependencies.tor_util] -path = "../tor_util" - -[dependencies.protover] -path = "../protover" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/tor_rust/include.am b/src/rust/tor_rust/include.am deleted file mode 100644 index ce673abbee..0000000000 --- a/src/rust/tor_rust/include.am +++ /dev/null @@ -1,28 +0,0 @@ -EXTRA_DIST +=\ - src/rust/tor_rust/Cargo.toml \ - src/rust/tor_rust/lib.rs - -EXTRA_CARGO_OPTIONS= - -@TOR_RUST_LIB_PATH@: FORCE - ( cd "$(abs_top_builddir)/src/rust" ; \ - CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \ - $(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \ - $(CARGO_ONLINE) \ - --manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" ) - -distclean-rust: - ( cd "$(abs_top_builddir)/src/rust" ; \ - CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \ - $(CARGO) clean $(EXTRA_CARGO_OPTIONS) \ - $(CARGO_ONLINE) \ - --manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" ) - rm -rf "$(abs_top_builddir)/src/rust/registry" - -if USE_RUST -build-rust: @TOR_RUST_LIB_PATH@ -else -build-rust: -endif - -FORCE: diff --git a/src/rust/tor_rust/lib.rs b/src/rust/tor_rust/lib.rs deleted file mode 100644 index 18519f8497..0000000000 --- a/src/rust/tor_rust/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -extern crate protover; -extern crate tor_util; - -pub use protover::*; -pub use tor_util::*; diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml deleted file mode 100644 index 9ffaeda8a6..0000000000 --- a/src/rust/tor_util/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -authors = ["The Tor Project"] -name = "tor_util" -version = "0.0.1" - -[lib] -name = "tor_util" -path = "lib.rs" - -[dependencies.tor_allocate] -path = "../tor_allocate" - -[dependencies.tor_log] -path = "../tor_log" - -[dependencies] -libc = "=0.2.39" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs deleted file mode 100644 index b71b2bd093..0000000000 --- a/src/rust/tor_util/ffi.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! FFI functions to announce Rust support during tor startup, only to be -//! called from C. -//! - -use tor_log::{LogDomain, LogSeverity}; - -/// Returns a short string to announce Rust support during startup. -/// -/// # Examples -/// ```c -/// char *rust_str = rust_welcome_string(); -/// printf("%s", rust_str); -/// tor_free(rust_str); -/// ``` -#[no_mangle] -pub extern "C" fn rust_log_welcome_string() { - tor_log_msg!( - LogSeverity::Notice, - LogDomain::General, - "rust_log_welcome_string", - "Tor is running with Rust integration. Please report \ - any bugs you encounter." - ); -} diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs deleted file mode 100644 index 8886767ede..0000000000 --- a/src/rust/tor_util/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! Small module to announce Rust support during startup for demonstration -//! purposes. - -extern crate libc; -extern crate tor_allocate; - -#[macro_use] -extern crate tor_log; - -pub mod ffi; -pub mod strings; diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs deleted file mode 100644 index ede42c6ea8..0000000000 --- a/src/rust/tor_util/strings.rs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! Utilities for working with static strings. - -/// Create a `CStr` from a literal byte slice, appending a NUL byte to it first. -/// -/// # Warning -/// -/// The literal byte slice which is taken as an argument *MUST NOT* have any NUL -/// bytes (`b"\0"`) in it, anywhere, or else an empty string will be returned -/// (`CStr::from_bytes_with_nul_unchecked(b"\0")`) so as to avoid `panic!()`ing. -/// -/// # Examples -/// -/// ``` -/// #[macro_use] -/// extern crate tor_util; -/// -/// use std::ffi::CStr; -/// -/// # fn do_test() -> Result<&'static CStr, &'static str> { -/// let message: &'static str = "This is a test of the tsunami warning system."; -/// let tuesday: &'static CStr; -/// let original: &str; -/// -/// tuesday = cstr!("This is a test of the tsunami warning system."); -/// original = tuesday.to_str().or(Err("Couldn't unwrap CStr!"))?; -/// -/// assert!(original == message); -/// # -/// # Ok(tuesday) -/// # } -/// # fn main() { -/// # do_test(); // so that we can use the ? operator in the test -/// # } -/// ``` -/// It is also possible to pass several string literals to this macro. They -/// will be concatenated together in the order of the arguments, unmodified, -/// before finally being suffixed with a NUL byte: -/// -/// ``` -/// #[macro_use] -/// extern crate tor_util; -/// # -/// # use std::ffi::CStr; -/// # -/// # fn do_test() -> Result<&'static CStr, &'static str> { -/// -/// let quux: &'static CStr = cstr!("foo", "bar", "baz"); -/// let orig: &'static str = quux.to_str().or(Err("Couldn't unwrap CStr!"))?; -/// -/// assert!(orig == "foobarbaz"); -/// # Ok(quux) -/// # } -/// # fn main() { -/// # do_test(); // so that we can use the ? operator in the test -/// # } -/// ``` -/// This is useful for passing static strings to C from Rust FFI code. To do so -/// so, use the `.as_ptr()` method on the resulting `&'static CStr` to convert -/// it to the Rust equivalent of a C `const char*`: -/// -/// ``` -/// #[macro_use] -/// extern crate tor_util; -/// -/// use std::ffi::CStr; -/// use std::os::raw::c_char; -/// -/// pub extern "C" fn give_static_borrowed_string_to_c() -> *const c_char { -/// let hello: &'static CStr = cstr!("Hello, language my parents wrote."); -/// -/// hello.as_ptr() -/// } -/// # fn main() { -/// # let greetings = give_static_borrowed_string_to_c(); -/// # } -/// ``` -/// Note that the C code this static borrowed string is passed to *MUST NOT* -/// attempt to free the memory for the string. -/// -/// # Note -/// -/// An unfortunate limitation of the rustc compiler (as of 1.25.0-nightly), is -/// that the first example above compiles, but if we were to change the -/// assignment of `tuesday` as follows, it will fail to compile, because Rust -/// macros are expanded at parse time, and at parse time there is no symbol -/// table available. -/// -/// ```ignore -/// tuesday = cstr!(message); -/// ``` -/// with the error message `error: expected a literal`. -/// -/// # Returns -/// -/// If the string literals passed as arguments contain no NUL bytes anywhere, -/// then an `&'static CStr` containing the (concatenated) bytes of the string -/// literal(s) passed as arguments, with a NUL byte appended, is returned. -/// Otherwise, an `&'static CStr` containing a single NUL byte is returned (an -/// "empty" string in C). -#[macro_export] -macro_rules! cstr { - ($($bytes:expr),*) => ( - ::std::ffi::CStr::from_bytes_with_nul( - concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or_default() - ) -} - -#[cfg(test)] -mod test { - use std::ffi::CStr; - - #[test] - fn cstr_macro() { - let _: &'static CStr = cstr!("boo"); - } - - #[test] - fn cstr_macro_multi_input() { - let quux: &'static CStr = cstr!("foo", "bar", "baz"); - - assert!(quux.to_str().unwrap() == "foobarbaz"); - } - - #[test] - fn cstr_macro_bad_input() { - let waving: &'static CStr = cstr!("waving not drowning o/"); - let drowning: &'static CStr = cstr!("\0 drowning not waving"); - - assert!(waving.to_str().unwrap() == "waving not drowning o/"); - assert!(drowning.to_str().unwrap() == "") - } -} diff --git a/src/test/fuzz/fuzz_address.c b/src/test/fuzz/fuzz_address.c new file mode 100644 index 0000000000..6dccd65e9d --- /dev/null +++ b/src/test/fuzz/fuzz_address.c @@ -0,0 +1,26 @@ +#include "lib/net/address.h" +#include "lib/malloc/malloc.h" + +#include "test/fuzz/fuzzing.h" + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + tor_addr_t addr; + char *fuzzing_data = tor_memdup_nulterm(data, sz); + tor_addr_parse(&addr, fuzzing_data); + tor_free(fuzzing_data); + return 0; +} diff --git a/src/test/fuzz/fuzz_addressPTR.c b/src/test/fuzz/fuzz_addressPTR.c new file mode 100644 index 0000000000..b503d53666 --- /dev/null +++ b/src/test/fuzz/fuzz_addressPTR.c @@ -0,0 +1,32 @@ +#include "lib/net/address.h" +#include "lib/net/socket.h" +#include "lib/cc/ctassert.h" +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/log/log.h" +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/net/address.h" +#include "test/fuzz/fuzzing.h" + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + tor_addr_t addr_result; + char *fuzzing_data = tor_memdup_nulterm(data, sz); + tor_addr_parse_PTR_name(&addr_result, fuzzing_data, AF_UNSPEC, 1); + tor_free(fuzzing_data); + return 0; +} diff --git a/src/test/fuzz/fuzz_hsdescv3_inner.c b/src/test/fuzz/fuzz_hsdescv3_inner.c new file mode 100644 index 0000000000..5aa719f5c3 --- /dev/null +++ b/src/test/fuzz/fuzz_hsdescv3_inner.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2017-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define HS_DESCRIPTOR_PRIVATE + +#include "core/or/or.h" +#include "trunnel/ed25519_cert.h" /* Trunnel interface. */ +#include "lib/crypt_ops/crypto_ed25519.h" +#include "feature/hs/hs_descriptor.h" +#include "feature/dirparse/unparseable.h" + +#include "test/fuzz/fuzzing.h" + +static void +mock_dump_desc__nodump(const char *desc, const char *type) +{ + (void)desc; + (void)type; +} + +static int +mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert, + const size_t crosscert_len, + const crypto_pk_t *rsa_id_key, + const ed25519_public_key_t *master_key, + const time_t reject_if_expired_before) +{ + (void) crosscert; + (void) crosscert_len; + (void) rsa_id_key; + (void) master_key; + (void) reject_if_expired_before; + return 0; +} + +static size_t +mock_decrypt_desc_layer(const hs_descriptor_t *desc, + const uint8_t *descriptor_cookie, + bool is_superencrypted_layer, + char **decrypted_out) +{ + (void)is_superencrypted_layer; + (void)desc; + (void)descriptor_cookie; + const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN; + const uint8_t *encrypted_blob = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob + : desc->superencrypted_data.encrypted_blob; + size_t encrypted_blob_size = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob_size + : desc->superencrypted_data.encrypted_blob_size; + + if (encrypted_blob_size < overhead) + return 0; + *decrypted_out = tor_memdup_nulterm( + encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN, + encrypted_blob_size - overhead); + size_t result = strlen(*decrypted_out); + if (result) { + return result; + } else { + tor_free(*decrypted_out); + return 0; + } +} + +static const uint8_t *decrypted_data = NULL; +static size_t decrypted_len = 0; +static size_t +mock_desc_decrypt_encrypted(const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + char **decrypted_out) +{ + (void)desc; + (void)client_auth_sk; + *decrypted_out = (char*)tor_memdup_nulterm(decrypted_data, decrypted_len); + return decrypted_len; +} + +int +fuzz_init(void) +{ + disable_signature_checking(); + MOCK(dump_desc, mock_dump_desc__nodump); + MOCK(rsa_ed25519_crosscert_check, mock_rsa_ed25519_crosscert_check); + MOCK(decrypt_desc_layer, mock_decrypt_desc_layer); + MOCK(desc_decrypt_encrypted, mock_desc_decrypt_encrypted); + ed25519_init(); + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + decrypted_data = data; + decrypted_len = sz; + + hs_descriptor_t *desc = tor_malloc_zero(sizeof(hs_descriptor_t)); + hs_desc_encrypted_data_t *output = tor_malloc_zero(sizeof(*output)); + curve25519_secret_key_t *client_auth_sk = NULL; + hs_desc_decode_status_t status; + + status = desc_decode_encrypted_v3(desc, client_auth_sk, output); + if (status == HS_DESC_DECODE_OK) { + log_debug(LD_GENERAL, "Decoding okay"); + } else { + log_debug(LD_GENERAL, "Decoding failed"); + } + + hs_descriptor_free(desc); + hs_desc_encrypted_data_free(output); + return 0; +} diff --git a/src/test/fuzz/fuzz_hsdescv3_middle.c b/src/test/fuzz/fuzz_hsdescv3_middle.c new file mode 100644 index 0000000000..66a9d52cf3 --- /dev/null +++ b/src/test/fuzz/fuzz_hsdescv3_middle.c @@ -0,0 +1,116 @@ +/* Copyright (c) 2017-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define HS_DESCRIPTOR_PRIVATE + +#include "core/or/or.h" +#include "trunnel/ed25519_cert.h" /* Trunnel interface. */ +#include "lib/crypt_ops/crypto_ed25519.h" +#include "feature/hs/hs_descriptor.h" +#include "feature/dirparse/unparseable.h" + +#include "test/fuzz/fuzzing.h" + +static void +mock_dump_desc__nodump(const char *desc, const char *type) +{ + (void)desc; + (void)type; +} + +static int +mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert, + const size_t crosscert_len, + const crypto_pk_t *rsa_id_key, + const ed25519_public_key_t *master_key, + const time_t reject_if_expired_before) +{ + (void) crosscert; + (void) crosscert_len; + (void) rsa_id_key; + (void) master_key; + (void) reject_if_expired_before; + return 0; +} + +static size_t +mock_decrypt_desc_layer(const hs_descriptor_t *desc, + const uint8_t *descriptor_cookie, + bool is_superencrypted_layer, + char **decrypted_out) +{ + (void)is_superencrypted_layer; + (void)desc; + (void)descriptor_cookie; + const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN; + const uint8_t *encrypted_blob = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob + : desc->superencrypted_data.encrypted_blob; + size_t encrypted_blob_size = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob_size + : desc->superencrypted_data.encrypted_blob_size; + + if (encrypted_blob_size < overhead) + return 0; + *decrypted_out = tor_memdup_nulterm( + encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN, + encrypted_blob_size - overhead); + size_t result = strlen(*decrypted_out); + if (result) { + return result; + } else { + tor_free(*decrypted_out); + return 0; + } +} + +static const uint8_t *decrypted_data = NULL; +static size_t decrypted_len = 0; +static size_t +mock_desc_decrypt_superencrypted(const hs_descriptor_t *desc, + char **decrypted_out) +{ + (void)desc; + *decrypted_out = (char*)tor_memdup_nulterm(decrypted_data, decrypted_len); + return decrypted_len; +} + +int +fuzz_init(void) +{ + disable_signature_checking(); + MOCK(dump_desc, mock_dump_desc__nodump); + MOCK(rsa_ed25519_crosscert_check, mock_rsa_ed25519_crosscert_check); + MOCK(decrypt_desc_layer, mock_decrypt_desc_layer); + MOCK(desc_decrypt_superencrypted, mock_desc_decrypt_superencrypted); + ed25519_init(); + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + decrypted_data = data; + decrypted_len = sz; + + hs_descriptor_t *desc = tor_malloc_zero(sizeof(hs_descriptor_t)); + hs_desc_superencrypted_data_t *output = tor_malloc_zero(sizeof(*output)); + hs_desc_decode_status_t status; + + status = desc_decode_superencrypted_v3(desc, output); + if (status == HS_DESC_DECODE_OK) { + log_debug(LD_GENERAL, "Decoding okay"); + } else { + log_debug(LD_GENERAL, "Decoding failed"); + } + + hs_descriptor_free(desc); + hs_desc_superencrypted_data_free(output); + return 0; +} diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 9bdced9e6f..9fece7d004 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -8,7 +8,6 @@ FUZZING_LDFLAG = \ @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @@ -23,17 +22,36 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) # ===== AFL fuzzers if UNITTESTS_ENABLED +src_test_fuzz_fuzz_address_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_address.c +src_test_fuzz_fuzz_address_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_address_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_address_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_address_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_addressPTR_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_addressPTR.c +src_test_fuzz_fuzz_addressPTR_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_addressPTR_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_addressPTR_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_addressPTR_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_consensus_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_consensus.c @@ -94,6 +112,28 @@ src_test_fuzz_fuzz_hsdescv3_LDADD = $(FUZZING_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_fuzz_hsdescv3_inner_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_hsdescv3_inner.c +src_test_fuzz_fuzz_hsdescv3_inner_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_hsdescv3_inner_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_hsdescv3_inner_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_hsdescv3_inner_LDADD = $(FUZZING_LIBS) +endif + + +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_hsdescv3_middle_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_hsdescv3_middle.c +src_test_fuzz_fuzz_hsdescv3_middle_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_hsdescv3_middle_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_hsdescv3_middle_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_hsdescv3_middle_LDADD = $(FUZZING_LIBS) +endif + + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_http_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_http.c @@ -155,12 +195,16 @@ endif if UNITTESTS_ENABLED FUZZERS = \ + src/test/fuzz/fuzz-address \ + src/test/fuzz/fuzz-addressPTR \ src/test/fuzz/fuzz-consensus \ src/test/fuzz/fuzz-descriptor \ src/test/fuzz/fuzz-diff \ src/test/fuzz/fuzz-diff-apply \ src/test/fuzz/fuzz-extrainfo \ src/test/fuzz/fuzz-hsdescv3 \ + src/test/fuzz/fuzz-hsdescv3-inner \ + src/test/fuzz/fuzz-hsdescv3-middle \ src/test/fuzz/fuzz-http \ src/test/fuzz/fuzz-http-connect \ src/test/fuzz/fuzz-microdesc \ @@ -173,6 +217,24 @@ endif if LIBFUZZER_ENABLED if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_address_SOURCES = \ + $(src_test_fuzz_fuzz_address_SOURCES) +src_test_fuzz_lf_fuzz_address_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_address_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_address_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_address_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_addressPTR_SOURCES = \ + $(src_test_fuzz_fuzz_addressPTR_SOURCES) +src_test_fuzz_lf_fuzz_addressPTR_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_addressPTR_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_addressPTR_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_addressPTR_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_consensus_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -227,6 +289,25 @@ src_test_fuzz_lf_fuzz_hsdescv3_LDADD = $(LIBFUZZER_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_hsdescv3_inner_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_inner_SOURCES) +src_test_fuzz_lf_fuzz_hsdescv3_inner_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_inner_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_inner_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_hsdescv3_inner_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_hsdescv3_middle_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_middle_SOURCES) +src_test_fuzz_lf_fuzz_hsdescv3_middle_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_middle_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_middle_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_hsdescv3_middle_LDADD = $(LIBFUZZER_LIBS) +endif + + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_http_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -281,12 +362,16 @@ src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS) endif LIBFUZZER_FUZZERS = \ + src/test/fuzz/lf-fuzz-address \ + src/test/fuzz/lf-fuzz-addressPTR \ src/test/fuzz/lf-fuzz-consensus \ src/test/fuzz/lf-fuzz-descriptor \ src/test/fuzz/lf-fuzz-diff \ src/test/fuzz/lf-fuzz-diff-apply \ src/test/fuzz/lf-fuzz-extrainfo \ src/test/fuzz/lf-fuzz-hsdescv3 \ + src/test/fuzz/lf-fuzz-hsdescv3-inner \ + src/test/fuzz/lf-fuzz-hsdescv3-middle \ src/test/fuzz/lf-fuzz-http \ src/test/fuzz/lf-fuzz-http-connect \ src/test/fuzz/lf-fuzz-microdesc \ @@ -302,6 +387,20 @@ endif if OSS_FUZZ_ENABLED if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_address_a_SOURCES = \ + $(src_test_fuzz_fuzz_address_SOURCES) +src_test_fuzz_liboss_fuzz_address_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_address_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_addressPTR_a_SOURCES = \ + $(src_test_fuzz_fuzz_addressPTR_SOURCES) +src_test_fuzz_liboss_fuzz_addressPTR_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_addressPTR_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -344,6 +443,20 @@ src_test_fuzz_liboss_fuzz_hsdescv3_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_hsdescv3_inner_a_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_inner_SOURCES) +src_test_fuzz_liboss_fuzz_hsdescv3_inner_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_hsdescv3_inner_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_hsdescv3_middle_a_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_middle_SOURCES) +src_test_fuzz_liboss_fuzz_hsdescv3_middle_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_hsdescv3_middle_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -386,12 +499,16 @@ src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif OSS_FUZZ_FUZZERS = \ + src/test/fuzz/liboss-fuzz-address.a \ + src/test/fuzz/liboss-fuzz-addressPTR.a \ src/test/fuzz/liboss-fuzz-consensus.a \ src/test/fuzz/liboss-fuzz-descriptor.a \ src/test/fuzz/liboss-fuzz-diff.a \ src/test/fuzz/liboss-fuzz-diff-apply.a \ src/test/fuzz/liboss-fuzz-extrainfo.a \ src/test/fuzz/liboss-fuzz-hsdescv3.a \ + src/test/fuzz/liboss-fuzz-hsdescv3-inner.a \ + src/test/fuzz/liboss-fuzz-hsdescv3-middle.a \ src/test/fuzz/liboss-fuzz-http.a \ src/test/fuzz/liboss-fuzz-http-connect.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ diff --git a/src/test/include.am b/src/test/include.am index d5dcebfaea..2765cf27d0 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -8,11 +8,7 @@ TESTS_ENVIRONMENT = \ export abs_top_builddir="$(abs_top_builddir)"; \ export builddir="$(builddir)"; \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \ - export CARGO="$(CARGO)"; \ - export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \ - export CARGO_ONLINE="$(CARGO_ONLINE)"; \ - export CCLD="$(CCLD)"; \ - export RUSTFLAGS="-C linker=`echo '$(CC)' | cut -d' ' -f 1` $(RUST_LINKER_OPTIONS)"; + export CCLD="$(CCLD)"; TESTSCRIPTS = \ src/test/fuzz_static_testcases.sh \ @@ -35,11 +31,6 @@ TESTSCRIPTS = \ src/test/unittest_part7.sh \ src/test/unittest_part8.sh -if USE_RUST -TESTSCRIPTS += \ - src/test/test_rust.sh -endif - if USEPYTHON TESTSCRIPTS += \ src/test/test_ntor.sh \ @@ -170,6 +161,7 @@ src_test_test_SOURCES += \ src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ + src/test/test_dirauth_ports.c \ src/test/test_dirvote.c \ src/test/test_dir_common.c \ src/test/test_dir_handle_get.c \ @@ -203,6 +195,7 @@ src_test_test_SOURCES += \ src/test/test_namemap.c \ src/test/test_netinfo.c \ src/test/test_nodelist.c \ + src/test/test_ntor_v3.c \ src/test/test_oom.c \ src/test/test_oos.c \ src/test/test_options.c \ @@ -229,6 +222,7 @@ src_test_test_SOURCES += \ src/test/test_routerkeys.c \ src/test/test_routerlist.c \ src/test/test_routerset.c \ + src/test/test_sandbox.c \ src/test/test_scheduler.c \ src/test/test_sendme.c \ src/test/test_shared_random.c \ @@ -303,7 +297,6 @@ src_test_test_switch_id_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ $(TOR_UTIL_TESTING_LIBS) \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@ \ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ @@ -311,7 +304,6 @@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_test_LDADD = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -340,7 +332,6 @@ src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_bench_LDADD = \ libtor.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -350,7 +341,6 @@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_test_workqueue_LDADD = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -362,7 +352,6 @@ src_test_test_timers_LDADD = \ src/lib/libtor-evloop-testing.a \ $(TOR_CRYPTO_TESTING_LIBS) \ $(TOR_UTIL_TESTING_LIBS) \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -398,7 +387,6 @@ src_test_test_ntor_cl_SOURCES = src/test/test_ntor_cl.c src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) src_test_test_ntor_cl_LDADD = \ libtor.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@ @@ -421,7 +409,6 @@ noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = \ $(TOR_UTIL_TESTING_LIBS) \ - $(rust_ldadd) \ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @TOR_TRACE_LIBS@ @@ -443,13 +430,11 @@ EXTRA_DIST += \ src/test/test_include.py \ src/test/zero_length_keys.sh \ scripts/maint/run_check_subsystem_order.sh \ - src/test/rust_supp.txt \ src/test/test_keygen.sh \ src/test/test_key_expiration.sh \ src/test/test_zero_length_keys.sh \ src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh \ src/test/test-network.sh \ - src/test/test_rust.sh \ src/test/test_switch_id.sh \ src/test/test_workqueue_cancel.sh \ src/test/test_workqueue_efd.sh \ @@ -467,6 +452,3 @@ EXTRA_DIST += \ src/test/unittest_part6.sh \ src/test/unittest_part7.sh \ src/test/unittest_part8.sh - -test-rust: - $(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh" diff --git a/src/test/ntor_v3_ref.py b/src/test/ntor_v3_ref.py new file mode 100755 index 0000000000..28bc077105 --- /dev/null +++ b/src/test/ntor_v3_ref.py @@ -0,0 +1,308 @@ +#!/usr/bin/python + +import binascii +import hashlib +import os +import struct + +import donna25519 +from Crypto.Cipher import AES +from Crypto.Util import Counter + +# Define basic wrappers. + +DIGEST_LEN = 32 +ENC_KEY_LEN = 32 +PUB_KEY_LEN = 32 +SEC_KEY_LEN = 32 +IDENTITY_LEN = 32 + +def sha3_256(s): + d = hashlib.sha3_256(s).digest() + assert len(d) == DIGEST_LEN + return d + +def shake_256(s): + # Note: In reality, you wouldn't want to generate more bytes than needed. + MAX_KEY_BYTES = 1024 + return hashlib.shake_256(s).digest(MAX_KEY_BYTES) + +def curve25519(pk, sk): + assert len(pk) == PUB_KEY_LEN + assert len(sk) == SEC_KEY_LEN + private = donna25519.PrivateKey.load(sk) + public = donna25519.PublicKey(pk) + return private.do_exchange(public) + +def keygen(): + private = donna25519.PrivateKey() + public = private.get_public() + return (private.private, public.public) + +def aes256_ctr(k, s): + assert len(k) == ENC_KEY_LEN + cipher = AES.new(k, AES.MODE_CTR, counter=Counter.new(128, initial_value=0)) + return cipher.encrypt(s) + +# Byte-oriented helper. We use this for decoding keystreams and messages. + +class ByteSeq: + def __init__(self, data): + self.data = data + + def take(self, n): + assert n <= len(self.data) + result = self.data[:n] + self.data = self.data[n:] + return result + + def exhausted(self): + return len(self.data) == 0 + + def remaining(self): + return len(self.data) + +# Low-level functions + +MAC_KEY_LEN = 32 +MAC_LEN = DIGEST_LEN + +hash_func = sha3_256 + +def encapsulate(s): + """encapsulate `s` with a length prefix. + + We use this whenever we need to avoid message ambiguities in + cryptographic inputs. + """ + assert len(s) <= 0xffffffff + header = b"\0\0\0\0" + struct.pack("!L", len(s)) + assert len(header) == 8 + return header + s + +def h(s, tweak): + return hash_func(encapsulate(tweak) + s) + +def mac(s, key, tweak): + return hash_func(encapsulate(tweak) + encapsulate(key) + s) + +def kdf(s, tweak): + data = shake_256(encapsulate(tweak) + s) + return ByteSeq(data) + +def enc(s, k): + return aes256_ctr(k, s) + +# Tweaked wrappers + +PROTOID = b"ntor3-curve25519-sha3_256-1" +T_KDF_PHASE1 = PROTOID + b":kdf_phase1" +T_MAC_PHASE1 = PROTOID + b":msg_mac" +T_KDF_FINAL = PROTOID + b":kdf_final" +T_KEY_SEED = PROTOID + b":key_seed" +T_VERIFY = PROTOID + b":verify" +T_AUTH = PROTOID + b":auth_final" + +def kdf_phase1(s): + return kdf(s, T_KDF_PHASE1) + +def kdf_final(s): + return kdf(s, T_KDF_FINAL) + +def mac_phase1(s, key): + return mac(s, key, T_MAC_PHASE1) + +def h_key_seed(s): + return h(s, T_KEY_SEED) + +def h_verify(s): + return h(s, T_VERIFY) + +def h_auth(s): + return h(s, T_AUTH) + +# Handshake. + +def client_phase1(msg, verification, B, ID): + assert len(B) == PUB_KEY_LEN + assert len(ID) == IDENTITY_LEN + + (x,X) = keygen() + p(["x", "X"], locals()) + p(["msg", "verification"], locals()) + Bx = curve25519(B, x) + secret_input_phase1 = Bx + ID + X + B + PROTOID + encapsulate(verification) + + phase1_keys = kdf_phase1(secret_input_phase1) + enc_key = phase1_keys.take(ENC_KEY_LEN) + mac_key = phase1_keys.take(MAC_KEY_LEN) + p(["enc_key", "mac_key"], locals()) + + msg_0 = ID + B + X + enc(msg, enc_key) + mac = mac_phase1(msg_0, mac_key) + p(["mac"], locals()) + + client_handshake = msg_0 + mac + state = dict(x=x, X=X, B=B, ID=ID, Bx=Bx, mac=mac, verification=verification) + + p(["client_handshake"], locals()) + + return (client_handshake, state) + +# server. + +class Reject(Exception): + pass + +def server_part1(cmsg, verification, b, B, ID): + assert len(B) == PUB_KEY_LEN + assert len(ID) == IDENTITY_LEN + assert len(b) == SEC_KEY_LEN + + if len(cmsg) < (IDENTITY_LEN + PUB_KEY_LEN * 2 + MAC_LEN): + raise Reject() + + mac_covered_portion = cmsg[0:-MAC_LEN] + cmsg = ByteSeq(cmsg) + cmsg_id = cmsg.take(IDENTITY_LEN) + cmsg_B = cmsg.take(PUB_KEY_LEN) + cmsg_X = cmsg.take(PUB_KEY_LEN) + cmsg_msg = cmsg.take(cmsg.remaining() - MAC_LEN) + cmsg_mac = cmsg.take(MAC_LEN) + + assert cmsg.exhausted() + + # XXXX for real purposes, you would use constant-time checks here + if cmsg_id != ID or cmsg_B != B: + raise Reject() + + Xb = curve25519(cmsg_X, b) + secret_input_phase1 = Xb + ID + cmsg_X + B + PROTOID + encapsulate(verification) + + phase1_keys = kdf_phase1(secret_input_phase1) + enc_key = phase1_keys.take(ENC_KEY_LEN) + mac_key = phase1_keys.take(MAC_KEY_LEN) + + mac_received = mac_phase1(mac_covered_portion, mac_key) + if mac_received != cmsg_mac: + raise Reject() + + client_msg = enc(cmsg_msg, enc_key) + state = dict( + b=b, + B=B, + X=cmsg_X, + mac_received=mac_received, + Xb=Xb, + ID=ID, + verification=verification) + + return (client_msg, state) + +def server_part2(state, server_msg): + X = state['X'] + Xb = state['Xb'] + B = state['B'] + b = state['b'] + ID = state['ID'] + mac_received = state['mac_received'] + verification = state['verification'] + + p(["server_msg"], locals()) + + (y,Y) = keygen() + p(["y", "Y"], locals()) + Xy = curve25519(X, y) + + secret_input = Xy + Xb + ID + B + X + Y + PROTOID + encapsulate(verification) + key_seed = h_key_seed(secret_input) + verify = h_verify(secret_input) + p(["key_seed", "verify"], locals()) + + keys = kdf_final(key_seed) + server_enc_key = keys.take(ENC_KEY_LEN) + p(["server_enc_key"], locals()) + + smsg_msg = enc(server_msg, server_enc_key) + + auth_input = verify + ID + B + Y + X + mac_received + encapsulate(smsg_msg) + PROTOID + b"Server" + + auth = h_auth(auth_input) + server_handshake = Y + auth + smsg_msg + p(["auth", "server_handshake"], locals()) + + return (server_handshake, keys) + +def client_phase2(state, smsg): + x = state['x'] + X = state['X'] + B = state['B'] + ID = state['ID'] + Bx = state['Bx'] + mac_sent = state['mac'] + verification = state['verification'] + + if len(smsg) < PUB_KEY_LEN + DIGEST_LEN: + raise Reject() + + smsg = ByteSeq(smsg) + Y = smsg.take(PUB_KEY_LEN) + auth_received = smsg.take(DIGEST_LEN) + server_msg = smsg.take(smsg.remaining()) + + Yx = curve25519(Y,x) + + secret_input = Yx + Bx + ID + B + X + Y + PROTOID + encapsulate(verification) + key_seed = h_key_seed(secret_input) + verify = h_verify(secret_input) + + auth_input = verify + ID + B + Y + X + mac_sent + encapsulate(server_msg) + PROTOID + b"Server" + + auth = h_auth(auth_input) + if auth != auth_received: + raise Reject() + + keys = kdf_final(key_seed) + enc_key = keys.take(ENC_KEY_LEN) + + server_msg_decrypted = enc(server_msg, enc_key) + + return (keys, server_msg_decrypted) + +def p(varnames, localvars): + for v in varnames: + label = v + val = localvars[label] + print('{} = "{}"'.format(label, binascii.b2a_hex(val).decode("ascii"))) + +def test(): + (b,B) = keygen() + ID = os.urandom(IDENTITY_LEN) + + p(["b", "B", "ID"], locals()) + + print("# ============") + (c_handshake, c_state) = client_phase1(b"hello world", b"xyzzy", B, ID) + + print("# ============") + + (c_msg_got, s_state) = server_part1(c_handshake, b"xyzzy", b, B, ID) + + #print(repr(c_msg_got)) + + (s_handshake, s_keys) = server_part2(s_state, b"Hola Mundo") + + print("# ============") + + (c_keys, s_msg_got) = client_phase2(c_state, s_handshake) + + #print(repr(s_msg_got)) + + c_keys_256 = c_keys.take(256) + p(["c_keys_256"], locals()) + + assert (c_keys_256 == s_keys.take(256)) + + +if __name__ == '__main__': + test() diff --git a/src/test/test.c b/src/test/test.c index 40c053a660..c38d78da30 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1,5 +1,4 @@ /* Copyright (c) 2001-2004, Roger Dingledine. -->a * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2021, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -53,6 +52,7 @@ #include "core/crypto/onion_fast.h" #include "core/crypto/onion_tap.h" #include "core/or/policies.h" +#include "lib/sandbox/sandbox.h" #include "app/config/statefile.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "feature/nodelist/networkstatus.h" @@ -673,6 +673,7 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, + { "dir/auth/ports/", dirauth_port_tests }, { "dir/auth/process_descs/", process_descs_tests }, { "dir/md/", microdesc_tests }, { "dirauth/dirvote/", dirvote_tests}, @@ -707,6 +708,7 @@ struct testgroup_t testgroups[] = { { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, + { "onion-handshake/ntor-v3/", ntor_v3_tests }, { "oos/", oos_tests }, { "options/", options_tests }, { "options/act/", options_act_tests }, @@ -731,6 +733,9 @@ struct testgroup_t testgroups[] = { { "routerkeys/", routerkeys_tests }, { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, +#ifdef USE_LIBSECCOMP + { "sandbox/" , sandbox_tests }, +#endif { "scheduler/", scheduler_tests }, { "sendme/", sendme_tests }, { "shared-random/", sr_tests }, diff --git a/src/test/test.h b/src/test/test.h index f88bc98498..e17bce427c 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -120,6 +120,7 @@ extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; extern struct testcase_t crypto_rng_tests[]; extern struct testcase_t crypto_tests[]; +extern struct testcase_t dirauth_port_tests[]; extern struct testcase_t dir_handle_get_tests[]; extern struct testcase_t dir_tests[]; extern struct testcase_t dirvote_tests[]; @@ -154,6 +155,7 @@ extern struct testcase_t microdesc_tests[]; extern struct testcase_t namemap_tests[]; extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; +extern struct testcase_t ntor_v3_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; @@ -182,6 +184,7 @@ extern struct testcase_t router_tests[]; extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; +extern struct testcase_t sandbox_tests[]; extern struct testcase_t scheduler_tests[]; extern struct testcase_t sendme_tests[]; extern struct testcase_t socks_tests[]; diff --git a/src/test/test_address.c b/src/test/test_address.c index 9c1415419c..015ca0807c 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -1326,6 +1326,42 @@ test_address_dirserv_router_addr_private(void *opt_dir_allow_private) UNMOCK(get_options); } +static void +test_address_parse_port_range(void *arg) +{ + int ret; + uint16_t min_out = 0; + uint16_t max_out = 0; + + (void) arg; + + /* Invalid. */ + ret = parse_port_range("0x00", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("0x01", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("1817161", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("65536", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("1-65536", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + + /* Valid. */ + ret = parse_port_range("65535", &min_out, &max_out); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(min_out, OP_EQ, 65535); + tt_int_op(max_out, OP_EQ, 65535); + + ret = parse_port_range("1-65535", &min_out, &max_out); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(min_out, OP_EQ, 1); + tt_int_op(max_out, OP_EQ, 65535); + + done: + ; +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } #define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \ @@ -1364,5 +1400,6 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_node_in_same_network_family, 0), ADDRESS_TEST(dirserv_router_addr_private, 0), ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"), + ADDRESS_TEST(parse_port_range, 0), END_OF_TESTCASES }; diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 5219c86097..ca7fee2c53 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -20,6 +20,7 @@ #include "lib/tls/tortls.h" #include "core/or/or_connection_st.h" +#include "core/or/congestion_control_common.h" /* Test suite stuff */ #include "test/test.h" @@ -155,7 +156,7 @@ test_channeltls_num_bytes_queued(void *arg) * - 2 cells. */ n = ch->num_cells_writeable(ch); - tt_int_op(n, OP_EQ, CEIL_DIV(OR_CONN_HIGHWATER, 512) - 2); + tt_int_op(n, OP_EQ, CEIL_DIV(or_conn_highwatermark(), 512) - 2); UNMOCK(buf_datalen); tlschan_buf_datalen_mock_target = NULL; tlschan_buf_datalen_mock_size = 0; diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 873391a84f..0a5c3530bd 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -113,7 +113,7 @@ test_new_route_len_safe_exit(void *arg) /* hidden service connecting to introduction point */ r = new_route_len(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, &dummy_ei, &dummy_nodes); - tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r); + tt_int_op(DEFAULT_ROUTE_LEN+1, OP_EQ, r); /* router testing its own reachability */ r = new_route_len(CIRCUIT_PURPOSE_TESTING, &dummy_ei, &dummy_nodes); diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 86baf54f40..6ced3f4111 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1367,7 +1367,7 @@ test_circuitpadding_wronghop(void *arg) tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); - tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); tt_int_op(n_client_cells, OP_EQ, 2); /* 6. Sending negotiated command to relay does nothing */ @@ -1396,11 +1396,9 @@ test_circuitpadding_wronghop(void *arg) /* verify no padding was negotiated */ tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); - tt_int_op(n_relay_cells, OP_EQ, 3); - tt_int_op(n_client_cells, OP_EQ, 2); /* verify no echo was sent */ - tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); tt_int_op(n_client_cells, OP_EQ, 2); /* Finish circuit */ diff --git a/src/test/test_connection.c b/src/test/test_connection.c index 9c726c07f8..87940f71e6 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -992,12 +992,12 @@ test_conn_describe(void *arg) #define STR(x) #x /* where arg is an expression (constant, variable, compound expression) */ -#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg) \ - { #name "_" STR(x), \ +#define CONNECTION_TESTCASE_ARG(name, extra, fork, setup, arg) \ + { STR(name)"/"extra, \ test_conn_##name, \ - fork, \ - &setup, \ - (void *)arg } + (fork), \ + &(setup), \ + (void *)(arg) } #endif /* !defined(COCCI) */ static const unsigned int PROXY_CONNECT_ARG = PROXY_CONNECT; @@ -1007,14 +1007,14 @@ struct testcase_t connection_tests[] = { CONNECTION_TESTCASE(get_basic, TT_FORK, test_conn_get_basic_st), CONNECTION_TESTCASE(get_rsrc, TT_FORK, test_conn_get_rsrc_st), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "microdesc", TT_FORK, test_conn_download_status_st, "microdesc"), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "ns", TT_FORK, test_conn_download_status_st, "ns"), - CONNECTION_TESTCASE_ARG(https_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(https_proxy_connect, "https", TT_FORK, test_conn_proxy_connect_st, &PROXY_CONNECT_ARG), - CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, "haproxy", TT_FORK, test_conn_proxy_connect_st, &PROXY_HAPROXY_ARG), //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair), diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0d2d6800ba..186e09f236 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -6652,13 +6652,7 @@ test_dir_find_dl_min_delay(void* data) dls.schedule = DL_SCHED_BRIDGE; /* client */ - mock_options->ClientOnly = 1; - mock_options->UseBridges = 1; - if (num_bridges_usable(0) > 0) { - tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge); - } else { - tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap); - } + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap); done: UNMOCK(networkstatus_consensus_is_bootstrapping); diff --git a/src/test/test_dirauth_ports.c b/src/test/test_dirauth_ports.c new file mode 100644 index 0000000000..5dc0b0b631 --- /dev/null +++ b/src/test/test_dirauth_ports.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONFIG_PRIVATE + +#include "core/or/or.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/dirlist.h" +#include "app/config/config.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +static void +test_dirauth_port_parsing(void *arg) +{ + (void)arg; + + // This one is okay. + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/ " + "download=http://128.31.0.39:9131 " + "vote=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + + // These have bad syntax. + setup_capture_of_logs(LOG_WARN); + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "uploadx=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + expect_log_msg_containing("Unrecognized flag"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=https://128.31.0.39:9131/ " // https is not recognized + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL scheme"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/tor " // suffix is not supported + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL prefix"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.256:9131/ " // "256" is not ipv4. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://xyz.example.com/ " // hostnames not supported. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + done: + teardown_capture_of_logs(); +} + +static void +test_dirauth_port_lookup(void *arg) +{ + (void)arg; + + clear_dir_servers(); + + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.40:9132/ " + "download=http://128.31.0.41:9133 " + "vote=http://128.31.0.42:9134/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + rv = parse_dir_authority_line( + "morgoth orport=9101 " + "v3ident=D586D18309DED4CDFFFFFFFFDB97EFA96D330566 " + "upload=http://128.31.0.43:9140/ " + "128.31.0.44:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + const smartlist_t *servers = router_get_trusted_dir_servers(); + tt_assert(servers); + tt_int_op(smartlist_len(servers), OP_EQ, 2); + const dir_server_t *moria = smartlist_get(servers, 0); + const dir_server_t *morgoth = smartlist_get(servers, 1); + tt_str_op(moria->nickname, OP_EQ, "moria1"); + tt_str_op(morgoth->nickname, OP_EQ, "morgoth"); + + const tor_addr_port_t *dirport; + + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9132); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9133); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9134); + + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9140); + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + + done: + ; +} + +#define T(name) \ + { #name, test_dirauth_port_ ## name, TT_FORK, NULL, NULL } + +struct testcase_t dirauth_port_tests[] = { + T(parsing), + T(lookup), + END_OF_TESTCASES +}; diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index c94b5d6a23..118b66dfa7 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -92,6 +92,12 @@ bfn_mock_node_get_by_id(const char *id) return NULL; } +static int +mock_router_have_minimum_dir_info(void) +{ + return 1; +} + /* Helper function to free a test node. */ static void test_node_free(node_t *n) @@ -3087,6 +3093,38 @@ test_entry_guard_vanguard_path_selection(void *arg) circuit_free_(circ); } +static void +test_entry_guard_layer2_guards(void *arg) +{ + (void) arg; + MOCK(router_have_minimum_dir_info, mock_router_have_minimum_dir_info); + + /* First check the enable/disable switch */ + get_options_mutable()->VanguardsLiteEnabled = 0; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 0); + + get_options_mutable()->VanguardsLiteEnabled = 1; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 1); + + get_options_mutable()->VanguardsLiteEnabled = -1; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 1); + + /* OK now let's move to actual testing */ + + /* Remove restrictions to route around Big Fake Network restrictions */ + get_options_mutable()->EnforceDistinctSubnets = 0; + + /* Create the L2 guardset */ + maintain_layer2_guards(); + + const routerset_t *l2_guards = get_layer2_guards(); + tt_assert(l2_guards); + tt_int_op(routerset_len(l2_guards), OP_EQ, 4); + + done: + UNMOCK(router_have_minimum_dir_info); +} + static const struct testcase_setup_t big_fake_network = { big_fake_network_setup, big_fake_network_cleanup }; @@ -3152,6 +3190,8 @@ struct testcase_t entrynodes_tests[] = { BFN_TEST(manage_primary), BFN_TEST(correct_cascading_order), + BFN_TEST(layer2_guards), + EN_TEST_FORK(guard_preferred), BFN_TEST(select_for_circuit_no_confirmed), diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 7cb6a36f8e..347a5b7174 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -808,13 +808,11 @@ test_parse_extended_hostname(void *arg) tt_assert(parse_extended_hostname(address4, &type)); tt_int_op(type, OP_EQ, NORMAL_HOSTNAME); - tt_assert(parse_extended_hostname(address5, &type)); - tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME); - tt_str_op(address5, OP_EQ, "abcdefghijklmnop"); + tt_assert(!parse_extended_hostname(address5, &type)); + tt_int_op(type, OP_EQ, BAD_HOSTNAME); - tt_assert(parse_extended_hostname(address6, &type)); - tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME); - tt_str_op(address6, OP_EQ, "abcdefghijklmnop"); + tt_assert(!parse_extended_hostname(address6, &type)); + tt_int_op(type, OP_EQ, BAD_HOSTNAME); tt_assert(!parse_extended_hostname(address7, &type)); tt_int_op(type, OP_EQ, BAD_HOSTNAME); diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index b036c5eada..c32803b380 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -798,7 +798,7 @@ test_hs_control_add_onion_helper_add_service(void *arg) hs_service_ht *global_map; hs_port_config_t *portcfg; smartlist_t *portcfgs; - char *address_out_good, *address_out_bad; + char *address_out_good = NULL, *address_out_bad = NULL; hs_service_t *service_good = NULL; hs_service_t *service_bad = NULL; diff --git a/src/test/test_hs_ob.c b/src/test/test_hs_ob.c index 3485655c2e..2f69bf31e0 100644 --- a/src/test/test_hs_ob.c +++ b/src/test/test_hs_ob.c @@ -174,6 +174,7 @@ test_get_subcredentials(void *arg) hs_subcredential_t *subcreds = NULL; (void) arg; + memset(&config, 0, sizeof(config)); MOCK(networkstatus_get_live_consensus, mock_networkstatus_get_live_consensus); diff --git a/src/test/test_ntor_v3.c b/src/test/test_ntor_v3.c new file mode 100644 index 0000000000..096ac6668f --- /dev/null +++ b/src/test/test_ntor_v3.c @@ -0,0 +1,172 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define ONION_NTOR_V3_PRIVATE +#include "core/or/or.h" +#include "test/test.h" +#include "lib/crypt_ops/crypto_curve25519.h" +#include "lib/crypt_ops/crypto_ed25519.h" +#include "core/crypto/onion_ntor_v3.h" + +#define unhex(arry, s) \ + { tt_int_op(sizeof(arry), OP_EQ, \ + base16_decode((char*)arry, sizeof(arry), s, strlen(s))); \ + } + +static void +test_ntor3_testvecs(void *arg) +{ + (void)arg; + char *mem_op_hex_tmp = NULL; // temp val to make test_memeq_hex work. + + ntor3_server_handshake_state_t *relay_state = NULL; + uint8_t *onion_skin = NULL; + size_t onion_skin_len; + ntor3_handshake_state_t *client_state = NULL; + uint8_t *cm = NULL, *sm = NULL; + size_t cm_len, sm_len; + di_digest256_map_t *private_keys = NULL; + uint8_t *server_handshake = NULL; + size_t server_handshake_len; + + // Test vectors from python implementation, confirmed with rust + // implementation. + curve25519_keypair_t relay_keypair_b; + curve25519_keypair_t client_keypair_x; + curve25519_keypair_t relay_keypair_y; + ed25519_public_key_t relay_id; + + unhex(relay_keypair_b.seckey.secret_key, + "4051daa5921cfa2a1c27b08451324919538e79e788a81b38cbed097a5dff454a"); + unhex(relay_keypair_b.pubkey.public_key, + "f8307a2bc1870b00b828bb74dbb8fd88e632a6375ab3bcd1ae706aaa8b6cdd1d"); + unhex(relay_id.pubkey, + "9fad2af287ef942632833d21f946c6260c33fae6172b60006e86e4a6911753a2"); + unhex(client_keypair_x.seckey.secret_key, + "b825a3719147bcbe5fb1d0b0fcb9c09e51948048e2e3283d2ab7b45b5ef38b49"); + unhex(client_keypair_x.pubkey.public_key, + "252fe9ae91264c91d4ecb8501f79d0387e34ad8ca0f7c995184f7d11d5da4f46"); + unhex(relay_keypair_y.seckey.secret_key, + "4865a5b7689dafd978f529291c7171bc159be076b92186405d13220b80e2a053"); + unhex(relay_keypair_y.pubkey.public_key, + "4bf4814326fdab45ad5184f5518bd7fae25dc59374062698201a50a22954246d"); + + uint8_t client_message[11]; + uint8_t verification[5]; + unhex(client_message, "68656c6c6f20776f726c64"); + unhex(verification, "78797a7a79"); + + // ========= Client handshake 1. + + onion_skin_ntor3_create_nokeygen( + &client_keypair_x, + &relay_id, + &relay_keypair_b.pubkey, + verification, + sizeof(verification), + client_message, + sizeof(client_message), + &client_state, + &onion_skin, + &onion_skin_len); + + const char expect_client_handshake[] = "9fad2af287ef942632833d21f946c6260c" + "33fae6172b60006e86e4a6911753a2f8307a2bc1870b00b828bb74dbb8fd88e632a6375" + "ab3bcd1ae706aaa8b6cdd1d252fe9ae91264c91d4ecb8501f79d0387e34ad8ca0f7c995" + "184f7d11d5da4f463bebd9151fd3b47c180abc9e044d53565f04d82bbb3bebed3d06cea" + "65db8be9c72b68cd461942088502f67"; + + tt_int_op(onion_skin_len, OP_EQ, strlen(expect_client_handshake)/2); + test_memeq_hex(onion_skin, expect_client_handshake); + + // ========= Relay handshake. + + dimap_add_entry(&private_keys, + relay_keypair_b.pubkey.public_key, + &relay_keypair_b); + + int r = onion_skin_ntor3_server_handshake_part1( + private_keys, + &client_keypair_x, + &relay_id, + onion_skin, + onion_skin_len, + verification, + sizeof(verification), + &cm, + &cm_len, + &relay_state); + tt_int_op(r, OP_EQ, 0); + tt_int_op(cm_len, OP_EQ, sizeof(client_message)); + tt_mem_op(cm, OP_EQ, client_message, cm_len); + + uint8_t server_message[10]; + unhex(server_message, "486f6c61204d756e646f"); + + uint8_t server_keys[256]; + onion_skin_ntor3_server_handshake_part2_nokeygen( + &relay_keypair_y, + relay_state, + verification, + sizeof(verification), + server_message, + sizeof(server_message), + &server_handshake, + &server_handshake_len, + server_keys, + sizeof(server_keys)); + + const char expect_server_handshake[] = "4bf4814326fdab45ad5184f5518bd7fae25" + "dc59374062698201a50a22954246d2fc5f8773ca824542bc6cf6f57c7c29bbf4e5476461" + "ab130c5b18ab0a91276651202c3e1e87c0d32054c"; + tt_int_op(server_handshake_len, OP_EQ, strlen(expect_server_handshake)/2); + test_memeq_hex(server_handshake, expect_server_handshake); + + uint8_t expect_keys[256]; + unhex(expect_keys, "9c19b631fd94ed86a817e01f6c80b0743a43f5faebd39cfaa8b00f" + "a8bcc65c3bfeaa403d91acbd68a821bf6ee8504602b094a254392a07737d5662768" + "c7a9fb1b2814bb34780eaee6e867c773e28c212ead563e98a1cd5d5b4576f5ee61c" + "59bde025ff2851bb19b721421694f263818e3531e43a9e4e3e2c661e2ad547d8984" + "caa28ebecd3e4525452299be26b9185a20a90ce1eac20a91f2832d731b54502b097" + "49b5a2a2949292f8cfcbeffb790c7790ed935a9d251e7e336148ea83b063a5618fc" + "ff674a44581585fd22077ca0e52c59a24347a38d1a1ceebddbf238541f226b8f88d" + "0fb9c07a1bcd2ea764bbbb5dacdaf5312a14c0b9e4f06309b0333b4a"); + tt_mem_op(server_keys, OP_EQ, expect_keys, 256); + + // ===== Client handshake 2 + + uint8_t client_keys[256]; + r = onion_ntor3_client_handshake( + client_state, + server_handshake, + server_handshake_len, + verification, + sizeof(verification), + client_keys, + sizeof(client_keys), + &sm, + &sm_len); + + tt_int_op(r, OP_EQ, 0); + tt_int_op(sm_len, OP_EQ, sizeof(server_message)); + tt_mem_op(sm, OP_EQ, server_message, sizeof(server_message)); + tt_mem_op(client_keys, OP_EQ, server_keys, 256); + + done: + tor_free(onion_skin); + tor_free(server_handshake); + tor_free(mem_op_hex_tmp); + ntor3_handshake_state_free(client_state); + ntor3_server_handshake_state_free(relay_state); + tor_free(cm); + tor_free(sm); + dimap_free(private_keys, NULL); +} + +struct testcase_t ntor_v3_tests[] = { + { "testvecs", test_ntor3_testvecs, 0, NULL, NULL, }, + END_OF_TESTCASES, +}; diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c index fa2657f6c2..5503fc69ee 100644 --- a/src/test/test_process_descs.c +++ b/src/test/test_process_descs.c @@ -39,12 +39,20 @@ test_process_descs_versions(void *arg) { "Tor 0.4.1.1-alpha", true }, { "Tor 0.4.1.4-rc", true }, { "Tor 0.4.1.5", true }, + { "Tor 0.4.2.1-alpha", true }, + { "Tor 0.4.2.4-rc", true }, + { "Tor 0.4.2.5", true }, + { "Tor 0.4.3.0-alpha-dev", true }, + { "Tor 0.4.3.8", true }, + { "Tor 0.4.4.9", true }, + { "Tor 0.4.5.5-rc", true }, // new enough to be supported { "Tor 0.3.5.7", false }, { "Tor 0.3.5.8", false }, - { "Tor 0.4.2.1-alpha", false }, - { "Tor 0.4.2.4-rc", false }, - { "Tor 0.4.3.0-alpha-dev", false }, + { "Tor 0.4.5.6", false }, + { "Tor 0.4.6.0-alpha-dev", false }, + { "Tor 0.4.6.5", false }, + { "Tor 0.4.7.0-alpha-dev", false }, // Very far in the future { "Tor 100.100.1.5", false }, }; diff --git a/src/test/test_protover.c b/src/test/test_protover.c index 4b25a86e26..8cc3bcf0e5 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -23,13 +23,6 @@ static void test_protover_parse(void *arg) { (void) arg; -#ifdef HAVE_RUST - /** This test is disabled on rust builds, because it only exists to test - * internal C functions. */ - tt_skip(); - done: - ; -#else /* !defined(HAVE_RUST) */ char *re_encoded = NULL; const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16"; @@ -64,18 +57,12 @@ test_protover_parse(void *arg) SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent)); smartlist_free(elts); tor_free(re_encoded); -#endif /* defined(HAVE_RUST) */ } static void test_protover_parse_fail(void *arg) { (void)arg; -#ifdef HAVE_RUST - /** This test is disabled on rust builds, because it only exists to test - * internal C functions. */ - tt_skip(); -#else smartlist_t *elts; /* random junk */ @@ -108,7 +95,6 @@ test_protover_parse_fail(void *arg) "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); tt_ptr_op(elts, OP_EQ, NULL); -#endif /* defined(HAVE_RUST) */ done: ; } @@ -265,7 +251,7 @@ test_protover_all_supported(void *arg) #endif /* !defined(ALL_BUGS_ARE_FATAL) */ /* Protocol name too long */ -#if !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL) +#if !defined(ALL_BUGS_ARE_FATAL) tor_capture_bugs_(1); tt_assert(protover_all_supported( "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -273,7 +259,7 @@ test_protover_all_supported(void *arg) "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaa=1-65536", &msg)); tor_end_capture_bugs_(); -#endif /* !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL) */ +#endif /* !defined(ALL_BUGS_ARE_FATAL) */ done: tor_end_capture_bugs_(); diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 27e74d5ebf..07c5032933 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -233,6 +233,10 @@ test_pt_protocol(void *arg) handle_proxy_line(line, mp); tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); + strlcpy(line,"CMETHOD-ERROR fakename not supported",sizeof(line)); + handle_proxy_line(line, mp); + tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); + strlcpy(line,"CMETHODS DONE",sizeof(line)); handle_proxy_line(line, mp); tt_assert(mp->conf_state == PT_PROTO_CONFIGURED); diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh deleted file mode 100755 index 804d2ada36..0000000000 --- a/src/test/test_rust.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# Test all Rust crates - -set -e - -export LSAN_OPTIONS=suppressions=${abs_top_srcdir:-../../..}/src/test/rust_supp.txt - -# When testing Cargo we pass a number of very specific linker flags down -# through Cargo. We do not, however, want these flags to affect things like -# build scripts, only the tests that we're compiling. To ensure this happens -# we unconditionally pass `--target` into Cargo, ensuring that `RUSTFLAGS` in -# the environment won't make their way into build scripts. -rustc_host=$(rustc -vV | grep host | sed 's/host: //') - -for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do - if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then - # shellcheck disable=SC2086 - cd "${abs_top_builddir:-../../..}/src/rust" && \ - CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \ - "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \ - --features "test_linking_hack" \ - --target "$rustc_host" \ - ${EXTRA_CARGO_OPTIONS} \ - --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1 - fi -done - -exit $exitcode diff --git a/src/test/test_sandbox.c b/src/test/test_sandbox.c new file mode 100644 index 0000000000..7ec08a3546 --- /dev/null +++ b/src/test/test_sandbox.c @@ -0,0 +1,349 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _LARGEFILE64_SOURCE +/** + * Temporarily required for O_LARGEFILE flag. Needs to be removed + * with the libevent fix. + */ +#define _LARGEFILE64_SOURCE +#endif /* !defined(_LARGEFILE64_SOURCE) */ + +#include "orconfig.h" + +#include "lib/sandbox/sandbox.h" + +#ifdef USE_LIBSECCOMP + +#include <dirent.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#include "core/or/or.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +typedef struct { + sandbox_cfg_t *cfg; + + char *file_ops_allowed; + char *file_ops_blocked; + + char *file_rename_target_allowed; + + char *dir_ops_allowed; + char *dir_ops_blocked; +} sandbox_data_t; + +/* All tests are skipped when coverage support is enabled (see further below) + * as the sandbox interferes with the use of gcov. Prevent a compiler warning + * by omitting these definitions in that case. */ +#ifndef ENABLE_COVERAGE +static void * +setup_sandbox(const struct testcase_t *testcase) +{ + sandbox_data_t *data = tor_malloc_zero(sizeof(*data)); + + (void)testcase; + + /* Establish common file and directory names within the test suite's + * temporary directory. */ + data->file_ops_allowed = tor_strdup(get_fname("file_ops_allowed")); + data->file_ops_blocked = tor_strdup(get_fname("file_ops_blocked")); + + data->file_rename_target_allowed = + tor_strdup(get_fname("file_rename_target_allowed")); + + data->dir_ops_allowed = tor_strdup(get_fname("dir_ops_allowed")); + data->dir_ops_blocked = tor_strdup(get_fname("dir_ops_blocked")); + + /* Create the corresponding filesystem objects. */ + creat(data->file_ops_allowed, S_IRWXU); + creat(data->file_ops_blocked, S_IRWXU); + mkdir(data->dir_ops_allowed, S_IRWXU); + mkdir(data->dir_ops_blocked, S_IRWXU); + + /* Create the sandbox configuration. */ + data->cfg = sandbox_cfg_new(); + + sandbox_cfg_allow_open_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_open_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_chmod_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_chmod_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + sandbox_cfg_allow_chown_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_chown_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_rename(&data->cfg, tor_strdup(data->file_ops_allowed), + tor_strdup(data->file_rename_target_allowed)); + + sandbox_cfg_allow_openat_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_opendir_dirname(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_stat_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_stat_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + /* Activate the sandbox, which will remain in effect until the process + * terminates. */ + sandbox_init(data->cfg); + + return data; +} + +static int +cleanup_sandbox(const struct testcase_t *testcase, void *data_) +{ + sandbox_data_t *data = data_; + + (void)testcase; + + tor_free(data->dir_ops_blocked); + tor_free(data->dir_ops_allowed); + tor_free(data->file_rename_target_allowed); + tor_free(data->file_ops_blocked); + tor_free(data->file_ops_allowed); + + tor_free(data); + + return 1; +} + +static const struct testcase_setup_t sandboxed_testcase_setup = { + .setup_fn = setup_sandbox, + .cleanup_fn = cleanup_sandbox +}; +#endif /* !defined(ENABLE_COVERAGE) */ + +static void +test_sandbox_is_active(void *ignored) +{ + (void)ignored; + + tt_assert(!sandbox_is_active()); + + sandbox_init(sandbox_cfg_new()); + tt_assert(sandbox_is_active()); + + done: + (void)0; +} + +static void +test_sandbox_open_filename(void *arg) +{ + sandbox_data_t *data = arg; + int fd, errsv; + + fd = open(sandbox_intern_string(data->file_ops_allowed), O_RDONLY); + if (fd == -1) + tt_abort_perror("open"); + close(fd); + + /* It might be nice to use sandbox_intern_string() in the line below as well + * (and likewise in the test cases that follow) but this would require + * capturing the warning message it logs, and the mechanism for doing so + * relies on system calls that are normally blocked by the sandbox and may + * vary across architectures. */ + fd = open(data->file_ops_blocked, O_RDONLY); + errsv = errno; + tt_int_op(fd, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + if (fd >= 0) + close(fd); +} + +static void +test_sandbox_chmod_filename(void *arg) +{ + sandbox_data_t *data = arg; + int rc, errsv; + + if (chmod(sandbox_intern_string(data->file_ops_allowed), + S_IRUSR | S_IWUSR) != 0) + tt_abort_perror("chmod"); + + rc = chmod(data->file_ops_blocked, S_IRUSR | S_IWUSR); + errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +static void +test_sandbox_chown_filename(void *arg) +{ + sandbox_data_t *data = arg; + int rc, errsv; + + if (chown(sandbox_intern_string(data->file_ops_allowed), -1, -1) != 0) + tt_abort_perror("chown"); + + rc = chown(data->file_ops_blocked, -1, -1); + errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +static void +test_sandbox_rename_filename(void *arg) +{ + sandbox_data_t *data = arg; + const char *fname_old = sandbox_intern_string(data->file_ops_allowed), + *fname_new = sandbox_intern_string(data->file_rename_target_allowed); + int rc, errsv; + + if (rename(fname_old, fname_new) != 0) + tt_abort_perror("rename"); + + rc = rename(fname_new, fname_old); + errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +static void +test_sandbox_openat_filename(void *arg) +{ + sandbox_data_t *data = arg; + int flags = O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_DIRECTORY | O_CLOEXEC; + int fd, errsv; + + fd = openat(AT_FDCWD, sandbox_intern_string(data->dir_ops_allowed), flags); + if (fd < 0) + tt_abort_perror("openat"); + close(fd); + + fd = openat(AT_FDCWD, data->dir_ops_blocked, flags); + errsv = errno; + tt_int_op(fd, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + if (fd >= 0) + close(fd); +} + +static void +test_sandbox_opendir_dirname(void *arg) +{ + sandbox_data_t *data = arg; + DIR *dir; + int errsv; + + dir = opendir(sandbox_intern_string(data->dir_ops_allowed)); + if (dir == NULL) + tt_abort_perror("opendir"); + closedir(dir); + + dir = opendir(data->dir_ops_blocked); + errsv = errno; + tt_ptr_op(dir, OP_EQ, NULL); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + if (dir) + closedir(dir); +} + +static void +test_sandbox_stat_filename(void *arg) +{ + sandbox_data_t *data = arg; + struct stat st; + + if (stat(sandbox_intern_string(data->file_ops_allowed), &st) != 0) + tt_abort_perror("stat"); + + int rc = stat(data->file_ops_blocked, &st); + int errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +#define SANDBOX_TEST_SKIPPED(name) \ + { #name, test_sandbox_ ## name, TT_SKIP, NULL, NULL } + +/* Skip all tests when coverage support is enabled, as the sandbox interferes + * with gcov and prevents it from producing any results. */ +#ifdef ENABLE_COVERAGE +#define SANDBOX_TEST(name, flags) SANDBOX_TEST_SKIPPED(name) +#define SANDBOX_TEST_IN_SANDBOX(name) SANDBOX_TEST_SKIPPED(name) +#else +#define SANDBOX_TEST(name, flags) \ + { #name, test_sandbox_ ## name, flags, NULL, NULL } +#define SANDBOX_TEST_IN_SANDBOX(name) \ + { #name, test_sandbox_ ## name, TT_FORK, &sandboxed_testcase_setup, NULL } +#endif /* defined(ENABLE_COVERAGE) */ + +struct testcase_t sandbox_tests[] = { + SANDBOX_TEST(is_active, TT_FORK), + +/* When Tor is built with fragile compiler-hardening the sandbox is unable to + * filter requests to open files or directories (on systems where glibc uses + * the "open" system call to provide this functionality), as doing so would + * interfere with the address sanitizer as it retrieves information about the + * running process via the filesystem. Skip these tests in that case as the + * corresponding functions are likely to have no effect and this will cause the + * tests to fail. */ +#ifdef ENABLE_FRAGILE_HARDENING + SANDBOX_TEST_SKIPPED(open_filename), + SANDBOX_TEST_SKIPPED(opendir_dirname), +#else + SANDBOX_TEST_IN_SANDBOX(open_filename), + SANDBOX_TEST_IN_SANDBOX(opendir_dirname), +#endif /* defined(ENABLE_FRAGILE_HARDENING) */ + + SANDBOX_TEST_IN_SANDBOX(openat_filename), + SANDBOX_TEST_IN_SANDBOX(chmod_filename), + SANDBOX_TEST_IN_SANDBOX(chown_filename), + SANDBOX_TEST_IN_SANDBOX(rename_filename), + +/* Currently the sandbox is unable to filter stat() calls on systems where + * glibc implements this function using either of the legacy "stat" or "stat64" + * system calls, or (in glibc version 2.33 and later) either of the newer + * "newfstatat" or "statx" syscalls. + * + * Skip testing sandbox_cfg_allow_stat_filename() if it seems the likely the + * function will have no effect and the test will therefore not succeed. */ +#if !defined(__NR_stat) && !defined(__NR_stat64) && !defined(__NR_newfstatat) \ + && !(defined(__i386__) && defined(__NR_statx)) + SANDBOX_TEST_IN_SANDBOX(stat_filename), +#else + SANDBOX_TEST_SKIPPED(stat_filename), +#endif + END_OF_TESTCASES +}; + +#endif /* defined(USE_SECCOMP) */ diff --git a/src/test/test_stats.c b/src/test/test_stats.c index af035ae54c..c7e0c10845 100644 --- a/src/test/test_stats.c +++ b/src/test/test_stats.c @@ -51,7 +51,7 @@ #include "feature/stats/bw_array_st.h" #include "feature/relay/router.h" -#include "event2/dns.h" +#include <event2/dns.h> /** Run unit tests for some stats code. */ static void @@ -721,7 +721,7 @@ test_overload_stats(void *arg) stats_str = rep_hist_get_overload_stats_lines(); tt_assert(!stats_str); - /* Note a overload */ + /* Note an overload */ rep_hist_note_overload(OVERLOAD_GENERAL); /* Move the time forward one hour */ @@ -742,7 +742,7 @@ test_overload_stats(void *arg) /* Now the time should be 2002-01-07 00:00:00 */ - /* Note a overload */ + /* Note an overload */ rep_hist_note_overload(OVERLOAD_GENERAL); stats_str = rep_hist_get_overload_general_line(); diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c index 4c5f3a3270..457b0fa796 100644 --- a/src/test/test_voting_flags.c +++ b/src/test/test_voting_flags.c @@ -62,7 +62,8 @@ check_result(flag_vote_test_cfg_t *c) bool result = false; routerstatus_t rs; memset(&rs, 0, sizeof(rs)); - dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, + 0, 0); tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); diff --git a/src/tools/include.am b/src/tools/include.am index 6daa27f6de..86cd0acbfd 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -10,7 +10,6 @@ src_tools_tor_resolve_LDADD = \ src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ - $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@ if COVERAGE_ENABLED @@ -34,7 +33,6 @@ src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) src_tools_tor_gencert_LDADD = \ $(TOR_CRYPTO_LIBS) \ $(TOR_UTIL_LIBS) \ - $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ endif diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 09ff8df4ab..ad52fdaa22 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -253,7 +253,7 @@ build_socks_resolve_request(uint8_t **out, } static void -onion_warning(const char *hostname) +onion_hs_warning(const char *hostname) { log_warn(LD_NET, "%s is a hidden service; those don't have IP addresses. " @@ -264,6 +264,15 @@ onion_warning(const char *hostname) hostname); } +static void +onion_exit_warning(const char *hostname) +{ + log_warn(LD_NET, + "%s is a link pointing to an exit node; however, .exit domains" + "have been long defunct and are not valid anymore.", + hostname); +} + /** Given a <b>len</b>-byte SOCKS4a response in <b>response</b>, set * *<b>addr_out</b> to the address it contains (in host order). * Return 0 on success, -1 on error. @@ -306,9 +315,15 @@ parse_socks4a_resolve_response(const char *hostname, if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); result = -1; goto cleanup; } + + if (!strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); + result = -1; goto cleanup; + } + result = -1; goto cleanup; } @@ -493,7 +508,11 @@ do_resolve(const char *hostname, (unsigned)reply_field, socks5_reason_to_string(reply_field)); if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); + } + + if (reply_field == 4 && !strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); } socks5_server_reply_free(reply); diff --git a/src/trunnel/flow_control_cells.c b/src/trunnel/flow_control_cells.c new file mode 100644 index 0000000000..df44756d6b --- /dev/null +++ b/src/trunnel/flow_control_cells.c @@ -0,0 +1,382 @@ +/* flow_control_cells.c -- generated by Trunnel v1.5.3. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "flow_control_cells.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int flowcontrolcells_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || flowcontrolcells_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +xoff_cell_t * +xoff_cell_new(void) +{ + xoff_cell_t *val = trunnel_calloc(1, sizeof(xoff_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +xoff_cell_clear(xoff_cell_t *obj) +{ + (void) obj; +} + +void +xoff_cell_free(xoff_cell_t *obj) +{ + if (obj == NULL) + return; + xoff_cell_clear(obj); + trunnel_memwipe(obj, sizeof(xoff_cell_t)); + trunnel_free_(obj); +} + +uint8_t +xoff_cell_get_version(const xoff_cell_t *inp) +{ + return inp->version; +} +int +xoff_cell_set_version(xoff_cell_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +const char * +xoff_cell_check(const xoff_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +xoff_cell_encoded_len(const xoff_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != xoff_cell_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + return result; +} +int +xoff_cell_clear_errors(xoff_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +xoff_cell_encode(uint8_t *output, const size_t avail, const xoff_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = xoff_cell_encoded_len(obj); +#endif + + if (NULL != (msg = xoff_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As xoff_cell_parse(), but do not allocate the output object. + */ +static ssize_t +xoff_cell_parse_into(xoff_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +xoff_cell_parse(xoff_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = xoff_cell_new(); + if (NULL == *output) + return -1; + result = xoff_cell_parse_into(*output, input, len_in); + if (result < 0) { + xoff_cell_free(*output); + *output = NULL; + } + return result; +} +xon_cell_t * +xon_cell_new(void) +{ + xon_cell_t *val = trunnel_calloc(1, sizeof(xon_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +xon_cell_clear(xon_cell_t *obj) +{ + (void) obj; +} + +void +xon_cell_free(xon_cell_t *obj) +{ + if (obj == NULL) + return; + xon_cell_clear(obj); + trunnel_memwipe(obj, sizeof(xon_cell_t)); + trunnel_free_(obj); +} + +uint8_t +xon_cell_get_version(const xon_cell_t *inp) +{ + return inp->version; +} +int +xon_cell_set_version(xon_cell_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint32_t +xon_cell_get_kbps_ewma(const xon_cell_t *inp) +{ + return inp->kbps_ewma; +} +int +xon_cell_set_kbps_ewma(xon_cell_t *inp, uint32_t val) +{ + inp->kbps_ewma = val; + return 0; +} +const char * +xon_cell_check(const xon_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +xon_cell_encoded_len(const xon_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != xon_cell_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u32 kbps_ewma */ + result += 4; + return result; +} +int +xon_cell_clear_errors(xon_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +xon_cell_encode(uint8_t *output, const size_t avail, const xon_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = xon_cell_encoded_len(obj); +#endif + + if (NULL != (msg = xon_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u32 kbps_ewma */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->kbps_ewma)); + written += 4; ptr += 4; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As xon_cell_parse(), but do not allocate the output object. + */ +static ssize_t +xon_cell_parse_into(xon_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u32 kbps_ewma */ + CHECK_REMAINING(4, truncated); + obj->kbps_ewma = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +xon_cell_parse(xon_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = xon_cell_new(); + if (NULL == *output) + return -1; + result = xon_cell_parse_into(*output, input, len_in); + if (result < 0) { + xon_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/flow_control_cells.h b/src/trunnel/flow_control_cells.h new file mode 100644 index 0000000000..b8108b9a24 --- /dev/null +++ b/src/trunnel/flow_control_cells.h @@ -0,0 +1,120 @@ +/* flow_control_cells.h -- generated by Trunnel v1.5.3. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_FLOW_CONTROL_CELLS_H +#define TRUNNEL_FLOW_CONTROL_CELLS_H + +#include <stdint.h> +#include "trunnel.h" + +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_XOFF_CELL) +struct xoff_cell_st { + uint8_t version; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct xoff_cell_st xoff_cell_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_XON_CELL) +struct xon_cell_st { + uint8_t version; + uint32_t kbps_ewma; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct xon_cell_st xon_cell_t; +/** Return a newly allocated xoff_cell with all elements set to zero. + */ +xoff_cell_t *xoff_cell_new(void); +/** Release all storage held by the xoff_cell in 'victim'. (Do nothing + * if 'victim' is NULL.) + */ +void xoff_cell_free(xoff_cell_t *victim); +/** Try to parse a xoff_cell from the buffer in 'input', using up to + * 'len_in' bytes from the input buffer. On success, return the number + * of bytes consumed and set *output to the newly allocated + * xoff_cell_t. On failure, return -2 if the input appears truncated, + * and -1 if the input is otherwise invalid. + */ +ssize_t xoff_cell_parse(xoff_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * xoff_cell in 'obj'. On failure, return a negative value. Note that + * this value may be an overestimate, and can even be an underestimate + * for certain unencodeable objects. + */ +ssize_t xoff_cell_encoded_len(const xoff_cell_t *obj); +/** Try to encode the xoff_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t xoff_cell_encode(uint8_t *output, size_t avail, const xoff_cell_t *input); +/** Check whether the internal state of the xoff_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *xoff_cell_check(const xoff_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int xoff_cell_clear_errors(xoff_cell_t *obj); +/** Return the value of the version field of the xoff_cell_t in 'inp' + */ +uint8_t xoff_cell_get_version(const xoff_cell_t *inp); +/** Set the value of the version field of the xoff_cell_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int xoff_cell_set_version(xoff_cell_t *inp, uint8_t val); +/** Return a newly allocated xon_cell with all elements set to zero. + */ +xon_cell_t *xon_cell_new(void); +/** Release all storage held by the xon_cell in 'victim'. (Do nothing + * if 'victim' is NULL.) + */ +void xon_cell_free(xon_cell_t *victim); +/** Try to parse a xon_cell from the buffer in 'input', using up to + * 'len_in' bytes from the input buffer. On success, return the number + * of bytes consumed and set *output to the newly allocated + * xon_cell_t. On failure, return -2 if the input appears truncated, + * and -1 if the input is otherwise invalid. + */ +ssize_t xon_cell_parse(xon_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * xon_cell in 'obj'. On failure, return a negative value. Note that + * this value may be an overestimate, and can even be an underestimate + * for certain unencodeable objects. + */ +ssize_t xon_cell_encoded_len(const xon_cell_t *obj); +/** Try to encode the xon_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t xon_cell_encode(uint8_t *output, size_t avail, const xon_cell_t *input); +/** Check whether the internal state of the xon_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *xon_cell_check(const xon_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int xon_cell_clear_errors(xon_cell_t *obj); +/** Return the value of the version field of the xon_cell_t in 'inp' + */ +uint8_t xon_cell_get_version(const xon_cell_t *inp); +/** Set the value of the version field of the xon_cell_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int xon_cell_set_version(xon_cell_t *inp, uint8_t val); +/** Return the value of the kbps_ewma field of the xon_cell_t in 'inp' + */ +uint32_t xon_cell_get_kbps_ewma(const xon_cell_t *inp); +/** Set the value of the kbps_ewma field of the xon_cell_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int xon_cell_set_kbps_ewma(xon_cell_t *inp, uint32_t val); + + +#endif diff --git a/src/trunnel/flow_control_cells.trunnel b/src/trunnel/flow_control_cells.trunnel new file mode 100644 index 0000000000..9d07b568a9 --- /dev/null +++ b/src/trunnel/flow_control_cells.trunnel @@ -0,0 +1,20 @@ +/* This file contains the xon and xoff cell definitions, for flow control. */ + +/* xoff cell definition. Tells the other endpoint to stop sending, because + * we have too much data queued for this stream. */ +struct xoff_cell { + /* Version field. */ + u8 version IN [0x00]; +} + +/* xon cell declaration. Tells the other endpoint to resume sending and/or + * update its sending rate on this stream based on advisory information. */ +struct xon_cell { + /* Version field. */ + u8 version IN [0x00]; + + /* Advisory field: The ewma rate of socket drain we have seen so far + * on this stream, in kilobytes/sec (1000 bytes/sec). May be zero, + * which means no rate advice. */ + u32 kbps_ewma; +} diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 6c3a5ff06b..00a96536f1 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -12,6 +12,7 @@ TRUNNELINPUTS = \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ src/trunnel/sendme_cell.trunnel \ + src/trunnel/flow_control_cells.trunnel \ src/trunnel/socks5.trunnel \ src/trunnel/circpad_negotiation.trunnel @@ -26,6 +27,7 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ src/trunnel/sendme_cell.c \ + src/trunnel/flow_control_cells.c \ src/trunnel/socks5.c \ src/trunnel/netinfo.c \ src/trunnel/circpad_negotiation.c @@ -43,6 +45,7 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ src/trunnel/sendme_cell.h \ + src/trunnel/flow_control_cells.h \ src/trunnel/socks5.h \ src/trunnel/netinfo.h \ src/trunnel/circpad_negotiation.h diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 141a603e2e..036314d39a 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -217,7 +217,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.6.9-dev" +#define VERSION "0.4.7.3-alpha-dev" #define HAVE_STRUCT_SOCKADDR_IN6 #define HAVE_STRUCT_IN6_ADDR |