diff options
412 files changed, 18571 insertions, 8255 deletions
diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000000..34e1c8b846 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,91 @@ +version: 1.0.{build} + +clone_depth: 50 + +environment: + compiler: mingw + + matrix: + - target: i686-w64-mingw32 + compiler_path: mingw32 + mingw_prefix: mingw-w64-i686 + hardening: + - target: x86_64-w64-mingw32 + compiler_path: mingw64 + mingw_prefix: mingw-w64-x86_64 + # hardening doesn't work with mingw-w64-x86_64-gcc, because it's gcc 8 + hardening: --disable-gcc-hardening + +install: +- ps: >- + Function Execute-Command ($commandPath) + { + & $commandPath $args 2>&1 + if ( $LastExitCode -ne 0 ) { + $host.SetShouldExit( $LastExitCode ) + } + } + Function Execute-Bash () + { + Execute-Command 'c:\msys64\usr\bin\bash' '-e' '-c' $args + } + <# mingw packages start with ${env:mingw_prefix} + # unprefixed packages are from MSYS2, which is like Cygwin. Avoid them. + # + # Use pacman --debug to show package downloads and install locations + #> + Execute-Command "C:\msys64\usr\bin\pacman" -Sy --verbose --needed --noconfirm ${env:mingw_prefix}-libevent ${env:mingw_prefix}-pkg-config ${env:mingw_prefix}-xz ${env:mingw_prefix}-zstd ; + +build_script: +- ps: >- + if ($env:compiler -eq "mingw") { + <# use the MSYS2 compiler and user binaries to build and install #> + $oldpath = ${env:Path} -split ';' + $buildpath = @("C:\msys64\${env:compiler_path}\bin", "C:\msys64\usr\bin") + $oldpath + $env:Path = @($buildpath) -join ';' + $env:build = @("${env:APPVEYOR_BUILD_FOLDER}", $env:target) -join '\' + Set-Location "${env:APPVEYOR_BUILD_FOLDER}" + Execute-Bash 'autoreconf -i' + mkdir "${env:build}" + Set-Location "${env:build}" + Execute-Bash "which ${env:target}-gcc" + Execute-Bash "${env:target}-gcc --version" + <# compile for mingw + # mingw zstd doesn't come with a pkg-config file, so we manually + # configure its flags. liblzma just works. + #> + Execute-Bash "ZSTD_CFLAGS='-L/${env:compiler_path}/include' ZSTD_LIBS='-L/${env:compiler_path}/lib -lzstd' ../configure --prefix=/${env:compiler_path} --build=${env:target} --host=${env:target} --with-openssl-dir=/${env:compiler_path} --disable-asciidoc --enable-fatal-warnings ${env:hardening}" + Execute-Bash "V=1 make -j2" + Execute-Bash "V=1 make -j2 install" + } + +test_script: +- ps: >- + if ($env:compiler -eq "mingw") { + <# use the MSYS2 compiler binaries to make check #> + $oldpath = ${env:Path} -split ';' + $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath + $env:Path = $buildpath -join ';' + Set-Location "${env:build}" + Execute-Bash "VERBOSE=1 make -j2 check" + } + +on_finish: +- ps: >- + if ($env:compiler -eq "mingw") { + <# use the MSYS2 user binaries to archive failures #> + $oldpath = ${env:Path} -split ';' + $buildpath = @("C:\msys64\usr\bin") + $oldpath + $env:Path = @($buildpath) -join ';' + Set-Location "${env:build}" + <# store logs as appveyor artifacts: see the artifacts tab #> + Execute-Bash "7z a logs.zip config.log || true" + Execute-Bash "7z a logs.zip test-suite.log || true" + Execute-Bash "appveyor PushArtifact logs.zip || true" + Execute-Bash "tail -1000 config.log || true" + Execute-Bash "cat test-suite.log || true" + } + +# notify the IRC channel of any failures +on_failure: +- cmd: C:\Python27\python.exe %APPVEYOR_BUILD_FOLDER%\scripts\test\appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure diff --git a/.gitignore b/.gitignore index c00fbe97e1..b3b13694af 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ uptime-*.json /autom4te.cache /build-stamp /compile +/config.rust /configure /Doxyfile /orconfig.h @@ -55,6 +56,7 @@ uptime-*.json /config.guess /config.sub /conftest* +/link_rust.sh /micro-revision.* /patch-stamp /stamp-h diff --git a/.travis.yml b/.travis.yml index c0bac19003..56d7dc0737 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,9 @@ matrix: ## Check rust online with distcheck, to make sure we remove rust products ## But without hardening (see above) - env: DISTCHECK="yes" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode" HARDENING_OPTIONS="" + ## Check disable module dirauth with and without rust + - env: MODULES_OPTIONS="--disable-module-dirauth" RUST_OPTIONS="--enable-rust" TOR_RUST_DEPENDENCIES=true HARDENING_OPTIONS="" + - env: MODULES_OPTIONS="--disable-module-dirauth" ## Uncomment to allow the build to report success (with non-required ## sub-builds continuing to run) if all required sub-builds have @@ -192,7 +195,7 @@ install: script: - ./autogen.sh - - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules" + - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules" - echo "Configure flags are $CONFIGURE_FLAGS" - ./configure $CONFIGURE_FLAGS ## We run `make check` because that's what https://jenkins.torproject.org does. diff --git a/CONTRIBUTING b/CONTRIBUTING new file mode 100644 index 0000000000..3569f45a88 --- /dev/null +++ b/CONTRIBUTING @@ -0,0 +1,39 @@ +Contributing to Tor +------------------- + +### Getting started + +Welcome! + +We have a bunch of documentation about how to develop Tor in the +doc/HACKING/ directory. We recommend that you start with +doc/HACKING/README.1st.md , and then go from there. It will tell +you how to find your way around the source code, how to get +involved with the Tor community, how to write patches, and much +more! + +You don't have to be a C developer to help with Tor: have a look +at https://www.torproject.org/getinvolved/volunteer ! + +The Tor Project is committed to fostering a inclusive community +where people feel safe to engage, share their points of view, and +participate. For the latest version of our Code of Conduct, please +see + +https://gitweb.torproject.org/community/policies.git/plain/code_of_conduct.txt + + + +### License issues + +Tor is distributed under the license terms in the LICENSE -- in +brief, the "3-clause BSD license". If you send us code to +distribute with Tor, it needs to be code that we can distribute +under those terms. Please don't send us patches unless you agree +to allow this. + +Some compatible licenses include: + + - 3-clause BSD + - 2-clause BSD + - CC0 Public Domain Dedication @@ -1,3 +1,1512 @@ +Changes in version 0.3.3.7 - 2018-06-12 + Tor 0.3.3.7 backports several changes from the 0.3.4.x series, including + fixes for bugs affecting compatibility and stability. + + o Directory authority changes: + - Add an IPv6 address for the "dannenberg" directory authority. + Closes ticket 26343. + + o Minor features (geoip): + - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2 + Country database. Closes ticket 26351. + + o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha): + - Work around a change in OpenSSL 1.1.1 where return values that + would previously indicate "no password" now indicate an empty + password. Without this workaround, Tor instances running with + OpenSSL 1.1.1 would accept descriptors that other Tor instances + would reject. Fixes bug 26116; bugfix on 0.2.5.16. + + o Minor bugfixes (compilation, backport from 0.3.4.2-alpha): + - Silence unused-const-variable warnings in zstd.h with some GCC + versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (controller, backport from 0.3.4.2-alpha): + - Improve accuracy of the BUILDTIMEOUT_SET control port event's + TIMEOUT_RATE and CLOSE_RATE fields. (We were previously + miscounting the total number of circuits for these field values.) + Fixes bug 26121; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (hardening, backport from 0.3.4.2-alpha): + - Prevent a possible out-of-bounds smartlist read in + protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (path selection, backport from 0.3.4.1-alpha): + - Only select relays when they have the descriptors we prefer to use + for them. This change fixes a bug where we could select a relay + because it had _some_ descriptor, but reject it later with a + nonfatal assertion error because it didn't have the exact one we + wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha. + + +Changes in version 0.3.4.2-alpha - 2018-06-12 + Tor 0.3.4.2-alpha fixes several minor bugs in the previous alpha + release, and forward-ports an authority-only security fix from 0.3.3.6. + + o Directory authority changes: + - Add an IPv6 address for the "dannenberg" directory authority. + Closes ticket 26343. + + o Major bugfixes (security, directory authority, denial-of-service, also in 0.3.3.6): + - Fix a bug that could have allowed an attacker to force a directory + authority to use up all its RAM by passing it a maliciously + crafted protocol versions string. Fixes bug 25517; bugfix on + 0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005. + + o Minor features (continuous integration): + - Add the necessary configuration files for continuous integration + testing on Windows, via the Appveyor platform. Closes ticket + 25549. Patches from Marcin Cieślak and Isis Lovecruft. + + o Minor features (geoip): + - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2 + Country database. Closes ticket 26351. + + o Minor bugfixes (compatibility, openssl): + - Work around a change in OpenSSL 1.1.1 where return values that + would previously indicate "no password" now indicate an empty + password. Without this workaround, Tor instances running with + OpenSSL 1.1.1 would accept descriptors that other Tor instances + would reject. Fixes bug 26116; bugfix on 0.2.5.16. + + o Minor bugfixes (compilation): + - Silence unused-const-variable warnings in zstd.h with some GCC + versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha. + - Fix compilation when using OpenSSL 1.1.0 with the "no-deprecated" + flag enabled. Fixes bug 26156; bugfix on 0.3.4.1-alpha. + - Avoid a compiler warning when casting the return value of + smartlist_len() to double with DEBUG_SMARTLIST enabled. Fixes bug + 26283; bugfix on 0.2.4.10-alpha. + + o Minor bugfixes (control port): + - Do not count 0-length RELAY_COMMAND_DATA cells as valid data in + CIRC_BW events. Previously, such cells were counted entirely in + the OVERHEAD field. Now they are not. Fixes bug 26259; bugfix + on 0.3.4.1-alpha. + + o Minor bugfixes (controller): + - Improve accuracy of the BUILDTIMEOUT_SET control port event's + TIMEOUT_RATE and CLOSE_RATE fields. (We were previously + miscounting the total number of circuits for these field values.) + Fixes bug 26121; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (hardening): + - Prevent a possible out-of-bounds smartlist read in + protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (onion services): + - Fix a bug that blocked the creation of ephemeral v3 onion + services. Fixes bug 25939; bugfix on 0.3.4.1-alpha. + + o Minor bugfixes (test coverage tools): + - Update our "cov-diff" script to handle output from the latest + version of gcov, and to remove extraneous timestamp information + from its output. Fixes bugs 26101 and 26102; bugfix + on 0.2.5.1-alpha. + + +Changes in version 0.3.3.6 - 2018-05-22 + Tor 0.3.3.6 is the first stable release in the 0.3.3 series. It + backports several important fixes from the 0.3.4.1-alpha. + + The Tor 0.3.3 series includes controller support and other + improvements for v3 onion services, official support for embedding Tor + within other applications, and our first non-trivial module written in + the Rust programming language. (Rust is still not enabled by default + when building Tor.) And as usual, there are numerous other smaller + bugfixes, features, and improvements. + + Below are the changes since 0.3.3.5-rc. For a list of all changes + since 0.3.2.10, see the ReleaseNotes file. + + o Major bugfixes (directory authorities, security, backport from 0.3.4.1-alpha): + - When directory authorities read a zero-byte bandwidth file, they + would previously log a warning with the contents of an + uninitialised buffer. They now log a warning about the empty file + instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha. + + o Major bugfixes (security, directory authority, denial-of-service): + - Fix a bug that could have allowed an attacker to force a directory + authority to use up all its RAM by passing it a maliciously + crafted protocol versions string. Fixes bug 25517; bugfix on + 0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005. + + o Major bugfixes (crash, backport from 0.3.4.1-alpha): + - Avoid a rare assertion failure in the circuit build timeout code + if we fail to allow any circuits to actually complete. Fixes bug + 25733; bugfix on 0.2.2.2-alpha. + + o Major bugfixes (directory authorities, backport from 0.3.4.1-alpha): + - Avoid a crash when testing router reachability on a router that + could have an ed25519 ID, but which does not. Fixes bug 25415; + bugfix on 0.3.3.2-alpha. + + o Major bugfixes (onion service, backport from 0.3.4.1-alpha): + - Correctly detect when onion services get disabled after HUP. Fixes + bug 25761; bugfix on 0.3.2.1. + + o Major bugfixes (relay, denial of service, backport from 0.3.4.1-alpha): + - Impose a limit on circuit cell queue size. The limit can be + controlled by a consensus parameter. Fixes bug 25226; bugfix + on 0.2.4.14-alpha. + + o Minor features (compatibility, backport from 0.3.4.1-alpha): + - Avoid some compilation warnings with recent versions of LibreSSL. + Closes ticket 26006. + + o Minor features (continuous integration, backport from 0.3.4.1-alpha): + - Our .travis.yml configuration now includes support for testing the + results of "make distcheck". (It's not uncommon for "make check" + to pass but "make distcheck" to fail.) Closes ticket 25814. + - Our Travis CI configuration now integrates with the Coveralls + coverage analysis tool. Closes ticket 25818. + + o Minor features (geoip): + - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country + database. Closes ticket 26104. + + o Minor bugfixes (client, backport from 0.3.4.1-alpha): + - Don't consider Tor running as a client if the ControlPort is open, + but no actual client ports are open. Fixes bug 26062; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha): + - Upon receiving a malformed connected cell, stop processing the + cell immediately. Previously we would mark the connection for + close, but continue processing the cell as if the connection were + open. Fixes bug 26072; bugfix on 0.2.4.7-alpha. + + o Minor bugfixes (documentation, backport from 0.3.4.1-alpha): + - Stop saying in the manual that clients cache ipv4 dns answers from + exit relays. We haven't used them since 0.2.6.3-alpha, and in + ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but + we forgot to say so in the man page. Fixes bug 26052; bugfix + on 0.3.2.6-alpha. + + o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha): + - Allow the nanosleep() system call, which glibc uses to implement + sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha. + + o Minor bugfixes (onion service, backport from 0.3.4.1-alpha): + - Fix a memory leak when a v3 onion service is configured and gets a + SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha. + - When parsing the descriptor signature, look for the token plus an + extra white-space at the end. This is more correct but also will + allow us to support new fields that might start with "signature". + Fixes bug 26069; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha): + - Avoid a crash when running with DirPort set but ORPort tuned off. + Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha. + + o Documentation (backport from 0.3.4.1-alpha): + - Correct an IPv6 error in the documentation for ExitPolicy. Closes + ticket 25857. Patch from "CTassisF". + + +Changes in version 0.3.4.1-alpha - 2018-05-17 + Tor 0.3.4.1-alpha is the first release in the 0.3.4.x series. It + includes refactoring to begin reducing Tor's binary size and idle CPU + usage on mobile, along with prep work for new bandwidth scanners, + improvements to the experimental "vanguards" feature, and numerous + other small features and bugfixes. + + o New system requirements: + - Tor no longer tries to support old operating systems without + mmap() or some local equivalent. Apparently, compilation on such + systems has been broken for some time, without anybody noticing or + complaining. Closes ticket 25398. + + o Major feature (directory authority, modularization): + - The directory authority subsystem has been modularized. The code + is now located in src/or/dirauth/, and is compiled in by default. + To disable the module, the configure option + --disable-module-dirauth has been added. This module may be + disabled by default in some future release. Closes ticket 25610. + + o Major features (main loop, CPU usage): + - When Tor is disabled (via DisableNetwork or via hibernation), it + no longer needs to run any per-second events. This change should + make it easier for mobile applications to disable Tor while the + device is sleeping, or Tor is not running. Closes ticket 26063. + - Tor no longer enables all of its periodic events by default. + Previously, Tor would enable all possible main loop events, + regardless of whether it needed them. Furthermore, many of these + events are now disabled with Tor is hibernating or DisableNetwork + is set. This is a big step towards reducing client CPU usage by + reducing the amount of wake-ups the daemon does. Closes ticket + 25376 and 25762. + - The bandwidth-limitation logic has been refactored so that + bandwidth calculations are performed on-demand, rather than every + TokenBucketRefillInterval milliseconds. This change should improve + the granularity of our bandwidth calculations, and limit the + number of times that the Tor process needs to wake up when it is + idle. Closes ticket 25373. + - Move responsibility for many operations from a once-per-second + callback to a callback that is only scheduled as needed. Moving + this functionality has allowed us to disable the callback when + Tor's network is disabled. Once enough items are removed from our + once-per-second callback, we can eliminate it entirely to conserve + CPU when idle. The functionality removed includes: closing + connections, circuits, and channels (ticket 25932); consensus + voting (25937); flushing log callbacks (25951); honoring delayed + SIGNEWNYM requests (25949); rescanning the consensus cache + (25931); saving the state file to disk (25948); warning relay + operators about unreachable ports (25952); and keeping track of + Tor's uptime (26009). + + o Major bugfixes (directory authorities, security): + - When directory authorities read a zero-byte bandwidth file, they + would previously log a warning with the contents of an + uninitialised buffer. They now log a warning about the empty file + instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha. + + o Major bugfixes (crash): + - Avoid a rare assertion failure in the circuit build timeout code + if we fail to allow any circuits to actually complete. Fixes bug + 25733; bugfix on 0.2.2.2-alpha. + + o Major bugfixes (directory authority): + - Avoid a crash when testing router reachability on a router that + could have an ed25519 ID, but which does not. Fixes bug 25415; + bugfix on 0.3.3.2-alpha. + + o Major bugfixes (onion service): + - Correctly detect when onion services get disabled after HUP. Fixes + bug 25761; bugfix on 0.3.2.1. + + o Major bugfixes (protover, voting): + - Revise Rust implementation of protover to use a more memory- + efficient voting algorithm and corresponding data structures, thus + avoiding a potential (but small impact) DoS attack where specially + crafted protocol strings would expand to several potential + megabytes in memory. In the process, several portions of code were + revised to be methods on new, custom types, rather than functions + taking interchangeable types, thus increasing type safety of the + module. Custom error types and handling were added as well, in + order to facilitate better error dismissal/handling in outside + crates and avoid mistakenly passing an internal error string to C + over the FFI boundary. Many tests were added, and some previous + differences between the C and Rust implementations have been + remedied. Fixes bug 24031; bugfix on 0.3.3.1-alpha. + + o Major bugfixes (relay, denial of service): + - Impose a limit on circuit cell queue size. The limit can be + controlled by a consensus parameter. Fixes bug 25226; bugfix + on 0.2.4.14-alpha. + + o Minor features (accounting): + - When Tor becomes dormant, it now uses a scheduled event to wake up + at the right time. Previously, we would use the per-second timer + to check whether to wake up, but we no longer have any per-second + timers enabled when the network is disabled. Closes ticket 26064. + + o Minor features (code quality): + - Add optional spell-checking for the Tor codebase, using the + "misspell" program. To use this feature, run "make check-typos". + Closes ticket 25024. + + o Minor features (compatibility): + - Tor now detects versions of OpenSSL 1.1.0 and later compiled with + the no-deprecated option, and builds correctly with them. Closes + tickets 19429, 19981, and 25353. + - Avoid some compilation warnings with recent versions of LibreSSL. + Closes ticket 26006. + + o Minor features (compression, zstd): + - When running with zstd, Tor now considers using advanced functions + that the zstd maintainers have labeled as potentially unstable. To + prevent breakage, Tor will only use this functionality when the + runtime version of the zstd library matches the version with which + Tor was compiled. Closes ticket 25162. + + o Minor features (configuration): + - The "DownloadSchedule" options have been renamed to end with + "DownloadInitialDelay". The old names are still allowed, but will + produce a warning. Comma-separated lists are still permitted for + these options, but all values after the first are ignored (as they + have been since 0.2.9). Closes ticket 23354. + + o Minor features (continuous integration): + - Our .travis.yml configuration now includes support for testing the + results of "make distcheck". (It's not uncommon for "make check" + to pass but "make distcheck" to fail.) Closes ticket 25814. + - Our Travis CI configuration now integrates with the Coveralls + coverage analysis tool. Closes ticket 25818. + + o Minor features (control port): + - Introduce GETINFO "current-time/{local,utc}" to return the local + and UTC times respectively in ISO format. This helps a controller + like Tor Browser detect a time-related error. Closes ticket 25511. + Patch by Neel Chauhan. + - Introduce new fields to the CIRC_BW event. There are two new + fields in each of the read and written directions. The DELIVERED + fields report the total valid data on the circuit, as measured by + the payload sizes of verified and error-checked relay command + cells. The OVERHEAD fields report the total unused bytes in each + of these cells. Closes ticket 25903. + + o Minor features (directory authority): + - Directory authorities now open their key-pinning files as O_SYNC, + to limit their chances of accidentally writing partial lines. + Closes ticket 23909. + + o Minor features (directory authority, forward compatibility): + - Make the lines of the measured bandwidth file able to contain + their entries in any order. Previously, the node_id entry needed + to come first. Closes ticket 26004. + + o Minor features (entry guards): + - Introduce a new torrc option NumPrimaryGuards for controlling the + number of primary guards. Closes ticket 25843. + + o Minor features (geoip): + - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country + database. Closes ticket 26104. + + o Minor features (performance): + - Avoid a needless call to malloc() when processing an incoming + relay cell. Closes ticket 24914. + - Make our timing-wheel code run a tiny bit faster on 32-bit + platforms, by preferring 32-bit math to 64-bit. Closes + ticket 24688. + - Avoid a needless malloc()/free() pair every time we handle an ntor + handshake. Closes ticket 25150. + + o Minor features (testing): + - Add a unit test for voting_schedule_get_start_of_next_interval(). + Closes ticket 26014, and helps make unit test coverage + more deterministic. + - A new unittests module specifically for testing the functions in + the (new-ish) bridges.c module has been created with new + unittests, raising the code coverage percentages. Closes 25425. + - We now have improved testing for addressmap_get_virtual_address() + function. This should improve our test coverage, and make our test + coverage more deterministic. Closes ticket 25993. + + o Minor features (timekeeping, circuit scheduling): + - When keeping track of how busy each circuit have been recently on + a given connection, use coarse-grained monotonic timers rather + than gettimeofday(). This change should marginally increase + accuracy and performance. Implements part of ticket 25927. + + o Minor bugfixes (bandwidth management): + - Consider ourselves "low on write bandwidth" if we have exhausted + our write bandwidth some time in the last second. This was the + documented behavior before, but the actual behavior was to change + this value every TokenBucketRefillInterval. Fixes bug 25828; + bugfix on 0.2.3.5-alpha. + + o Minor bugfixes (C correctness): + - Add a missing lock acquisition in the shutdown code of the control + subsystem. Fixes bug 25675; bugfix on 0.2.7.3-rc. Found by + Coverity; this is CID 1433643. + + o Minor bugfixes (circuit path selection): + - Don't count path selection failures as circuit build failures. + This change should eliminate cases where Tor blames its guard or + the network for situations like insufficient microdescriptors + and/or overly restrictive torrc settings. Fixes bug 25705; bugfix + on 0.3.3.1-alpha. + + o Minor bugfixes (client): + - Don't consider Tor running as a client if the ControlPort is open, + but no actual client ports are open. Fixes bug 26062; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (code style): + - Fixed multiple includes of transports.h in src/or/connection.c + Fixes bug 25261; bugfix on 0.2.5.1-alpha. + - Remove the unused variable n_possible from the function + channel_get_for_extend(). Fixes bug 25645; bugfix on 0.2.4.4-alpha + + o Minor bugfixes (control interface): + - Respond with more human-readable error messages to GETINFO exit- + policy/* requests. Also, let controller know if an error is + transient (response code 551) or not (response code 552). Fixes + bug 25852; bugfix on 0.2.8.1-alpha. + + o Minor bugfixes (controller): + - Make CIRC_BW event reflect the total of all data sent on a + circuit, including padding and dropped cells. Also fix a mis- + counting bug when STREAM_BW events were enabled. Fixes bug 25400; + bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (correctness, client): + - Upon receiving a malformed connected cell, stop processing the cell + immediately. Previously we would mark the connection for close, but + continue processing the cell as if the connection were open. Fixes bug + 26072; bugfix on 0.2.4.7-alpha. + + o Minor bugfixes (directory client): + - When unverified-consensus is verified, rename it to cached- + consenus. Fixes bug 4187; bugfix on 0.2.0.3-alpha. + - Fixed launching a certificate fetch always during the scheduled + periodic consensus fetch by fetching only in those cases when + consensus are waiting for certs. Fixes bug 24740; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (documentation): + - Stop saying in the manual that clients cache ipv4 dns answers from + exit relays. We haven't used them since 0.2.6.3-alpha, and in + ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but + we forgot to say so in the man page. Fixes bug 26052; bugfix + on 0.3.2.6-alpha. + + o Minor bugfixes (error reporting): + - Improve tolerance for directory authorities with skewed clocks. + Previously, an authority with a clock more than 60 seconds ahead + could cause a client with a correct clock to warn that the + client's clock was behind. Now the clocks of a majority of + directory authorities have to be ahead of the client before this + warning will occur. Fixes bug 25756; bugfix on 0.2.2.25-alpha. + + o Minor bugfixes (Linux seccomp2 sandbox): + - Allow the nanosleep() system call, which glibc uses to implement + sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha. + + o Minor bugfixes (onion service): + - Fix a memory leak when a v3 onion service is configured and gets a + SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha. + - When parsing the descriptor signature, look for the token plus an + extra white-space at the end. This is more correct but also will + allow us to support new fields that might start with "signature". + Fixes bug 26069; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (path selection): + - Only select relays when they have the descriptors we prefer to use + for them. This change fixes a bug where we could select a relay + because it had _some_ descriptor, but reject it later with a + nonfatal assertion error because it didn't have the exact one we + wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (portability): + - Do not align mmap length, as it is not required by POSIX, and the + getpagesize function is deprecated. Fixes bug 25399; bugfix + on 0.1.1.23. + + o Minor bugfixes (portability, FreeBSD): + - In have_enough_mem_for_dircache(), the variable DIRCACHE_MIN_MEM_MB + does not stringify on FreeBSD, so we switch to tor_asprintf(). + Fixes bug 20887; bugfix on 0.2.8.1-alpha. Patch by Neel Chauhan. + + o Minor bugfixes (relay statistics): + - When a relay is collecting internal statistics about how many + create cell requests it has seen of each type, accurately count + the requests from relays that temporarily fall out of the + consensus. (To be extra conservative, we were already ignoring + requests from clients in our counts, and we continue ignoring them + here.) Fixes bug 24910; bugfix on 0.2.4.17-rc. + + o Minor bugfixes (relay, crash): + - Avoid a crash when running with DirPort set but ORPort tuned off. + Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (restart-in-process): + - When shutting down, Tor now clears all the flags in the control.c + module. This should prevent a bug where authentication cookies are + not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (testing): + - When testing workqueue event-cancellation, make sure that we + actually cancel an event, and that cancel each event with equal + probability. (It was previously possible, though extremely + unlikely, for our event-canceling test not to cancel any events.) + Fixes bug 26008; bugfix on 0.2.6.3-alpha. + - Repeat part of the test in test_client_pick_intro() a number of + times, to give it consistent coverage. Fixes bug 25996; bugfix + on 0.3.2.1-alpha. + - Remove randomness from the hs_common/responsible_hsdirs test, so + that it always takes the same path through the function it tests. + Fixes bug 25997; bugfix on 0.3.2.1-alpha. + - Change the behavior of the "channel/outbound" test so that it + never causes a 10-second rollover for the EWMA circuitmux code. + Previously, this behavior would happen randomly, and result in + fluctuating test coverage. Fixes bug 25994; bugfix + on 0.3.3.1-alpha. + - Use X509_new() to allocate certificates that will be freed later + with X509_free(). Previously, some parts of the unit tests had + used tor_malloc_zero(), which is incorrect, and which caused test + failures on Windows when they were built with extra hardening. + Fixes bugs 25943 and 25944; bugfix on 0.2.8.1-alpha. Patch by + Marcin Cieślak. + - While running the circuit_timeout test, fix the PRNG to a + deterministic AES stream, so that the test coverage from this test + will itself be deterministic. Fixes bug 25995; bugfix + on 0.2.2.2-alpha. + + o Minor bugfixes (vanguards): + - Allow the last hop in a vanguard circuit to be the same as our + first, to prevent the adversary from influencing guard node choice + by choice of last hop. Also prevent the creation of A - B - A + paths, or A - A paths, which are forbidden by relays. Fixes bug + 25870; bugfix on 0.3.3.1-alpha. + + o Code simplification and refactoring: + - Remove duplicate code in parse_{c,s}method_line and bootstrap + their functionalities into a single function. Fixes bug 6236; + bugfix on 0.2.3.6-alpha. + - We remove the PortForwsrding and PortForwardingHelper options, + related functions, and the port_forwarding tests. These options + were used by the now-deprecated Vidalia to help ordinary users + become Tor relays or bridges. Closes ticket 25409. Patch by + Neel Chauhan. + - In order to make the OR and dir checking function in router.c less + confusing we renamed some functions and + consider_testing_reachability() has been split into + router_should_check_reachability() and + router_do_reachability_checks(). Also we improved the documentation + in some functions. Closes ticket 18918. + - Initial work to isolate Libevent usage to a handful of modules in + our codebase, to simplify our call structure, and so that we can + more easily change event loops in the future if needed. Closes + ticket 23750. + - Introduce a function to call getsockname() and return tor_addr_t, + to save a little complexity throughout the codebase. Closes + ticket 18105. + - Make hsdir_index in node_t a hsdir_index_t rather than a pointer + as hsdir_index is always present. Also, we move hsdir_index_t into + or.h. Closes ticket 23094. Patch by Neel Chauhan. + - Merge functions used for describing nodes and suppress the + functions that do not allocate memory for the output buffer + string. NODE_DESC_BUF_LEN constant and format_node_description() + function cannot be used externally from router.c module anymore. + Closes ticket 25432. Patch by valentecaio. + - Our main loop has been simplified so that all important operations + happen inside events. Previously, some operations had to happen + outside the event loop, to prevent infinite sequences of event + activations. Closes ticket 25374. + - Put a SHA1 public key digest in hs_service_intro_point_t, and use + it in register_intro_circ() and service_intro_point_new(). This + prevents the digest from being re-calculated each time. Closes + ticket 23107. Patch by Neel Chauhan. + - Refactor token-bucket implementations to use a common backend. + Closes ticket 25766. + - Remove extern declaration of stats_n_seconds_working variable from + main, protecting its accesses with get_uptime() and reset_uptime() + functions. Closes ticket 25081, patch by “valentecaio”. + - Remove our previous logic for "cached gettimeofday()" -- our + coarse monotonic timers are fast enough for this purpose, and far + less error-prone. Implements part of ticket 25927. + - Remove the return value for fascist_firewall_choose_address_base(), + and sister functions such as fascist_firewall_choose_address_node() + and fascist_firewall_choose_address_rs(). Also, while we're here, + initialize the ap argument as leaving it uninitialized can pose a + security hazard. Closes ticket 24734. Patch by Neel Chauhan. + - Rename two fields of connection_t struct. timestamp_lastwritten is + renamed to timestamp_last_write_allowed and timestamp_lastread is + renamed to timestamp_last_read_allowed. Closes ticket 24714, patch + by "valentecaio". + - Since Tor requires C99, remove our old workaround code for libc + implementations where free(NULL) doesn't work. Closes ticket 24484. + - Use our standard rate-limiting code to deal with excessive + libevent failures, rather than the hand-rolled logic we had + before. Closes ticket 26016. + - We remove the return value of node_get_prim_orport() and + node_get_prim_dirport(), and introduce node_get_prim_orport() in + node_ipv6_or_preferred() and node_ipv6_dir_preferred() in order to + check for a null address. Closes ticket 23873. Patch by + Neel Chauhan. + - We switch to should_record_bridge_info() in + geoip_note_client_seen() and options_need_geoip_info() instead of + accessing the configuration values directly. Fixes bug 25290; + bugfix on 0.2.1.6-alpha. Patch by Neel Chauhan. + + o Deprecated features: + - As we are not recommending 0.2.5 anymore, we require relays that + once had an ed25519 key associated with their RSA key to always + have that key, instead of allowing them to drop back to a version + that didn't support ed25519. This means they need to use a new RSA + key if the want to downgrade to an older version of tor without + ed25519. Closes ticket 20522. + + o Documentation: + - Correct an IPv6 error in the documentation for ExitPolicy. Closes + ticket 25857. Patch from "CTassisF". + + o Removed features: + - Directory authorities will no longer support voting according to + any consensus method before consensus method 25. This keeps + authorities compatible with all authorities running 0.2.9.8 and + later, and does not break any clients or relays. Implements ticket + 24378 and proposal 290. + - The PortForwarding and PortForwardingHelper features have been + removed. The reasoning is, given that implementations of NAT + traversal protocols within common consumer grade routers are + frequently buggy, and that the target audience for a NAT punching + feature is a perhaps less-technically-inclined relay operator, + when the helper fails to setup traversal the problems are usually + deep, ugly, and very router specific, making them horrendously + impossible for technical support to reliable assist with, and thus + resulting in frustration all around. Unfortunately, relay + operators who would like to run relays behind NATs will need to + become more familiar with the port forwarding configurations on + their local router. Closes 25409. + - The TestingEnableTbEmptyEvent option has been removed. It was used + in testing simulations to measure how often connection buckets + were emptied, in order to improve our scheduling, but it has not + been actively used in years. Closes ticket 25760. + - The old "round-robin" circuit multiplexer (circuitmux) + implementation has been removed, along with a fairly large set of + code that existed to support it. It has not been the default + circuitmux since we introduced the "EWMA" circuitmux in 0.2.4.x, + but it still required an unreasonable amount of memory and CPU. + Closes ticket 25268. + + +Changes in version 0.3.3.5-rc - 2018-04-15 + Tor 0.3.3.5-rc fixes various bugs in earlier versions of Tor, + including some that could affect reliability or correctness. + + This is the first release candidate in the 0.3.3 series. If we find no + new bugs or regression here, then the first stable 0.3.3 release will + be nearly identical to this one. + + o Major bugfixes (security, protover, voting): + - Revise Rust implementation of protover to use a more memory- + efficient voting algorithm and corresponding data structures, thus + avoiding a potential memory-based DoS attack where specially + crafted protocol strings would expand to fill available memory. + Fixes bug 24031; bugfix on 0.3.3.1-alpha. + + o Major bugfixes (performance, load balancing): + - Directory authorities no longer vote in favor of the Guard flag + for relays without directory support. Starting in Tor + 0.3.0.1-alpha, clients have been avoiding using such relays in the + Guard position, leading to increasingly broken load balancing for + the 5%-or-so of Guards that don't advertise directory support. + Fixes bug 22310; bugfix on 0.3.0.6. + + o Minor feature (continuous integration): + - Update the Travis CI configuration to use the stable Rust channel, + now that we have decided to require that. Closes ticket 25714. + + o Minor features (config options): + - Change the way the default value for MaxMemInQueues is calculated. + We now use 40% of the hardware RAM if the system has 8 GB RAM or + more. Otherwise we use the former value of 75%. Closes + ticket 24782. + + o Minor features (geoip): + - Update geoip and geoip6 to the April 3 2018 Maxmind GeoLite2 + Country database. Closes ticket 25718. + + o Minor bugfixes (client): + - When using a listed relay as a bridge, and also using + microdescriptors, and considering that relay as a non-bridge in a + circuit, treat its microdescriptor as a valid source of + information about that relay. This change should prevent a non- + fatal assertion error. Fixes bug 25691; bugfix on 0.3.3.4-alpha. + + o Minor bugfixes (controller): + - Restore the correct operation of the RESOLVE command, which had + been broken since we added the ability to enable/disable DNS on + specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha. + + o Minor bugfixes (distribution, compilation, rust): + - Build correctly when the rust dependencies submodule is loaded, + but the TOR_RUST_DEPENDENCIES environment variable is not set. + Fixes bug 25679; bugfix on 0.3.3.1-alpha. + - Actually include all of our Rust source in our source + distributions. (Previously, a few of the files were accidentally + omitted.) Fixes bug 25732; bugfix on 0.3.3.2-alpha. + + o Minor bugfixes (documentation): + - Document that the PerConnBW{Rate,Burst} options will fall back to + their corresponding consensus parameters only if those parameters + are set. Previously we had claimed that these values would always + be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha. + - Revert a misformatting issue in the ExitPolicy documentation. + Fixes bug 25582; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (exit relay DNS retries): + - Re-attempt timed-out DNS queries 3 times before failure, since our + timeout is 5 seconds for them, but clients wait 10-15. Also allow + slightly more timeouts per resolver when an exit has multiple + resolvers configured. Fixes bug 21394; bugfix on 0.3.1.9. + + o Minor bugfixes (onion services): + - Re-instate counting the client HSDir fetch circuits against the + MaxClientCircuitsPending rate limit. Fixes bug 24989; bugfix + on 0.3.3.1-alpha. + - Remove underscores from the _HSLayer{2,3}Nodes options. This + expert-user configuration can now be enabled as HSLayer{2,3}Nodes. + Fixes bug 25581; bugfix on 0.3.3.1-alpha + + o Code simplification and refactoring: + - Move the list of default directory authorities to its own file. + Closes ticket 24854. Patch by "beastr0". + + o Documentation (manpage, denial of service): + - Provide more detail about the denial-of-service options, by + listing each mitigation and explaining how they relate. Closes + ticket 25248. + + +Changes in version 0.3.3.4-alpha - 2018-03-29 + Tor 0.3.3.4-alpha includes various bugfixes for issues found during + the alpha testing of earlier releases in its series. We are + approaching a stable 0.3.3.4-alpha release: more testing is welcome! + + o New system requirements: + - When built with Rust, Tor now depends on version 0.2.39 of the + libc crate. Closes tickets 25310 and 25664. + + o Major bugfixes (relay, connection): + - If we have failed to connect to a relay and received a connection + refused, timeout, or similar error (at the TCP level), do not try + that same address/port again for 60 seconds after the failure has + occurred. Fixes bug 24767; bugfix on 0.0.6. + + o Minor features (geoip): + - Update geoip and geoip6 to the March 8 2018 Maxmind GeoLite2 + Country database. Closes ticket 25469. + + o Minor features (log messages): + - Improve log message in the out-of-memory handler to include + information about memory usage from the different compression + backends. Closes ticket 25372. + + o Minor features (sandbox): + - Explicitly permit the poll() system call when the Linux + seccomp2-based sandbox is enabled: apparently, some versions of + libc use poll() when calling getpwnam(). Closes ticket 25313. + + o Minor bugfixes (C correctness): + - Fix a very unlikely (impossible, we believe) null pointer + dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by + Coverity; this is CID 1430932. + + o Minor bugfixes (channel, client): + - Better identify client connection when reporting to the geoip + client cache. Fixes bug 24904; bugfix on 0.3.1.7. + + o Minor bugfixes (compilation): + - Fix a C99 compliance issue in our configuration script that caused + compilation issues when compiling Tor with certain versions of + xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (controller, reliability): + - Avoid a (nonfatal) assertion failure when extending a one-hop + circuit from the controller to become a multihop circuit. Fixes + bug 24903; bugfix on 0.2.5.2-alpha. + + o Major bugfixes (networking): + - Tor will no longer reject IPv6 address strings from Tor Browser + when they are passed as hostnames in SOCKS5 requests. Fixes bug + 25036, bugfix on Tor 0.3.1.2. + + o Minor bugfixes (networking): + - string_is_valid_hostname() will not consider IP strings to be + valid hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5. + + o Minor bugfixes (onion service v3): + - Avoid an assertion failure when the next onion service + descriptor rotation type is out of sync with the consensus's + valid-after time. Instead, log a warning message with extra + information, so we can better hunt down the cause of this + assertion. Fixes bug 25306; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (testing): + - Avoid intermittent test failures due to a test that had relied on + onion service introduction point creation finishing within 5 + seconds of real clock time. Fixes bug 25450; bugfix + on 0.3.1.3-alpha. + - Rust crates are now automatically detected and tested. Previously, + some crates were not tested by `make test-rust` due to a static + string in the `src/test/test_rust.sh` script specifying which + crates to test. Fixes bug 25560; bugfix on 0.3.3.3-alpha. + + o Minor bugfixes (testing, benchmarks): + - Fix a crash when running benchmark tests on win32 systems. The + crash was due to a mutex that wasn't initialized before logging + and options were initialized. Fixes bug 25479; bugfix + on 0.3.3.3-alpha. + + o Minor bugfixes (warnings, ipv6): + - Avoid a bug warning that could occur when trying to connect to a + relay over IPv6. This warning would occur on a Tor instance that + downloads router descriptors, but prefers to use microdescriptors. + Fixes bug 25213; bugfix on 0.3.3.1-alpha. + + o Code simplification and refactoring: + - Remove the old (deterministic) directory retry logic entirely: + We've used exponential backoff exclusively for some time. Closes + ticket 23814. + + o Documentation: + - Improved the documentation of AccountingStart parameter. Closes + ticket 23635. + - Update the documentation for "Log" to include the current list of + logging domains. Closes ticket 25378. + + +Changes in version 0.3.1.10 - 2018-03-03 + Tor 0.3.1.10 backports a number of bugfixes, including important fixes for + security issues. + + It includes an important security fix for a remote crash attack + against directory authorities, tracked as TROVE-2018-001. + + This release also backports our new system for improved resistance to + denial-of-service attacks against relays. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + All directory authorities should upgrade to one of the versions + released today. Relays running 0.3.1.x may wish to update to one of + the versions released today, for the DoS mitigations. + + Please note: according to our release calendar, Tor 0.3.1 will no + longer be supported after 1 July 2018. If you will be running Tor + after that date, you should make sure to plan to upgrade to the latest + stable version, or downgrade to 0.2.9 (which will receive long-term + support). + + o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha): + - Give relays some defenses against the recent network overload. We + start with three defenses (default parameters in parentheses). + First: if a single client address makes too many concurrent + connections (>100), hang up on further connections. Second: if a + single client address makes circuits too quickly (more than 3 per + second, with an allowed burst of 90) while also having too many + connections open (3), refuse new create cells for the next while + (1-2 hours). Third: if a client asks to establish a rendezvous + point to you directly, ignore the request. These defenses can be + manually controlled by new torrc options, but relays will also + take guidance from consensus parameters, so there's no need to + configure anything manually. Implements ticket 24902. + + o Minor features (linux seccomp2 sandbox, backport from 0.3.2.5-alpha): + - Update the sandbox rules so that they should now work correctly + with Glibc 2.26. Closes ticket 24315. + + o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha): + - Fix an "off by 2" error in counting rendezvous failures on the + onion service side. While we thought we would stop the rendezvous + attempt after one failed circuit, we were actually making three + circuit attempts before giving up. Now switch to a default of 2, + and allow the consensus parameter "hs_service_max_rdv_failures" to + override. Fixes bug 24895; bugfix on 0.0.6. + + o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha): + - Add Link protocol version 5 to the supported protocols list. Fixes + bug 25070; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (relay, backport from 0.3.3.1-alpha): + - Fix a set of false positives where relays would consider + connections to other relays as being client-only connections (and + thus e.g. deserving different link padding schemes) if those + relays fell out of the consensus briefly. Now we look only at the + initial handshake and whether the connection authenticated as a + relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha. + + o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha): + - Make our OOM handler aware of the geoip client history cache so it + doesn't fill up the memory. This check is important for IPv6 and + our DoS mitigation subsystem. Closes ticket 25122. + + o Minor feature (relay statistics, backport from 0.3.2.6-alpha): + - Change relay bandwidth reporting stats interval from 4 hours to 24 + hours in order to reduce the efficiency of guard discovery + attacks. Fixes ticket 23856. + + o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (fallback directory mirrors, backport from 0.3.2.9): + - The fallback directory list has been re-generated based on the + current status of the network. Tor uses fallback directories to + bootstrap when it doesn't yet have up-to-date directory + information. Closes ticket 24801. + - Make the default DirAuthorityFallbackRate 0.1, so that clients + prefer to bootstrap from fallback directory mirrors. This is a + follow-up to 24679, which removed weights from the default + fallbacks. Implements ticket 24681. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2 + Country database. + + o Minor bugfix (channel connection, backport from 0.3.3.2-alpha): + - Use the actual observed address of an incoming relay connection, + not the canonical address of the relay from its descriptor, when + making decisions about how to handle the incoming connection. + Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera". + + o Minor bugfix (directory authority, backport from 0.3.3.2-alpha): + - Directory authorities, when refusing a descriptor from a rejected + relay, now explicitly tell the relay (in its logs) to set a valid + ContactInfo address and contact the bad-relays@ mailing list. + Fixes bug 25170; bugfix on 0.2.9.1. + + o Minor bugfixes (address selection, backport from 0.3.2.9): + - When the fascist_firewall_choose_address_ functions don't find a + reachable address, set the returned address to the null address + and port. This is a precautionary measure, because some callers do + not check the return value. Fixes bug 24736; bugfix + on 0.2.8.2-alpha. + + o Major bugfixes (bootstrapping, backport from 0.3.2.5-alpha): + - Fetch descriptors aggressively whenever we lack enough to build + circuits, regardless of how many descriptors we are missing. + Previously, we would delay launching the fetch when we had fewer + than 15 missing descriptors, even if some of those descriptors + were blocking circuits from building. Fixes bug 23985; bugfix on + 0.1.1.11-alpha. The effects of this bug became worse in + 0.3.0.3-alpha, when we began treating missing descriptors from our + primary guards as a reason to delay circuits. + - Don't try fetching microdescriptors from relays that have failed + to deliver them in the past. Fixes bug 23817; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (compilation, backport from 0.3.2.7-rc): + - Fix a signed/unsigned comparison warning introduced by our fix to + TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16. + + o Minor bugfixes (control port, linux seccomp2 sandbox, backport from 0.3.2.5-alpha): + - Avoid a crash when attempting to use the seccomp2 sandbox together + with the OwningControllerProcess feature. Fixes bug 24198; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfixes (directory cache, backport from 0.3.2.5-alpha): + - Recover better from empty or corrupt files in the consensus cache + directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha. + - When a consensus diff calculation is only partially successful, + only record the successful parts as having succeeded. Partial + success can happen if (for example) one compression method fails + but the others succeed. Previously we misrecorded all the + calculations as having succeeded, which would later cause a + nonfatal assertion failure. Fixes bug 24086; bugfix + on 0.3.1.1-alpha. + + o Minor bugfixes (entry guards, backport from 0.3.2.3-alpha): + - Tor now updates its guard state when it reads a consensus + regardless of whether it's missing descriptors. That makes tor use + its primary guards to fetch descriptors in some edge cases where + it would previously have used fallback directories. Fixes bug + 23862; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (logging, backport from 0.3.3.2-alpha): + - Don't treat inability to store a cached consensus object as a bug: + it can happen normally when we are out of disk space. Fixes bug + 24859; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (memory usage, backport from 0.3.2.8-rc): + - When queuing DESTROY cells on a channel, only queue the circuit-id + and reason fields: not the entire 514-byte cell. This fix should + help mitigate any bugs or attacks that fill up these queues, and + free more RAM for other uses. Fixes bug 24666; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (network layer, backport from 0.3.2.5-alpha): + - When closing a connection via close_connection_immediately(), we + mark it as "not blocked on bandwidth", to prevent later calls from + trying to unblock it, and give it permission to read. This fixes a + backtrace warning that can happen on relays under various + circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc. + + o Minor bugfixes (path selection, backport from 0.3.2.4-alpha): + - When selecting relays by bandwidth, avoid a rounding error that + could sometimes cause load to be imbalanced incorrectly. + Previously, we would always round upwards; now, we round towards + the nearest integer. This had the biggest effect when a relay's + weight adjustments should have given it weight 0, but it got + weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha. + - When calculating the fraction of nodes that have descriptors, and + all nodes in the network have zero bandwidths, count the number of + nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha. + - Actually log the total bandwidth in compute_weighted_bandwidths(). + Fixes bug 24170; bugfix on 0.2.4.3-alpha. + + o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha): + - Improve the performance of our consensus-diff application code + when Tor is built with the --enable-fragile-hardening option set. + Fixes bug 24826; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (OSX, backport from 0.3.3.1-alpha): + - Don't exit the Tor process if setrlimit() fails to change the file + limit (which can happen sometimes on some versions of OSX). Fixes + bug 21074; bugfix on 0.0.9pre5. + + o Minor bugfixes (portability, msvc, backport from 0.3.2.9): + - Fix a bug in the bit-counting parts of our timing-wheel code on + MSVC. (Note that MSVC is still not a supported build platform, due + to cyptographic timing channel risks.) Fixes bug 24633; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (relay, partial backport): + - Make the internal channel_is_client() function look at what sort + of connection handshake the other side used, rather than whether + the other side ever sent a create_fast cell to us. Backports part + of the fixes from bugs 22805 and 24898. + + o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + o Code simplification and refactoring (backport from 0.3.3.3-alpha): + - Update the "rust dependencies" submodule to be a project-level + repository, rather than a user repository. Closes ticket 25323. + + +Changes in version 0.2.9.15 - 2018-03-03 + Tor 0.2.9.15 backports important security and stability bugfixes from + later Tor releases. + + It includes an important security fix for a remote crash attack + against directory authorities, tracked as TROVE-2018-001. + + This release also backports our new system for improved resistance to + denial-of-service attacks against relays. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + All directory authorities should upgrade to one of the versions + released today. Relays running 0.2.9.x may wish to update to one of + the versions released today, for the DoS mitigations. + + o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Major features (denial-of-service mitigation): + - Give relays some defenses against the recent network overload. We + start with three defenses (default parameters in parentheses). + First: if a single client address makes too many concurrent + connections (>100), hang up on further connections. Second: if a + single client address makes circuits too quickly (more than 3 per + second, with an allowed burst of 90) while also having too many + connections open (3), refuse new create cells for the next while + (1-2 hours). Third: if a client asks to establish a rendezvous + point to you directly, ignore the request. These defenses can be + manually controlled by new torrc options, but relays will also + take guidance from consensus parameters, so there's no need to + configure anything manually. Implements ticket 24902. + + o Major bugfixes (bootstrapping): + - Fetch descriptors aggressively whenever we lack enough to build + circuits, regardless of how many descriptors we are missing. + Previously, we would delay launching the fetch when we had fewer + than 15 missing descriptors, even if some of those descriptors + were blocking circuits from building. Fixes bug 23985; bugfix on + 0.1.1.11-alpha. The effects of this bug became worse in + 0.3.0.3-alpha, when we began treating missing descriptors from our + primary guards as a reason to delay circuits. + + o Major bugfixes (onion services, retry behavior): + - Fix an "off by 2" error in counting rendezvous failures on the + onion service side. While we thought we would stop the rendezvous + attempt after one failed circuit, we were actually making three + circuit attempts before giving up. Now switch to a default of 2, + and allow the consensus parameter "hs_service_max_rdv_failures" to + override. Fixes bug 24895; bugfix on 0.0.6. + + o Minor feature (relay statistics): + - Change relay bandwidth reporting stats interval from 4 hours to 24 + hours in order to reduce the efficiency of guard discovery + attacks. Fixes ticket 23856. + + o Minor features (compatibility, OpenSSL): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (denial-of-service avoidance): + - Make our OOM handler aware of the geoip client history cache so it + doesn't fill up the memory. This check is important for IPv6 and + our DoS mitigation subsystem. Closes ticket 25122. + + o Minor features (fallback directory mirrors): + - The fallback directory list has been re-generated based on the + current status of the network. Tor uses fallback directories to + bootstrap when it doesn't yet have up-to-date directory + information. Closes ticket 24801. + - Make the default DirAuthorityFallbackRate 0.1, so that clients + prefer to bootstrap from fallback directory mirrors. This is a + follow-up to 24679, which removed weights from the default + fallbacks. Implements ticket 24681. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2 + Country database. + + o Minor features (linux seccomp2 sandbox): + - Update the sandbox rules so that they should now work correctly + with Glibc 2.26. Closes ticket 24315. + + o Minor bugfix (channel connection): + - Use the actual observed address of an incoming relay connection, + not the canonical address of the relay from its descriptor, when + making decisions about how to handle the incoming connection. + Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera". + + o Minor bugfix (directory authority): + - Directory authorities, when refusing a descriptor from a rejected + relay, now explicitly tell the relay (in its logs) to set a valid + ContactInfo address and contact the bad-relays@ mailing list. + Fixes bug 25170; bugfix on 0.2.9.1. + + o Minor bugfixes (address selection): + - When the fascist_firewall_choose_address_ functions don't find a + reachable address, set the returned address to the null address + and port. This is a precautionary measure, because some callers do + not check the return value. Fixes bug 24736; bugfix + on 0.2.8.2-alpha. + + o Minor bugfixes (compilation): + - Fix a signed/unsigned comparison warning introduced by our fix to + TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16. + + o Minor bugfixes (control port, linux seccomp2 sandbox): + - Avoid a crash when attempting to use the seccomp2 sandbox together + with the OwningControllerProcess feature. Fixes bug 24198; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfixes (memory usage): + - When queuing DESTROY cells on a channel, only queue the circuit-id + and reason fields: not the entire 514-byte cell. This fix should + help mitigate any bugs or attacks that fill up these queues, and + free more RAM for other uses. Fixes bug 24666; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (network layer): + - When closing a connection via close_connection_immediately(), we + mark it as "not blocked on bandwidth", to prevent later calls from + trying to unblock it, and give it permission to read. This fixes a + backtrace warning that can happen on relays under various + circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc. + + o Minor bugfixes (OSX): + - Don't exit the Tor process if setrlimit() fails to change the file + limit (which can happen sometimes on some versions of OSX). Fixes + bug 21074; bugfix on 0.0.9pre5. + + o Minor bugfixes (path selection): + - When selecting relays by bandwidth, avoid a rounding error that + could sometimes cause load to be imbalanced incorrectly. + Previously, we would always round upwards; now, we round towards + the nearest integer. This had the biggest effect when a relay's + weight adjustments should have given it weight 0, but it got + weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha. + - When calculating the fraction of nodes that have descriptors, and + all nodes in the network have zero bandwidths, count the number of + nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha. + - Actually log the total bandwidth in compute_weighted_bandwidths(). + Fixes bug 24170; bugfix on 0.2.4.3-alpha. + + o Minor bugfixes (portability, msvc): + - Fix a bug in the bit-counting parts of our timing-wheel code on + MSVC. (Note that MSVC is still not a supported build platform, due + to cryptographic timing channel risks.) Fixes bug 24633; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (relay): + - Make the internal channel_is_client() function look at what sort + of connection handshake the other side used, rather than whether + the other side ever sent a create_fast cell to us. Backports part + of the fixes from bugs 22805 and 24898. + + o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + +Changes in version 0.3.2.10 - 2018-03-03 + Tor 0.3.2.10 is the second stable release in the 0.3.2 series. It + backports a number of bugfixes, including important fixes for security + issues. + + It includes an important security fix for a remote crash attack + against directory authorities, tracked as TROVE-2018-001. + + Additionally, it backports a fix for a bug whose severity we have + upgraded: Bug 24700, which was fixed in 0.3.3.2-alpha, can be remotely + triggered in order to crash relays with a use-after-free pattern. As + such, we are now tracking that bug as TROVE-2018-002 and + CVE-2018-0491, and backporting it to earlier releases. This bug + affected versions 0.3.2.1-alpha through 0.3.2.9, as well as version + 0.3.3.1-alpha. + + This release also backports our new system for improved resistance to + denial-of-service attacks against relays. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + Relays running 0.3.2.x SHOULD upgrade to one of the versions released + today, for the fix to TROVE-2018-002. Directory authorities should + also upgrade. (Relays on earlier versions might want to update too for + the DoS mitigations.) + + o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Major bugfixes (scheduler, KIST, denial-of-service, backport from 0.3.3.2-alpha): + - Avoid adding the same channel twice in the KIST scheduler pending + list, which could lead to remote denial-of-service use-after-free + attacks against relays. Fixes bug 24700; bugfix on 0.3.2.1-alpha. + + o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha): + - Give relays some defenses against the recent network overload. We + start with three defenses (default parameters in parentheses). + First: if a single client address makes too many concurrent + connections (>100), hang up on further connections. Second: if a + single client address makes circuits too quickly (more than 3 per + second, with an allowed burst of 90) while also having too many + connections open (3), refuse new create cells for the next while + (1-2 hours). Third: if a client asks to establish a rendezvous + point to you directly, ignore the request. These defenses can be + manually controlled by new torrc options, but relays will also + take guidance from consensus parameters, so there's no need to + configure anything manually. Implements ticket 24902. + + o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha): + - Fix an "off by 2" error in counting rendezvous failures on the + onion service side. While we thought we would stop the rendezvous + attempt after one failed circuit, we were actually making three + circuit attempts before giving up. Now switch to a default of 2, + and allow the consensus parameter "hs_service_max_rdv_failures" to + override. Fixes bug 24895; bugfix on 0.0.6. + - New-style (v3) onion services now obey the "max rendezvous circuit + attempts" logic. Previously they would make as many rendezvous + circuit attempts as they could fit in the MAX_REND_TIMEOUT second + window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha. + + o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha): + - Add Link protocol version 5 to the supported protocols list. Fixes + bug 25070; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (relay, backport from 0.3.3.1-alpha): + - Fix a set of false positives where relays would consider + connections to other relays as being client-only connections (and + thus e.g. deserving different link padding schemes) if those + relays fell out of the consensus briefly. Now we look only at the + initial handshake and whether the connection authenticated as a + relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (scheduler, consensus, backport from 0.3.3.2-alpha): + - The scheduler subsystem was failing to promptly notice changes in + consensus parameters, making it harder to switch schedulers + network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha. + + o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha): + - Make our OOM handler aware of the geoip client history cache so it + doesn't fill up the memory. This check is important for IPv6 and + our DoS mitigation subsystem. Closes ticket 25122. + + o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2 + Country database. + + o Minor features (logging, diagnostic, backport from 0.3.3.2-alpha): + - When logging a failure to create an onion service's descriptor, + also log what the problem with the descriptor was. Diagnostic + for ticket 24972. + + o Minor bugfix (channel connection, backport from 0.3.3.2-alpha): + - Use the actual observed address of an incoming relay connection, + not the canonical address of the relay from its descriptor, when + making decisions about how to handle the incoming connection. + Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera". + + o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfix (directory authority, backport from 0.3.3.2-alpha): + - Directory authorities, when refusing a descriptor from a rejected + relay, now explicitly tell the relay (in its logs) to set a valid + ContactInfo address and contact the bad-relays@ mailing list. + Fixes bug 25170; bugfix on 0.2.9.1. + + o Minor bugfixes (build, rust, backport from 0.3.3.1-alpha): + - When building with Rust on OSX, link against libresolv, to work + around the issue at https://github.com/rust-lang/rust/issues/46797. + Fixes bug 24652; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (onion services, backport from 0.3.3.2-alpha): + - Remove a BUG() statement when a client fetches an onion descriptor + that has a lower revision counter than the one in its cache. This + can happen in normal circumstances due to HSDir desync. Fixes bug + 24976; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (logging, backport from 0.3.3.2-alpha): + - Don't treat inability to store a cached consensus object as a bug: + it can happen normally when we are out of disk space. Fixes bug + 24859; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha): + - Improve the performance of our consensus-diff application code + when Tor is built with the --enable-fragile-hardening option set. + Fixes bug 24826; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (OSX, backport from 0.3.3.1-alpha): + - Don't exit the Tor process if setrlimit() fails to change the file + limit (which can happen sometimes on some versions of OSX). Fixes + bug 21074; bugfix on 0.0.9pre5. + + o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (testing, backport from 0.3.3.1-alpha): + - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug + 25005; bugfix on 0.3.2.7-rc. + + o Minor bugfixes (v3 onion services, backport from 0.3.3.2-alpha): + - Look at the "HSRend" protocol version, not the "HSDir" protocol + version, when deciding whether a consensus entry can support the + v3 onion service protocol as a rendezvous point. Fixes bug 25105; + bugfix on 0.3.2.1-alpha. + + o Code simplification and refactoring (backport from 0.3.3.3-alpha): + - Update the "rust dependencies" submodule to be a project-level + repository, rather than a user repository. Closes ticket 25323. + + o Documentation (backport from 0.3.3.1-alpha) + - Document that operators who run more than one relay or bridge are + expected to set MyFamily and ContactInfo correctly. Closes + ticket 24526. + + +Changes in version 0.3.3.3-alpha - 2018-03-03 + Tor 0.3.3.3-alpha is the third alpha release for the 0.3.3.x series. + It includes an important security fix for a remote crash attack + against directory authorities tracked as TROVE-2018-001. + + Additionally, with this release, we are upgrading the severity of a + bug fixed in 0.3.3.2-alpha. Bug 24700, which was fixed in + 0.3.3.2-alpha, can be remotely triggered in order to crash relays with + a use-after-free pattern. As such, we are now tracking that bug as + TROVE-2018-002 and CVE-2018-0491. This bug affected versions + 0.3.2.1-alpha through 0.3.2.9, as well as 0.3.3.1-alpha. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + Relays running 0.3.2.x should upgrade to one of the versions released + today, for the fix to TROVE-2018-002. Directory authorities should + also upgrade. (Relays on earlier versions might want to update too for + the DoS mitigations.) + + o Major bugfixes (denial-of-service, directory authority): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Minor features (compatibility, OpenSSL): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (logging): + - Clarify the log messages produced when getrandom() or a related + entropy-generation mechanism gives an error. Closes ticket 25120. + + o Minor features (testing): + - Add a "make test-rust" target to run the rust tests only. Closes + ticket 25071. + + o Minor bugfixes (denial-of-service): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfixes (DoS mitigation): + - Add extra safety checks when refilling the circuit creation bucket + to ensure we never set a value above the allowed maximum burst. + Fixes bug 25202; bugfix on 0.3.3.2-alpha. + - When a new consensus arrives, don't update our DoS-mitigation + parameters if we aren't a public relay. Fixes bug 25223; bugfix + on 0.3.3.2-alpha. + + o Minor bugfixes (man page, SocksPort): + - Remove dead code from the old "SocksSocket" option, and rename + SocksSocketsGroupWritable to UnixSocksGroupWritable. The old option + still works, but is deprecated. Fixes bug 24343; bugfix on 0.2.6.3. + + o Minor bugfixes (performance): + - Reduce the number of circuits that will be opened at once during + the circuit build timeout phase. This is done by increasing the + idle timeout to 3 minutes, and lowering the maximum number of + concurrent learning circuits to 10. Fixes bug 24769; bugfix + on 0.3.1.1-alpha. + + o Minor bugfixes (spec conformance): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (spec conformance, rust): + - Resolve a denial-of-service issue caused by an infinite loop in + the rust protover code. Fixes bug 25250, bugfix on 0.3.3.1-alpha. + Also tracked as TROVE-2018-003. + + o Code simplification and refactoring: + - Update the "rust dependencies" submodule to be a project-level + repository, rather than a user repository. Closes ticket 25323. + + Changes in version 0.3.3.2-alpha - 2018-02-10 Tor 0.3.3.2-alpha is the second alpha in the 0.3.3.x series. It introduces a mechanism to handle the high loads that many relay @@ -113,7 +1622,7 @@ Changes in version 0.3.3.2-alpha - 2018-02-10 would call the Rust implementation of protover_get_supported_protocols(). This was due to the C version returning a static string, whereas the Rust version newly allocated - a CString to pass accross the FFI boundary. Consequently, the C + a CString to pass across the FFI boundary. Consequently, the C code was not expecting to need to free() what it was given. Fixes bug 25127; bugfix on 0.3.2.1-alpha. @@ -171,12 +1680,12 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 o Major features (IPv6, directory documents): - Add consensus method 27, which adds IPv6 ORPorts to the microdesc consensus. This information makes it easier for IPv6 clients to - bootstrap and choose reachable entry guards. Implements 23826. + bootstrap and choose reachable entry guards. Implements ticket 23826. - Add consensus method 28, which removes IPv6 ORPorts from - microdescriptors. Now that the consensus contains IPv6 ORPorts, - they are redundant in microdescs. This change will be used by Tor - clients on 0.2.8.x and later. (That is to say, with all Tor - clients having IPv6 bootstrap and guard support.) Implements 23828. + microdescriptors. Now that the consensus contains IPv6 ORPorts, they + are redundant in microdescs. This change will be used by Tor clients + on 0.2.8.x and later. (That is to say, with all Tor clients that + have IPv6 bootstrap and guard support.) Implements ticket 23828. - Expand the documentation for AuthDirHasIPv6Connectivity when it is set by different numbers of authorities. Fixes 23870 on 0.2.4.1-alpha. @@ -206,7 +1715,7 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 experience with Rust, and plan future Rust integration work. Implementation by Chelsea Komlo. Closes ticket 22840. - o Major features (storage, configuration): + o Minor features (storage, configuration): - Users can store cached directory documents somewhere other than the DataDirectory by using the CacheDirectory option. Similarly, the storage location for relay's keys can be overridden with the @@ -243,10 +1752,10 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 o Minor feature (IPv6): - Make IPv6-only clients wait for microdescs for relays, even if we were previously using descriptors (or were using them as a bridge) - and have a cached descriptor for them. Implements 23827. + and have a cached descriptor for them. Implements ticket 23827. - When a consensus has IPv6 ORPorts, make IPv6-only clients use them, rather than waiting to download microdescriptors. - Implements 23827. + Implements ticket 23827. o Minor features (cleanup): - Tor now deletes the CookieAuthFile and ExtORPortCookieAuthFile @@ -260,14 +1769,8 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 - Where possible, the tor_free() macro now only evaluates its input once. Part of ticket 24337. - Check that microdesc ed25519 ids are non-zero in - node_get_ed25519_id() before returning them. Implements 24001, - patch by "aruna1234". - - o Minor features (directory authority): - - Make the "Exit" flag assignment only depend on whether the exit - policy allows connections to ports 80 and 443. Previously relays - would get the Exit flag if they allowed connections to one of - these ports and also port 6667. Resolves ticket 23637. + node_get_ed25519_id() before returning them. Implements ticket + 24001, patch by "aruna1234". o Minor features (embedding): - Tor can now start with a preauthenticated control connection @@ -279,7 +1782,7 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 - On most errors that would cause Tor to exit, it now tries to return from the tor_main() function, rather than calling the system exit() function. Most users won't notice a difference here, - but it should make a significant for programs that run Tor inside + but it should be significant for programs that run Tor inside a separate thread: they should now be able to survive Tor's exit conditions rather than having Tor shut down the entire process. Closes ticket 23848. @@ -379,7 +1882,7 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 SIO_IDEAL_SEND_BACKLOG_QUERY. Closes ticket 22798. Patch from Vort. - o Minor features (relay): + o Major features (relay): - Implement an option, ReducedExitPolicy, to allow an Tor exit relay operator to use a more reasonable ("reduced") exit policy, rather than the default one. If you want to run an exit node without @@ -539,7 +2042,7 @@ Changes in version 0.3.3.1-alpha - 2018-01-25 adding very little except for unit test. o Code simplification and refactoring (circuit rendezvous): - - Split the client-size rendezvous circuit lookup into two + - Split the client-side rendezvous circuit lookup into two functions: one that returns only established circuits and another that returns all kinds of circuits. Closes ticket 23459. @@ -1740,7 +3243,7 @@ Changes in version 0.3.2.2-alpha - 2017-09-29 include better testing and logging. The following comprises the complete list of changes included - in tor-0.3.2.2-alpha: + in 0.3.2.2-alpha: o Major bugfixes (relay, crash, assertion failure): - Fix a timing-based assertion failure that could occur when the @@ -3143,7 +4646,7 @@ Changes in version 0.3.0.8 - 2017-06-08 o Major bugfixes (hidden service v3, backport from 0.3.1.1-alpha): - Stop rejecting v3 hidden service descriptors because their size did not match an old padding rule. Fixes bug 22447; bugfix on - tor-0.3.0.1-alpha. + 0.3.0.1-alpha. o Minor features (fallback directory list, backport from 0.3.1.3-alpha): - Replace the 177 fallbacks originally introduced in Tor 0.2.9.8 in @@ -4348,7 +5851,7 @@ Changes in version 0.3.0.4-rc - 2017-03-01 o Major bugfixes (hidden service directory v3): - Stop crashing on a failed v3 hidden service descriptor lookup - failure. Fixes bug 21471; bugfixes on tor-0.3.0.1-alpha. + failure. Fixes bug 21471; bugfixes on 0.3.0.1-alpha. o Major bugfixes (parsing): - When parsing a malformed content-length field from an HTTP @@ -4433,7 +5936,7 @@ Changes in version 0.3.0.4-rc - 2017-03-01 o Minor bugfixes (testing): - Fix Raspbian build issues related to missing socket errno in - test_util.c. Fixes bug 21116; bugfix on tor-0.2.8.2. Patch + test_util.c. Fixes bug 21116; bugfix on 0.2.8.2. Patch by "hein". - Rename "make fuzz" to "make test-fuzz-corpora", since it doesn't actually fuzz anything. Fixes bug 21447; bugfix on 0.3.0.3-alpha. @@ -5067,7 +6570,7 @@ Changes in version 0.3.0.1-alpha - 2016-12-19 - When finishing writing a file to disk, if we were about to replace the file with the temporary file created before and we fail to replace it, remove the temporary file so it doesn't stay on disk. - Fixes bug 20646; bugfix on tor-0.2.0.7-alpha. Patch by fk. + Fixes bug 20646; bugfix on 0.2.0.7-alpha. Patch by fk. o Minor bugfixes (Windows): - Check for getpagesize before using it to mmap files. This fixes @@ -5103,13 +6606,13 @@ Changes in version 0.3.0.1-alpha - 2016-12-19 o Documentation: - Include the "TBits" unit in Tor's man page. Fixes part of bug - 20622; bugfix on tor-0.2.5.1-alpha. + 20622; bugfix on 0.2.5.1-alpha. - Change '1' to 'weight_scale' in consensus bw weights calculation comments, as that is reality. Closes ticket 20273. Patch from pastly. - Correct the value for AuthDirGuardBWGuarantee in the manpage, from 250 KBytes to 2 MBytes. Fixes bug 20435; bugfix - on tor-0.2.5.6-alpha. + on 0.2.5.6-alpha. - Stop the man page from incorrectly stating that HiddenServiceDir must already exist. Fixes 20486. - Clarify that when ClientRejectInternalAddresses is enabled (which diff --git a/Makefile.am b/Makefile.am index dcb445c970..c58140ae74 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,8 +26,9 @@ TESTING_TOR_BINARY=$(top_builddir)/src/or/tor$(EXEEXT) endif if USE_RUST -rust_ldadd=$(top_builddir)/src/rust/target/release/@TOR_RUST_STATIC_NAME@ \ - @TOR_RUST_EXTRA_LIBS@ +## this MUST be $(), otherwise am__DEPENDENCIES will not track it +rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \ + $(TOR_RUST_EXTRA_LIBS) else rust_ldadd= endif @@ -38,6 +39,7 @@ include contrib/include.am EXTRA_DIST+= \ ChangeLog \ + CONTRIBUTING \ INSTALL \ LICENSE \ Makefile.nmake \ @@ -51,14 +53,14 @@ AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s' if COVERAGE_ENABLED TEST_CFLAGS=-fno-inline -fprofile-arcs -ftest-coverage if DISABLE_ASSERTS_IN_UNIT_TESTS -TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS +TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE -DDISABLE_ASSERTS_IN_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@ else -TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE +TEST_CPPFLAGS=-DTOR_UNIT_TESTS -DTOR_COVERAGE @TOR_MODULES_ALL_ENABLED@ endif TEST_NETWORK_FLAGS=--coverage --hs-multi-client 1 else TEST_CFLAGS= -TEST_CPPFLAGS=-DTOR_UNIT_TESTS +TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@ TEST_NETWORK_FLAGS=--hs-multi-client 1 endif TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings @@ -218,6 +220,26 @@ check-logs: $(top_srcdir)/scripts/maint/checkLogs.pl \ $(top_srcdir)/src/*/*.[ch] | sort -n +.PHONY: check-typos +check-typos: + @if test -x "`which misspell 2>&1;true`"; then \ + echo "Checking for Typos ..."; \ + (misspell \ + $(top_srcdir)/src/[^e]*/*.[ch] \ + $(top_srcdir)/doc \ + $(top_srcdir)/contrib \ + $(top_srcdir)/scripts \ + $(top_srcdir)/README \ + $(top_srcdir)/ChangeLog \ + $(top_srcdir)/INSTALL \ + $(top_srcdir)/ReleaseNotes \ + $(top_srcdir)/LICENSE); \ + else \ + echo "Tor can use misspell to check for typos."; \ + echo "It seems that you don't have misspell installed."; \ + echo "You can install the latest version of misspell here: https://github.com/client9/misspell#install"; \ + fi + .PHONY: check-changes check-changes: if USEPYTHON diff --git a/ReleaseNotes b/ReleaseNotes index 6c31d8ab0c..8a38289c2c 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -1,6 +1,1263 @@ -This document summarizes new features and bugfixes in each stable release -of Tor. If you want to see more detailed descriptions of the changes in -each development snapshot, see the ChangeLog file. +This document summarizes new features and bugfixes in each stable +release of Tor. If you want to see more detailed descriptions of the +changes in each development snapshot, see the ChangeLog file. + +Changes in version 0.3.3.7 - 2018-06-12 + Tor 0.3.3.7 backports several changes from the 0.3.4.x series, including + fixes for bugs affecting compatibility and stability. + + o Directory authority changes: + - Add an IPv6 address for the "dannenberg" directory authority. + Closes ticket 26343. + + o Minor features (geoip): + - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2 + Country database. Closes ticket 26351. + + o Minor bugfixes (compatibility, openssl, backport from 0.3.4.2-alpha): + - Work around a change in OpenSSL 1.1.1 where return values that + would previously indicate "no password" now indicate an empty + password. Without this workaround, Tor instances running with + OpenSSL 1.1.1 would accept descriptors that other Tor instances + would reject. Fixes bug 26116; bugfix on 0.2.5.16. + + o Minor bugfixes (compilation, backport from 0.3.4.2-alpha): + - Silence unused-const-variable warnings in zstd.h with some GCC + versions. Fixes bug 26272; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (controller, backport from 0.3.4.2-alpha): + - Improve accuracy of the BUILDTIMEOUT_SET control port event's + TIMEOUT_RATE and CLOSE_RATE fields. (We were previously + miscounting the total number of circuits for these field values.) + Fixes bug 26121; bugfix on 0.3.3.1-alpha. + + o Minor bugfixes (hardening, backport from 0.3.4.2-alpha): + - Prevent a possible out-of-bounds smartlist read in + protover_compute_vote(). Fixes bug 26196; bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (path selection, backport from 0.3.4.1-alpha): + - Only select relays when they have the descriptors we prefer to use + for them. This change fixes a bug where we could select a relay + because it had _some_ descriptor, but reject it later with a + nonfatal assertion error because it didn't have the exact one we + wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha. + + +Changes in version 0.3.3.6 - 2018-05-22 + Tor 0.3.3.6 is the first stable release in the 0.3.3 series. It + backports several important fixes from the 0.3.4.1-alpha. + + The Tor 0.3.3 series includes controller support and other + improvements for v3 onion services, official support for embedding Tor + within other applications, and our first non-trivial module written in + the Rust programming language. (Rust is still not enabled by default + when building Tor.) And as usual, there are numerous other smaller + bugfixes, features, and improvements. + + Below are the changes since 0.3.2.10. For a list of only the changes + since 0.3.3.5-rc, see the ChangeLog file. + + o New system requirements: + - When built with Rust, Tor now depends on version 0.2.39 of the + libc crate. Closes tickets 25310 and 25664. + + o Major features (embedding): + - There is now a documented stable API for programs that need to + embed Tor. See tor_api.h for full documentation and known bugs. + Closes ticket 23684. + - Tor now has support for restarting in the same process. + Controllers that run Tor using the "tor_api.h" interface can now + restart Tor after Tor has exited. This support is incomplete, + however: we fixed crash bugs that prevented it from working at + all, but many bugs probably remain, including a possibility of + security issues. Implements ticket 24581. + + o Major features (IPv6, directory documents): + - Add consensus method 27, which adds IPv6 ORPorts to the microdesc + consensus. This information makes it easier for IPv6 clients to + bootstrap and choose reachable entry guards. Implements + ticket 23826. + - Add consensus method 28, which removes IPv6 ORPorts from + microdescriptors. Now that the consensus contains IPv6 ORPorts, + they are redundant in microdescs. This change will be used by Tor + clients on 0.2.8.x and later. (That is to say, with all Tor + clients that have IPv6 bootstrap and guard support.) Implements + ticket 23828. + - Expand the documentation for AuthDirHasIPv6Connectivity when it is + set by different numbers of authorities. Fixes 23870 + on 0.2.4.1-alpha. + + o Major features (onion service v3, control port): + - The control port now supports commands and events for v3 onion + services. It is now possible to create ephemeral v3 services using + ADD_ONION. Additionally, several events (HS_DESC, HS_DESC_CONTENT, + CIRC and CIRC_MINOR) and commands (GETINFO, HSPOST, ADD_ONION and + DEL_ONION) have been extended to support v3 onion services. Closes + ticket 20699; implements proposal 284. + + o Major features (onion services): + - Provide torrc options to pin the second and third hops of onion + service circuits to a list of nodes. The option HSLayer2Guards + pins the second hop, and the option HSLayer3Guards pins the third + hop. These options are for use in conjunction with experiments + with "vanguards" for preventing guard enumeration attacks. Closes + ticket 13837. + - When v3 onion service clients send introduce cells, they now + include the IPv6 address of the rendezvous point, if it has one. + Current v3 onion services running 0.3.2 ignore IPv6 addresses, but + in future Tor versions, IPv6-only v3 single onion services will be + able to use IPv6 addresses to connect directly to the rendezvous + point. Closes ticket 23577. Patch by Neel Chauhan. + + o Major features (relay): + - Implement an option, ReducedExitPolicy, to allow an Tor exit relay + operator to use a more reasonable ("reduced") exit policy, rather + than the default one. If you want to run an exit node without + thinking too hard about which ports to allow, this one is for you. + Closes ticket 13605. Patch from Neel Chauhan. + + o Major features (rust, portability, experimental): + - Tor now ships with an optional implementation of one of its + smaller modules (protover.c) in the Rust programming language. To + try it out, install a Rust build environment, and configure Tor + with "--enable-rust --enable-cargo-online-mode". This should not + cause any user-visible changes, but should help us gain more + experience with Rust, and plan future Rust integration work. + Implementation by Chelsea Komlo. Closes ticket 22840. + + o Major bugfixes (directory authorities, security, backport from 0.3.4.1-alpha): + - When directory authorities read a zero-byte bandwidth file, they + would previously log a warning with the contents of an + uninitialised buffer. They now log a warning about the empty file + instead. Fixes bug 26007; bugfix on 0.2.2.1-alpha. + + o Major bugfixes (security, directory authority, denial-of-service): + - Fix a bug that could have allowed an attacker to force a directory + authority to use up all its RAM by passing it a maliciously + crafted protocol versions string. Fixes bug 25517; bugfix on + 0.2.9.4-alpha. This issue is also tracked as TROVE-2018-005. + + o Major bugfixes (crash, backport from 0.3.4.1-alpha): + - Avoid a rare assertion failure in the circuit build timeout code + if we fail to allow any circuits to actually complete. Fixes bug + 25733; bugfix on 0.2.2.2-alpha. + + o Major bugfixes (netflow padding): + - Stop adding unneeded channel padding right after we finish + flushing to a connection that has been trying to flush for many + seconds. Instead, treat all partial or complete flushes as + activity on the channel, which will defer the time until we need + to add padding. This fix should resolve confusing and scary log + messages like "Channel padding timeout scheduled 221453ms in the + past." Fixes bug 22212; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (networking): + - Tor will no longer reject IPv6 address strings from Tor Browser + when they are passed as hostnames in SOCKS5 requests. Fixes bug + 25036, bugfix on Tor 0.3.1.2. + + o Major bugfixes (onion service, backport from 0.3.4.1-alpha): + - Correctly detect when onion services get disabled after HUP. Fixes + bug 25761; bugfix on 0.3.2.1. + + o Major bugfixes (performance, load balancing): + - Directory authorities no longer vote in favor of the Guard flag + for relays without directory support. Starting in Tor + 0.3.0.1-alpha, clients have been avoiding using such relays in the + Guard position, leading to increasingly broken load balancing for + the 5%-or-so of Guards that don't advertise directory support. + Fixes bug 22310; bugfix on 0.3.0.6. + + o Major bugfixes (relay): + - If we have failed to connect to a relay and received a connection + refused, timeout, or similar error (at the TCP level), do not try + that same address/port again for 60 seconds after the failure has + occurred. Fixes bug 24767; bugfix on 0.0.6. + + o Major bugfixes (relay, denial of service, backport from 0.3.4.1-alpha): + - Impose a limit on circuit cell queue size. The limit can be + controlled by a consensus parameter. Fixes bug 25226; bugfix + on 0.2.4.14-alpha. + + o Minor features (cleanup): + - Tor now deletes the CookieAuthFile and ExtORPortCookieAuthFile + when it stops. Closes ticket 23271. + + o Minor features (compatibility, backport from 0.3.4.1-alpha): + - Avoid some compilation warnings with recent versions of LibreSSL. + Closes ticket 26006. + + o Minor features (config options): + - Change the way the default value for MaxMemInQueues is calculated. + We now use 40% of the hardware RAM if the system has 8 GB RAM or + more. Otherwise we use the former value of 75%. Closes + ticket 24782. + + o Minor features (continuous integration): + - Update the Travis CI configuration to use the stable Rust channel, + now that we have decided to require that. Closes ticket 25714. + + o Minor features (continuous integration, backport from 0.3.4.1-alpha): + - Our .travis.yml configuration now includes support for testing the + results of "make distcheck". (It's not uncommon for "make check" + to pass but "make distcheck" to fail.) Closes ticket 25814. + - Our Travis CI configuration now integrates with the Coveralls + coverage analysis tool. Closes ticket 25818. + + o Minor features (defensive programming): + - Most of the functions in Tor that free objects have been replaced + with macros that free the objects and set the corresponding + pointers to NULL. This change should help prevent a large class of + dangling pointer bugs. Closes ticket 24337. + - Where possible, the tor_free() macro now only evaluates its input + once. Part of ticket 24337. + - Check that microdesc ed25519 ids are non-zero in + node_get_ed25519_id() before returning them. Implements ticket + 24001, patch by "aruna1234". + + o Minor features (directory authority): + - When directory authorities are unable to add signatures to a + pending consensus, log the reason why. Closes ticket 24849. + + o Minor features (embedding): + - Tor can now start with a preauthenticated control connection + created by the process that launched it. This feature is meant for + use by programs that want to launch and manage a Tor process + without allowing other programs to manage it as well. For more + information, see the __OwningControllerFD option documented in + control-spec.txt. Closes ticket 23900. + - On most errors that would cause Tor to exit, it now tries to + return from the tor_main() function, rather than calling the + system exit() function. Most users won't notice a difference here, + but it should be significant for programs that run Tor inside a + separate thread: they should now be able to survive Tor's exit + conditions rather than having Tor shut down the entire process. + Closes ticket 23848. + - Applications that want to embed Tor can now tell Tor not to + register any of its own POSIX signal handlers, using the + __DisableSignalHandlers option. Closes ticket 24588. + + o Minor features (fallback directory list): + - Avoid selecting fallbacks that change their IP addresses too + often. Select more fallbacks by ignoring the Guard flag, and + allowing lower cutoffs for the Running and V2Dir flags. Also allow + a lower bandwidth, and a higher number of fallbacks per operator + (5% of the list). Implements ticket 24785. + - Update the fallback whitelist and blacklist based on opt-ins and + relay changes. Closes tickets 22321, 24678, 22527, 24135, + and 24695. + + o Minor features (fallback directory mirror configuration): + - Add a nickname to each fallback in a C comment. This makes it + easier for operators to find their relays, and allows stem to use + nicknames to identify fallbacks. Implements ticket 24600. + - Add a type and version header to the fallback directory mirror + file. Also add a delimiter to the end of each fallback entry. This + helps external parsers like stem and Relay Search. Implements + ticket 24725. + - Add an extrainfo cache flag for each fallback in a C comment. This + allows stem to use fallbacks to fetch extra-info documents, rather + than using authorities. Implements ticket 22759. + - Add the generateFallbackDirLine.py script for automatically + generating fallback directory mirror lines from relay fingerprints. + No more typos! Add the lookupFallbackDirContact.py script for + automatically looking up operator contact info from relay + fingerprints. Implements ticket 24706, patch by teor and atagar. + - Reject any fallback directory mirror that serves an expired + consensus. Implements ticket 20942, patch by "minik". + - Remove commas and equals signs from external string inputs to the + fallback list. This avoids format confusion attacks. Implements + ticket 24726. + - Remove the "weight=10" line from fallback directory mirror + entries. Ticket 24681 will maintain the current fallback weights + by changing Tor's default fallback weight to 10. Implements + ticket 24679. + - Stop logging excessive information about fallback netblocks. + Implements ticket 24791. + + o Minor features (forward-compatibility): + - If a relay supports some link authentication protocol that we do + not recognize, then include that relay's ed25519 key when telling + other relays to extend to it. Previously, we treated future + versions as if they were too old to support ed25519 link + authentication. Closes ticket 20895. + + o Minor features (geoip): + - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 Country + database. Closes ticket 26104. + + o Minor features (heartbeat): + - Add onion service information to our heartbeat logs, displaying + stats about the activity of configured onion services. Closes + ticket 24896. + + o Minor features (instrumentation, development): + - Add the MainloopStats option to allow developers to get + instrumentation information from the main event loop via the + heartbeat messages. We hope to use this to improve Tor's behavior + when it's trying to sleep. Closes ticket 24605. + + o Minor features (IPv6): + - Make IPv6-only clients wait for microdescs for relays, even if we + were previously using descriptors (or were using them as a bridge) + and have a cached descriptor for them. Implements ticket 23827. + - When a consensus has IPv6 ORPorts, make IPv6-only clients use + them, rather than waiting to download microdescriptors. Implements + ticket 23827. + + o Minor features (log messages): + - Improve log message in the out-of-memory handler to include + information about memory usage from the different compression + backends. Closes ticket 25372. + - Improve a warning message that happens when we fail to re-parse an + old router because of an expired certificate. Closes ticket 20020. + - Make the log more quantitative when we hit MaxMemInQueues + threshold exposing some values. Closes ticket 24501. + + o Minor features (logging): + - Clarify the log messages produced when getrandom() or a related + entropy-generation mechanism gives an error. Closes ticket 25120. + - Added support for the Android logging subsystem. Closes + ticket 24362. + + o Minor features (performance): + - Support predictive circuit building for onion service circuits + with multiple layers of guards. Closes ticket 23101. + - Use stdatomic.h where available, rather than mutexes, to implement + atomic_counter_t. Closes ticket 23953. + + o Minor features (performance, 32-bit): + - Improve performance on 32-bit systems by avoiding 64-bit division + when calculating the timestamp in milliseconds for channel padding + computations. Implements ticket 24613. + - Improve performance on 32-bit systems by avoiding 64-bit division + when timestamping cells and buffer chunks for OOM calculations. + Implements ticket 24374. + + o Minor features (performance, OSX, iOS): + - Use the mach_approximate_time() function (when available) to + implement coarse monotonic time. Having a coarse time function + should avoid a large number of system calls, and improve + performance slightly, especially under load. Closes ticket 24427. + + o Minor features (performance, windows): + - Improve performance on Windows Vista and Windows 7 by adjusting + TCP send window size according to the recommendation from + SIO_IDEAL_SEND_BACKLOG_QUERY. Closes ticket 22798. Patch + from Vort. + + o Minor features (sandbox): + - Explicitly permit the poll() system call when the Linux + seccomp2-based sandbox is enabled: apparently, some versions of + libc use poll() when calling getpwnam(). Closes ticket 25313. + + o Minor features (storage, configuration): + - Users can store cached directory documents somewhere other than + the DataDirectory by using the CacheDirectory option. Similarly, + the storage location for relay's keys can be overridden with the + KeyDirectory option. Closes ticket 22703. + + o Minor features (testing): + - Add a "make test-rust" target to run the rust tests only. Closes + ticket 25071. + + o Minor features (testing, debugging, embedding): + - For development purposes, Tor now has a mode in which it runs for + a few seconds, then stops, and starts again without exiting the + process. This mode is meant to help us debug various issues with + ticket 23847. To use this feature, compile with + --enable-restart-debugging, and set the TOR_DEBUG_RESTART + environment variable. This is expected to crash a lot, and is + really meant for developers only. It will likely be removed in a + future release. Implements ticket 24583. + + o Minor bugfixes (build, rust): + - Fix output of autoconf checks to display success messages for Rust + dependencies and a suitable rustc compiler version. Fixes bug + 24612; bugfix on 0.3.1.3-alpha. + - Don't pass the --quiet option to cargo: it seems to suppress some + errors, which is not what we want to do when building. Fixes bug + 24518; bugfix on 0.3.1.7. + - Build correctly when building from outside Tor's source tree with + the TOR_RUST_DEPENDENCIES option set. Fixes bug 22768; bugfix + on 0.3.1.7. + + o Minor bugfixes (C correctness): + - Fix a very unlikely (impossible, we believe) null pointer + dereference. Fixes bug 25629; bugfix on 0.2.9.15. Found by + Coverity; this is CID 1430932. + + o Minor bugfixes (channel, client): + - Better identify client connection when reporting to the geoip + client cache. Fixes bug 24904; bugfix on 0.3.1.7. + + o Minor bugfixes (circuit, cannibalization): + - Don't cannibalize preemptively-built circuits if we no longer + recognize their first hop. This situation can happen if our Guard + relay went off the consensus after the circuit was created. Fixes + bug 24469; bugfix on 0.0.6. + + o Minor bugfixes (client, backport from 0.3.4.1-alpha): + - Don't consider Tor running as a client if the ControlPort is open, + but no actual client ports are open. Fixes bug 26062; bugfix + on 0.2.9.4-alpha. + + o Minor bugfixes (compilation): + - Fix a C99 compliance issue in our configuration script that caused + compilation issues when compiling Tor with certain versions of + xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha. + + o Minor bugfixes (controller): + - Restore the correct operation of the RESOLVE command, which had + been broken since we added the ability to enable/disable DNS on + specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha. + - Avoid a (nonfatal) assertion failure when extending a one-hop + circuit from the controller to become a multihop circuit. Fixes + bug 24903; bugfix on 0.2.5.2-alpha. + + o Minor bugfixes (correctness): + - Remove a nonworking, unnecessary check to see whether a circuit + hop's identity digest was set when the circuit failed. Fixes bug + 24927; bugfix on 0.2.4.4-alpha. + + o Minor bugfixes (correctness, client, backport from 0.3.4.1-alpha): + - Upon receiving a malformed connected cell, stop processing the + cell immediately. Previously we would mark the connection for + close, but continue processing the cell as if the connection were + open. Fixes bug 26072; bugfix on 0.2.4.7-alpha. + + o Minor bugfixes (directory authorities, IPv6): + - When creating a routerstatus (vote) from a routerinfo (descriptor), + set the IPv6 address to the unspecified IPv6 address, and + explicitly initialize the port to zero. Fixes bug 24488; bugfix + on 0.2.4.1-alpha. + + o Minor bugfixes (documentation): + - Document that the PerConnBW{Rate,Burst} options will fall back to + their corresponding consensus parameters only if those parameters + are set. Previously we had claimed that these values would always + be set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha. + + o Minor bugfixes (documentation, backport from 0.3.4.1-alpha): + - Stop saying in the manual that clients cache ipv4 dns answers from + exit relays. We haven't used them since 0.2.6.3-alpha, and in + ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, but + we forgot to say so in the man page. Fixes bug 26052; bugfix + on 0.3.2.6-alpha. + + o Minor bugfixes (exit relay DNS retries): + - Re-attempt timed-out DNS queries 3 times before failure, since our + timeout is 5 seconds for them, but clients wait 10-15. Also allow + slightly more timeouts per resolver when an exit has multiple + resolvers configured. Fixes bug 21394; bugfix on 0.3.1.9. + + o Minor bugfixes (fallback directory mirrors): + - Make updateFallbackDirs.py search harder for python. (Some OSs + don't put it in /usr/bin.) Fixes bug 24708; bugfix + on 0.2.8.1-alpha. + + o Minor bugfixes (hibernation, bandwidth accounting, shutdown): + - When hibernating, close connections normally and allow them to + flush. Fixes bug 23571; bugfix on 0.2.4.7-alpha. Also fixes + bug 7267. + - Do not attempt to launch self-reachability tests when entering + hibernation. Fixes a case of bug 12062; bugfix on 0.0.9pre5. + - Resolve several bugs related to descriptor fetching on bridge + clients with bandwidth accounting enabled. (This combination is + not recommended!) Fixes a case of bug 12062; bugfix + on 0.2.0.3-alpha. + - When hibernating, do not attempt to launch DNS checks. Fixes a + case of bug 12062; bugfix on 0.1.2.2-alpha. + - When hibernating, do not try to upload or download descriptors. + Fixes a case of bug 12062; bugfix on 0.0.9pre5. + + o Minor bugfixes (IPv6, bridges): + - Tor now always sets IPv6 preferences for bridges. Fixes bug 24573; + bugfix on 0.2.8.2-alpha. + - Tor now sets IPv6 address in the routerstatus as well as in the + router descriptors when updating addresses for a bridge. Closes + ticket 24572; bugfix on 0.2.4.5-alpha. Patch by "ffmancera". + + o Minor bugfixes (Linux seccomp2 sandbox): + - When running with the sandbox enabled, reload configuration files + correctly even when %include was used. Previously we would crash. + Fixes bug 22605; bugfix on 0.3.1. Patch from Daniel Pinto. + + o Minor bugfixes (Linux seccomp2 sandbox, backport from 0.3.4.1-alpha): + - Allow the nanosleep() system call, which glibc uses to implement + sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha. + + o Minor bugfixes (logging): + - Fix a (mostly harmless) race condition when invoking + LOG_PROTOCOL_WARN message from a subthread while the torrc options + are changing. Fixes bug 23954; bugfix on 0.1.1.9-alpha. + + o Minor bugfixes (man page, SocksPort): + - Remove dead code from the old "SocksSocket" option, and rename + SocksSocketsGroupWritable to UnixSocksGroupWritable. The old + option still works, but is deprecated. Fixes bug 24343; bugfix + on 0.2.6.3. + + o Minor bugfixes (memory leaks): + - Avoid possible at-exit memory leaks related to use of Libevent's + event_base_once() function. (This function tends to leak memory if + the event_base is closed before the event fires.) Fixes bug 24584; + bugfix on 0.2.8.1-alpha. + - Fix a harmless memory leak in tor-resolve. Fixes bug 24582; bugfix + on 0.2.1.1-alpha. + + o Minor bugfixes (network IPv6 test): + - Tor's test scripts now check if "ping -6 ::1" works when the user + runs "make test-network-all". Fixes bug 24677; bugfix on + 0.2.9.3-alpha. Patch by "ffmancera". + + o Minor bugfixes (networking): + - string_is_valid_hostname() will not consider IP strings to be + valid hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5. + + o Minor bugfixes (onion service v3): + - Avoid an assertion failure when the next onion service descriptor + rotation type is out of sync with the consensus's valid-after + time. Instead, log a warning message with extra information, so we + can better hunt down the cause of this assertion. Fixes bug 25306; + bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (onion service, backport from 0.3.4.1-alpha): + - Fix a memory leak when a v3 onion service is configured and gets a + SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha. + - When parsing the descriptor signature, look for the token plus an + extra white-space at the end. This is more correct but also will + allow us to support new fields that might start with "signature". + Fixes bug 26069; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (onion services): + - If we are configured to offer a single onion service, don't log + long-term established one hop rendezvous points in the heartbeat. + Fixes bug 25116; bugfix on 0.2.9.6-rc. + + o Minor bugfixes (performance): + - Reduce the number of circuits that will be opened at once during + the circuit build timeout phase. This is done by increasing the + idle timeout to 3 minutes, and lowering the maximum number of + concurrent learning circuits to 10. Fixes bug 24769; bugfix + on 0.3.1.1-alpha. + - Avoid calling protocol_list_supports_protocol() from inside tight + loops when running with cached routerinfo_t objects. Instead, + summarize the relevant protocols as flags in the routerinfo_t, as + we do for routerstatus_t objects. This change simplifies our code + a little, and saves a large amount of short-term memory allocation + operations. Fixes bug 25008; bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (performance, timeouts): + - Consider circuits for timeout as soon as they complete a hop. This + is more accurate than applying the timeout in + circuit_expire_building() because that function is only called + once per second, which is now too slow for typical timeouts on the + current network. Fixes bug 23114; bugfix on 0.2.2.2-alpha. + - Use onion service circuits (and other circuits longer than 3 hops) + to calculate a circuit build timeout. Previously, Tor only + calculated its build timeout based on circuits that planned to be + exactly 3 hops long. With this change, we include measurements + from all circuits at the point where they complete their third + hop. Fixes bug 23100; bugfix on 0.2.2.2-alpha. + + o Minor bugfixes (relay, crash, backport from 0.3.4.1-alpha): + - Avoid a crash when running with DirPort set but ORPort tuned off. + Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (Rust FFI): + - Fix a minor memory leak which would happen whenever the C code + would call the Rust implementation of + protover_get_supported_protocols(). This was due to the C version + returning a static string, whereas the Rust version newly allocated + a CString to pass across the FFI boundary. Consequently, the C + code was not expecting to need to free() what it was given. Fixes + bug 25127; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (spelling): + - Use the "misspell" tool to detect and fix typos throughout the + source code. Fixes bug 23650; bugfix on various versions of Tor. + Patch from Deepesh Pathak. + + o Minor bugfixes (testing): + - Avoid intermittent test failures due to a test that had relied on + onion service introduction point creation finishing within 5 + seconds of real clock time. Fixes bug 25450; bugfix + on 0.3.1.3-alpha. + - Give out Exit flags in bootstrapping networks. Fixes bug 24137; + bugfix on 0.2.3.1-alpha. + + o Minor bugfixes (unit test, monotonic time): + - Increase a constant (1msec to 10msec) in the monotonic time test + that makes sure the nsec/usec/msec times read are synchronized. + This change was needed to accommodate slow systems like armel or + when the clock_gettime() is not a VDSO on the running kernel. + Fixes bug 25113; bugfix on 0.2.9.1. + + o Code simplification and refactoring: + - Move the list of default directory authorities to its own file. + Closes ticket 24854. Patch by "beastr0". + - Remove the old (deterministic) directory retry logic entirely: + We've used exponential backoff exclusively for some time. Closes + ticket 23814. + - Remove the unused nodelist_recompute_all_hsdir_indices(). Closes + ticket 25108. + - Remove a series of counters used to track circuit extend attempts + and connection status but that in reality we aren't using for + anything other than stats logged by a SIGUSR1 signal. Closes + ticket 25163. + - Remove /usr/athena from search path in configure.ac. Closes + ticket 24363. + - Remove duplicate code in node_has_curve25519_onion_key() and + node_get_curve25519_onion_key(), and add a check for a zero + microdesc curve25519 onion key. Closes ticket 23966, patch by + "aruna1234" and teor. + - Rewrite channel_rsa_id_group_set_badness to reduce temporary + memory allocations with large numbers of OR connections (e.g. + relays). Closes ticket 24119. + - Separate the function that deletes ephemeral files when Tor + stops gracefully. + - Small changes to Tor's buf_t API to make it suitable for use as a + general-purpose safe string constructor. Closes ticket 22342. + - Switch -Wnormalized=id to -Wnormalized=nfkc in configure.ac to + avoid source code identifier confusion. Closes ticket 24467. + - The tor_git_revision[] constant no longer needs to be redeclared + by everything that links against the rest of Tor. Done as part of + ticket 23845, to simplify our external API. + - We make extend_info_from_node() use node_get_curve25519_onion_key() + introduced in ticket 23577 to access the curve25519 public keys + rather than accessing it directly. Closes ticket 23760. Patch by + Neel Chauhan. + - Add a function to log channels' scheduler state changes to aid + debugging efforts. Closes ticket 24531. + + o Documentation: + - Improved the documentation of AccountingStart parameter. Closes + ticket 23635. + - Update the documentation for "Log" to include the current list of + logging domains. Closes ticket 25378. + - Add documentation on how to build tor with Rust dependencies + without having to be online. Closes ticket 22907; bugfix + on 0.3.0.3-alpha. + - Clarify the behavior of RelayBandwidth{Rate,Burst} with client + traffic. Closes ticket 24318. + - Document that OutboundBindAddress doesn't apply to DNS requests. + Closes ticket 22145. Patch from Aruna Maurya. + + o Code simplification and refactoring (channels): + - Remove the incoming and outgoing channel queues. These were never + used, but still took up a step in our fast path. + - The majority of the channel unit tests have been rewritten and the + code coverage has now been raised to 83.6% for channel.c. Closes + ticket 23709. + - Remove other dead code from the channel subsystem: All together, + this cleanup has removed more than 1500 lines of code overall and + adding very little except for unit test. + + o Code simplification and refactoring (circuit rendezvous): + - Split the client-side rendezvous circuit lookup into two + functions: one that returns only established circuits and another + that returns all kinds of circuits. Closes ticket 23459. + + o Code simplification and refactoring (controller): + - Make most of the variables in networkstatus_getinfo_by_purpose() + const. Implements ticket 24489. + + o Documentation (backport from 0.3.4.1-alpha): + - Correct an IPv6 error in the documentation for ExitPolicy. Closes + ticket 25857. Patch from "CTassisF". + + o Documentation (man page): + - The HiddenServiceVersion torrc option accepts only one number: + either version 2 or 3. Closes ticket 25026; bugfix + on 0.3.2.2-alpha. + + o Documentation (manpage, denial of service): + - Provide more detail about the denial-of-service options, by + listing each mitigation and explaining how they relate. Closes + ticket 25248. + + +Changes in version 0.3.1.10 - 2018-03-03 + Tor 0.3.1.10 backports a number of bugfixes, including important fixes for + security issues. + + It includes an important security fix for a remote crash attack + against directory authorities, tracked as TROVE-2018-001. + + This release also backports our new system for improved resistance to + denial-of-service attacks against relays. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + All directory authorities should upgrade to one of the versions + released today. Relays running 0.3.1.x may wish to update to one of + the versions released today, for the DoS mitigations. + + Please note: according to our release calendar, Tor 0.3.1 will no + longer be supported after 1 July 2018. If you will be running Tor + after that date, you should make sure to plan to upgrade to the latest + stable version, or downgrade to 0.2.9 (which will receive long-term + support). + + o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha): + - Give relays some defenses against the recent network overload. We + start with three defenses (default parameters in parentheses). + First: if a single client address makes too many concurrent + connections (>100), hang up on further connections. Second: if a + single client address makes circuits too quickly (more than 3 per + second, with an allowed burst of 90) while also having too many + connections open (3), refuse new create cells for the next while + (1-2 hours). Third: if a client asks to establish a rendezvous + point to you directly, ignore the request. These defenses can be + manually controlled by new torrc options, but relays will also + take guidance from consensus parameters, so there's no need to + configure anything manually. Implements ticket 24902. + + o Minor features (linux seccomp2 sandbox, backport from 0.3.2.5-alpha): + - Update the sandbox rules so that they should now work correctly + with Glibc 2.26. Closes ticket 24315. + + o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha): + - Fix an "off by 2" error in counting rendezvous failures on the + onion service side. While we thought we would stop the rendezvous + attempt after one failed circuit, we were actually making three + circuit attempts before giving up. Now switch to a default of 2, + and allow the consensus parameter "hs_service_max_rdv_failures" to + override. Fixes bug 24895; bugfix on 0.0.6. + + o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha): + - Add Link protocol version 5 to the supported protocols list. Fixes + bug 25070; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (relay, backport from 0.3.3.1-alpha): + - Fix a set of false positives where relays would consider + connections to other relays as being client-only connections (and + thus e.g. deserving different link padding schemes) if those + relays fell out of the consensus briefly. Now we look only at the + initial handshake and whether the connection authenticated as a + relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha. + + o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha): + - Make our OOM handler aware of the geoip client history cache so it + doesn't fill up the memory. This check is important for IPv6 and + our DoS mitigation subsystem. Closes ticket 25122. + + o Minor feature (relay statistics, backport from 0.3.2.6-alpha): + - Change relay bandwidth reporting stats interval from 4 hours to 24 + hours in order to reduce the efficiency of guard discovery + attacks. Fixes ticket 23856. + + o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (fallback directory mirrors, backport from 0.3.2.9): + - The fallback directory list has been re-generated based on the + current status of the network. Tor uses fallback directories to + bootstrap when it doesn't yet have up-to-date directory + information. Closes ticket 24801. + - Make the default DirAuthorityFallbackRate 0.1, so that clients + prefer to bootstrap from fallback directory mirrors. This is a + follow-up to 24679, which removed weights from the default + fallbacks. Implements ticket 24681. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2 + Country database. + + o Minor bugfix (channel connection, backport from 0.3.3.2-alpha): + - Use the actual observed address of an incoming relay connection, + not the canonical address of the relay from its descriptor, when + making decisions about how to handle the incoming connection. + Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera". + + o Minor bugfix (directory authority, backport from 0.3.3.2-alpha): + - Directory authorities, when refusing a descriptor from a rejected + relay, now explicitly tell the relay (in its logs) to set a valid + ContactInfo address and contact the bad-relays@ mailing list. + Fixes bug 25170; bugfix on 0.2.9.1. + + o Minor bugfixes (address selection, backport from 0.3.2.9): + - When the fascist_firewall_choose_address_ functions don't find a + reachable address, set the returned address to the null address + and port. This is a precautionary measure, because some callers do + not check the return value. Fixes bug 24736; bugfix + on 0.2.8.2-alpha. + + o Major bugfixes (bootstrapping, backport from 0.3.2.5-alpha): + - Fetch descriptors aggressively whenever we lack enough to build + circuits, regardless of how many descriptors we are missing. + Previously, we would delay launching the fetch when we had fewer + than 15 missing descriptors, even if some of those descriptors + were blocking circuits from building. Fixes bug 23985; bugfix on + 0.1.1.11-alpha. The effects of this bug became worse in + 0.3.0.3-alpha, when we began treating missing descriptors from our + primary guards as a reason to delay circuits. + - Don't try fetching microdescriptors from relays that have failed + to deliver them in the past. Fixes bug 23817; bugfix + on 0.3.0.1-alpha. + + o Minor bugfixes (compilation, backport from 0.3.2.7-rc): + - Fix a signed/unsigned comparison warning introduced by our fix to + TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16. + + o Minor bugfixes (control port, linux seccomp2 sandbox, backport from 0.3.2.5-alpha): + - Avoid a crash when attempting to use the seccomp2 sandbox together + with the OwningControllerProcess feature. Fixes bug 24198; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfixes (directory cache, backport from 0.3.2.5-alpha): + - Recover better from empty or corrupt files in the consensus cache + directory. Fixes bug 24099; bugfix on 0.3.1.1-alpha. + - When a consensus diff calculation is only partially successful, + only record the successful parts as having succeeded. Partial + success can happen if (for example) one compression method fails + but the others succeed. Previously we misrecorded all the + calculations as having succeeded, which would later cause a + nonfatal assertion failure. Fixes bug 24086; bugfix + on 0.3.1.1-alpha. + + o Minor bugfixes (entry guards, backport from 0.3.2.3-alpha): + - Tor now updates its guard state when it reads a consensus + regardless of whether it's missing descriptors. That makes tor use + its primary guards to fetch descriptors in some edge cases where + it would previously have used fallback directories. Fixes bug + 23862; bugfix on 0.3.0.1-alpha. + + o Minor bugfixes (logging, backport from 0.3.3.2-alpha): + - Don't treat inability to store a cached consensus object as a bug: + it can happen normally when we are out of disk space. Fixes bug + 24859; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (memory usage, backport from 0.3.2.8-rc): + - When queuing DESTROY cells on a channel, only queue the circuit-id + and reason fields: not the entire 514-byte cell. This fix should + help mitigate any bugs or attacks that fill up these queues, and + free more RAM for other uses. Fixes bug 24666; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (network layer, backport from 0.3.2.5-alpha): + - When closing a connection via close_connection_immediately(), we + mark it as "not blocked on bandwidth", to prevent later calls from + trying to unblock it, and give it permission to read. This fixes a + backtrace warning that can happen on relays under various + circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc. + + o Minor bugfixes (path selection, backport from 0.3.2.4-alpha): + - When selecting relays by bandwidth, avoid a rounding error that + could sometimes cause load to be imbalanced incorrectly. + Previously, we would always round upwards; now, we round towards + the nearest integer. This had the biggest effect when a relay's + weight adjustments should have given it weight 0, but it got + weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha. + - When calculating the fraction of nodes that have descriptors, and + all nodes in the network have zero bandwidths, count the number of + nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha. + - Actually log the total bandwidth in compute_weighted_bandwidths(). + Fixes bug 24170; bugfix on 0.2.4.3-alpha. + + o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha): + - Improve the performance of our consensus-diff application code + when Tor is built with the --enable-fragile-hardening option set. + Fixes bug 24826; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (OSX, backport from 0.3.3.1-alpha): + - Don't exit the Tor process if setrlimit() fails to change the file + limit (which can happen sometimes on some versions of OSX). Fixes + bug 21074; bugfix on 0.0.9pre5. + + o Minor bugfixes (portability, msvc, backport from 0.3.2.9): + - Fix a bug in the bit-counting parts of our timing-wheel code on + MSVC. (Note that MSVC is still not a supported build platform, due + to cyptographic timing channel risks.) Fixes bug 24633; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (relay, partial backport): + - Make the internal channel_is_client() function look at what sort + of connection handshake the other side used, rather than whether + the other side ever sent a create_fast cell to us. Backports part + of the fixes from bugs 22805 and 24898. + + o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + o Code simplification and refactoring (backport from 0.3.3.3-alpha): + - Update the "rust dependencies" submodule to be a project-level + repository, rather than a user repository. Closes ticket 25323. + + +Changes in version 0.2.9.15 - 2018-03-03 + Tor 0.2.9.15 backports important security and stability bugfixes from + later Tor releases. + + It includes an important security fix for a remote crash attack + against directory authorities, tracked as TROVE-2018-001. + + This release also backports our new system for improved resistance to + denial-of-service attacks against relays. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + All directory authorities should upgrade to one of the versions + released today. Relays running 0.2.9.x may wish to update to one of + the versions released today, for the DoS mitigations. + + o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Major features (denial-of-service mitigation): + - Give relays some defenses against the recent network overload. We + start with three defenses (default parameters in parentheses). + First: if a single client address makes too many concurrent + connections (>100), hang up on further connections. Second: if a + single client address makes circuits too quickly (more than 3 per + second, with an allowed burst of 90) while also having too many + connections open (3), refuse new create cells for the next while + (1-2 hours). Third: if a client asks to establish a rendezvous + point to you directly, ignore the request. These defenses can be + manually controlled by new torrc options, but relays will also + take guidance from consensus parameters, so there's no need to + configure anything manually. Implements ticket 24902. + + o Major bugfixes (bootstrapping): + - Fetch descriptors aggressively whenever we lack enough to build + circuits, regardless of how many descriptors we are missing. + Previously, we would delay launching the fetch when we had fewer + than 15 missing descriptors, even if some of those descriptors + were blocking circuits from building. Fixes bug 23985; bugfix on + 0.1.1.11-alpha. The effects of this bug became worse in + 0.3.0.3-alpha, when we began treating missing descriptors from our + primary guards as a reason to delay circuits. + + o Major bugfixes (onion services, retry behavior): + - Fix an "off by 2" error in counting rendezvous failures on the + onion service side. While we thought we would stop the rendezvous + attempt after one failed circuit, we were actually making three + circuit attempts before giving up. Now switch to a default of 2, + and allow the consensus parameter "hs_service_max_rdv_failures" to + override. Fixes bug 24895; bugfix on 0.0.6. + + o Minor feature (relay statistics): + - Change relay bandwidth reporting stats interval from 4 hours to 24 + hours in order to reduce the efficiency of guard discovery + attacks. Fixes ticket 23856. + + o Minor features (compatibility, OpenSSL): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (denial-of-service avoidance): + - Make our OOM handler aware of the geoip client history cache so it + doesn't fill up the memory. This check is important for IPv6 and + our DoS mitigation subsystem. Closes ticket 25122. + + o Minor features (fallback directory mirrors): + - The fallback directory list has been re-generated based on the + current status of the network. Tor uses fallback directories to + bootstrap when it doesn't yet have up-to-date directory + information. Closes ticket 24801. + - Make the default DirAuthorityFallbackRate 0.1, so that clients + prefer to bootstrap from fallback directory mirrors. This is a + follow-up to 24679, which removed weights from the default + fallbacks. Implements ticket 24681. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2 + Country database. + + o Minor features (linux seccomp2 sandbox): + - Update the sandbox rules so that they should now work correctly + with Glibc 2.26. Closes ticket 24315. + + o Minor bugfix (channel connection): + - Use the actual observed address of an incoming relay connection, + not the canonical address of the relay from its descriptor, when + making decisions about how to handle the incoming connection. + Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera". + + o Minor bugfix (directory authority): + - Directory authorities, when refusing a descriptor from a rejected + relay, now explicitly tell the relay (in its logs) to set a valid + ContactInfo address and contact the bad-relays@ mailing list. + Fixes bug 25170; bugfix on 0.2.9.1. + + o Minor bugfixes (address selection): + - When the fascist_firewall_choose_address_ functions don't find a + reachable address, set the returned address to the null address + and port. This is a precautionary measure, because some callers do + not check the return value. Fixes bug 24736; bugfix + on 0.2.8.2-alpha. + + o Minor bugfixes (compilation): + - Fix a signed/unsigned comparison warning introduced by our fix to + TROVE-2017-009. Fixes bug 24480; bugfix on 0.2.5.16. + + o Minor bugfixes (control port, linux seccomp2 sandbox): + - Avoid a crash when attempting to use the seccomp2 sandbox together + with the OwningControllerProcess feature. Fixes bug 24198; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfixes (memory usage): + - When queuing DESTROY cells on a channel, only queue the circuit-id + and reason fields: not the entire 514-byte cell. This fix should + help mitigate any bugs or attacks that fill up these queues, and + free more RAM for other uses. Fixes bug 24666; bugfix + on 0.2.5.1-alpha. + + o Minor bugfixes (network layer): + - When closing a connection via close_connection_immediately(), we + mark it as "not blocked on bandwidth", to prevent later calls from + trying to unblock it, and give it permission to read. This fixes a + backtrace warning that can happen on relays under various + circumstances. Fixes bug 24167; bugfix on 0.1.0.1-rc. + + o Minor bugfixes (OSX): + - Don't exit the Tor process if setrlimit() fails to change the file + limit (which can happen sometimes on some versions of OSX). Fixes + bug 21074; bugfix on 0.0.9pre5. + + o Minor bugfixes (path selection): + - When selecting relays by bandwidth, avoid a rounding error that + could sometimes cause load to be imbalanced incorrectly. + Previously, we would always round upwards; now, we round towards + the nearest integer. This had the biggest effect when a relay's + weight adjustments should have given it weight 0, but it got + weight 1 instead. Fixes bug 23318; bugfix on 0.2.4.3-alpha. + - When calculating the fraction of nodes that have descriptors, and + all nodes in the network have zero bandwidths, count the number of + nodes instead. Fixes bug 23318; bugfix on 0.2.4.10-alpha. + - Actually log the total bandwidth in compute_weighted_bandwidths(). + Fixes bug 24170; bugfix on 0.2.4.3-alpha. + + o Minor bugfixes (portability, msvc): + - Fix a bug in the bit-counting parts of our timing-wheel code on + MSVC. (Note that MSVC is still not a supported build platform, due + to cryptographic timing channel risks.) Fixes bug 24633; bugfix + on 0.2.9.1-alpha. + + o Minor bugfixes (relay): + - Make the internal channel_is_client() function look at what sort + of connection handshake the other side used, rather than whether + the other side ever sent a create_fast cell to us. Backports part + of the fixes from bugs 22805 and 24898. + + o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + +Changes in version 0.3.2.10 - 2018-03-03 + Tor 0.3.2.10 is the second stable release in the 0.3.2 series. It + backports a number of bugfixes, including important fixes for security + issues. + + It includes an important security fix for a remote crash attack + against directory authorities, tracked as TROVE-2018-001. + + Additionally, it backports a fix for a bug whose severity we have + upgraded: Bug 24700, which was fixed in 0.3.3.2-alpha, can be remotely + triggered in order to crash relays with a use-after-free pattern. As + such, we are now tracking that bug as TROVE-2018-002 and + CVE-2018-0491, and backporting it to earlier releases. This bug + affected versions 0.3.2.1-alpha through 0.3.2.9, as well as version + 0.3.3.1-alpha. + + This release also backports our new system for improved resistance to + denial-of-service attacks against relays. + + This release also fixes several minor bugs and annoyances from + earlier releases. + + Relays running 0.3.2.x SHOULD upgrade to one of the versions released + today, for the fix to TROVE-2018-002. Directory authorities should + also upgrade. (Relays on earlier versions might want to update too for + the DoS mitigations.) + + o Major bugfixes (denial-of-service, directory authority, backport from 0.3.3.3-alpha): + - Fix a protocol-list handling bug that could be used to remotely crash + directory authorities with a null-pointer exception. Fixes bug 25074; + bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001 and + CVE-2018-0490. + + o Major bugfixes (scheduler, KIST, denial-of-service, backport from 0.3.3.2-alpha): + - Avoid adding the same channel twice in the KIST scheduler pending + list, which could lead to remote denial-of-service use-after-free + attacks against relays. Fixes bug 24700; bugfix on 0.3.2.1-alpha. + + o Major features (denial-of-service mitigation, backport from 0.3.3.2-alpha): + - Give relays some defenses against the recent network overload. We + start with three defenses (default parameters in parentheses). + First: if a single client address makes too many concurrent + connections (>100), hang up on further connections. Second: if a + single client address makes circuits too quickly (more than 3 per + second, with an allowed burst of 90) while also having too many + connections open (3), refuse new create cells for the next while + (1-2 hours). Third: if a client asks to establish a rendezvous + point to you directly, ignore the request. These defenses can be + manually controlled by new torrc options, but relays will also + take guidance from consensus parameters, so there's no need to + configure anything manually. Implements ticket 24902. + + o Major bugfixes (onion services, retry behavior, backport from 0.3.3.1-alpha): + - Fix an "off by 2" error in counting rendezvous failures on the + onion service side. While we thought we would stop the rendezvous + attempt after one failed circuit, we were actually making three + circuit attempts before giving up. Now switch to a default of 2, + and allow the consensus parameter "hs_service_max_rdv_failures" to + override. Fixes bug 24895; bugfix on 0.0.6. + - New-style (v3) onion services now obey the "max rendezvous circuit + attempts" logic. Previously they would make as many rendezvous + circuit attempts as they could fit in the MAX_REND_TIMEOUT second + window before giving up. Fixes bug 24894; bugfix on 0.3.2.1-alpha. + + o Major bugfixes (protocol versions, backport from 0.3.3.2-alpha): + - Add Link protocol version 5 to the supported protocols list. Fixes + bug 25070; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (relay, backport from 0.3.3.1-alpha): + - Fix a set of false positives where relays would consider + connections to other relays as being client-only connections (and + thus e.g. deserving different link padding schemes) if those + relays fell out of the consensus briefly. Now we look only at the + initial handshake and whether the connection authenticated as a + relay. Fixes bug 24898; bugfix on 0.3.1.1-alpha. + + o Major bugfixes (scheduler, consensus, backport from 0.3.3.2-alpha): + - The scheduler subsystem was failing to promptly notice changes in + consensus parameters, making it harder to switch schedulers + network-wide. Fixes bug 24975; bugfix on 0.3.2.1-alpha. + + o Minor features (denial-of-service avoidance, backport from 0.3.3.2-alpha): + - Make our OOM handler aware of the geoip client history cache so it + doesn't fill up the memory. This check is important for IPv6 and + our DoS mitigation subsystem. Closes ticket 25122. + + o Minor features (compatibility, OpenSSL, backport from 0.3.3.3-alpha): + - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. + Previous versions of Tor would not have worked with OpenSSL 1.1.1, + since they neither disabled TLS 1.3 nor enabled any of the + ciphersuites it requires. Now we enable the TLS 1.3 ciphersuites. + Closes ticket 24978. + + o Minor features (geoip): + - Update geoip and geoip6 to the February 7 2018 Maxmind GeoLite2 + Country database. + + o Minor features (logging, diagnostic, backport from 0.3.3.2-alpha): + - When logging a failure to create an onion service's descriptor, + also log what the problem with the descriptor was. Diagnostic + for ticket 24972. + + o Minor bugfix (channel connection, backport from 0.3.3.2-alpha): + - Use the actual observed address of an incoming relay connection, + not the canonical address of the relay from its descriptor, when + making decisions about how to handle the incoming connection. + Fixes bug 24952; bugfix on 0.2.4.11-alpha. Patch by "ffmancera". + + o Minor bugfixes (denial-of-service, backport from 0.3.3.3-alpha): + - Fix a possible crash on malformed consensus. If a consensus had + contained an unparseable protocol line, it could have made clients + and relays crash with a null-pointer exception. To exploit this + issue, however, an attacker would need to be able to subvert the + directory authority system. Fixes bug 25251; bugfix on + 0.2.9.4-alpha. Also tracked as TROVE-2018-004. + + o Minor bugfix (directory authority, backport from 0.3.3.2-alpha): + - Directory authorities, when refusing a descriptor from a rejected + relay, now explicitly tell the relay (in its logs) to set a valid + ContactInfo address and contact the bad-relays@ mailing list. + Fixes bug 25170; bugfix on 0.2.9.1. + + o Minor bugfixes (build, rust, backport from 0.3.3.1-alpha): + - When building with Rust on OSX, link against libresolv, to work + around the issue at https://github.com/rust-lang/rust/issues/46797. + Fixes bug 24652; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (onion services, backport from 0.3.3.2-alpha): + - Remove a BUG() statement when a client fetches an onion descriptor + that has a lower revision counter than the one in its cache. This + can happen in normal circumstances due to HSDir desync. Fixes bug + 24976; bugfix on 0.3.2.1-alpha. + + o Minor bugfixes (logging, backport from 0.3.3.2-alpha): + - Don't treat inability to store a cached consensus object as a bug: + it can happen normally when we are out of disk space. Fixes bug + 24859; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (performance, fragile-hardening, backport from 0.3.3.1-alpha): + - Improve the performance of our consensus-diff application code + when Tor is built with the --enable-fragile-hardening option set. + Fixes bug 24826; bugfix on 0.3.1.1-alpha. + + o Minor bugfixes (OSX, backport from 0.3.3.1-alpha): + - Don't exit the Tor process if setrlimit() fails to change the file + limit (which can happen sometimes on some versions of OSX). Fixes + bug 21074; bugfix on 0.0.9pre5. + + o Minor bugfixes (spec conformance, backport from 0.3.3.3-alpha): + - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on + 0.2.9.4-alpha. + - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; + bugfix on 0.2.9.4-alpha. + + o Minor bugfixes (testing, backport from 0.3.3.1-alpha): + - Fix a memory leak in the scheduler/loop_kist unit test. Fixes bug + 25005; bugfix on 0.3.2.7-rc. + + o Minor bugfixes (v3 onion services, backport from 0.3.3.2-alpha): + - Look at the "HSRend" protocol version, not the "HSDir" protocol + version, when deciding whether a consensus entry can support the + v3 onion service protocol as a rendezvous point. Fixes bug 25105; + bugfix on 0.3.2.1-alpha. + + o Code simplification and refactoring (backport from 0.3.3.3-alpha): + - Update the "rust dependencies" submodule to be a project-level + repository, rather than a user repository. Closes ticket 25323. + + o Documentation (backport from 0.3.3.1-alpha) + - Document that operators who run more than one relay or bridge are + expected to set MyFamily and ContactInfo correctly. Closes + ticket 24526. + Changes in version 0.3.2.9 - 2018-01-09 Tor 0.3.2.9 is the first stable release in the 0.3.2 series. diff --git a/changes/25857 b/changes/25857 deleted file mode 100644 index e457af2f82..0000000000 --- a/changes/25857 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Correct an IPv6 error in the documentation for ExitPolicy. - Closes ticket 25857. Patch from "CTassisF". diff --git a/changes/TROVE-2018-005 b/changes/TROVE-2018-005 deleted file mode 100644 index 769c653f43..0000000000 --- a/changes/TROVE-2018-005 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (security, directory authority, denial-of-service): - - Fix a bug that could have allowed an attacker to force a - directory authority to use up all its RAM by passing it a - maliciously crafted protocol versions string. Fixes bug 25517; - bugfix on 0.2.9.4-alpha. This issue is also tracked as - TROVE-2018-005. diff --git a/changes/bug21394.2 b/changes/bug21394.2 deleted file mode 100644 index b580d2a786..0000000000 --- a/changes/bug21394.2 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfix (Exit node DNS retries): - - Re-attempt timed-out DNS queries 3 times before failure, since our - timeout is 5 seconds for them, but clients wait 10-15. Also allow - slightly more timeouts per resolver before giving up on it in the - case where an exit has multiple resolvers configured. Fixes bug 21394; - bugfix on 0.3.1.9. - diff --git a/changes/bug22310 b/changes/bug22310 deleted file mode 100644 index c8017daffe..0000000000 --- a/changes/bug22310 +++ /dev/null @@ -1,8 +0,0 @@ - o Major bugfixes (performance, load balancing): - - Directory authorities no longer vote in favor of the Guard flag - for relays that don't advertise directory support. Starting in Tor - 0.3.0.1-alpha, Tor clients have been avoiding using such relays in - the Guard position, leading to increasingly broken load balancing - for the 5%-or-so of Guards that don't advertise directory support. - Fixes bug 22310; bugfix on 0.3.0.6. - diff --git a/changes/bug23693.1 b/changes/bug23693.1 deleted file mode 100644 index 4b16788814..0000000000 --- a/changes/bug23693.1 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (relay, crash): - - Avoid a crash when running with DirPort set but ORPort tuned off. - Fixes a case of bug 23693; bugfix on 0.3.1.1-alpha. - diff --git a/changes/bug24031 b/changes/bug24031 deleted file mode 100644 index 2bb0e83091..0000000000 --- a/changes/bug24031 +++ /dev/null @@ -1,13 +0,0 @@ - o Major bugfixes (protover, voting): - - Revise Rust implementation of protover to use a more memory-efficient - voting algorithm and corresponding data structures, thus avoiding a - potential (but small impact) DoS attack where specially crafted protocol - strings would expand to several potential megabytes in memory. In the - process, several portions of code were revised to be methods on new, - custom types, rather than functions taking interchangeable types, thus - increasing type safety of the module. Custom error types and handling - were added as well, in order to facilitate better error dismissal/handling - in outside crates and avoid mistakenly passing an internal error string to - C over the FFI boundary. Many tests were added, and some previous - differences between the C and Rust implementations have been - remedied. Fixes bug 24031; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug24767 b/changes/bug24767 deleted file mode 100644 index 56fbe51a98..0000000000 --- a/changes/bug24767 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (relay, connection): - - Refuse to connect again to a relay from which we failed previously with - a connection refused, timeout or error (at the TCP level). The relay - won't be retried for 60 seconds after the failure occured. Fixes bug - 24767; bugfix on 0.0.6. diff --git a/changes/bug24769 b/changes/bug24769 deleted file mode 100644 index 2893e0ff07..0000000000 --- a/changes/bug24769 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (performance): - - Reduce the number of circuits that can be opened at once during the - circuit build timeout phase. This is done by increasing the idle timeout - to 3 minutes, and lowering the maximum number of concurrent learning - circuits to 10. Fixes bug 24769; bugfix on 0.3.1.1-alpha. - - diff --git a/changes/bug24782 b/changes/bug24782 deleted file mode 100644 index 59bbdad123..0000000000 --- a/changes/bug24782 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (config options): - - Change the way the default value for MaxMemInQueues is calculated. We now - use 0.4 * RAM if the system have 8 GB RAM or more, otherwise we use the - former value of 0.75 * RAM. Closes ticket 24782. diff --git a/changes/bug24854 b/changes/bug24854 deleted file mode 100644 index 64e10772e0..0000000000 --- a/changes/bug24854 +++ /dev/null @@ -1,3 +0,0 @@ - o Code simplification and refactoring: - - Move the list of default directory authorities to their own file for - inclusion using the C preprocessor. Closes ticket 24854. Patch by "beastr0". diff --git a/changes/bug24891 b/changes/bug24891 new file mode 100644 index 0000000000..403b2b1123 --- /dev/null +++ b/changes/bug24891 @@ -0,0 +1,4 @@ + o Minor bugfixes (compilation): + - Refrain from compiling unit testing related object files + when --disable-unittests is set to configure script. + Fixes bug 24891; bugfix on 0.2.5.1-alpha. diff --git a/changes/bug24903 b/changes/bug24903 deleted file mode 100644 index 01c9b53f23..0000000000 --- a/changes/bug24903 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (controller, reliability): - - Avoid a (nonfatal) assertion failure when extending a one-hop circuit - from the controller to become a multihop circuit. Fixes bug 24903; - bugfix on 0.2.5.2-alpha. - diff --git a/changes/bug24904 b/changes/bug24904 deleted file mode 100644 index 648d9a5834..0000000000 --- a/changes/bug24904 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfix (channel, client): - - Better identify client connection when reporting to the geoip client - cache. Fixes bug 24904; bugfix on 0.3.1.7. - diff --git a/changes/bug24969 b/changes/bug24969 deleted file mode 100644 index 46b2bae6f4..0000000000 --- a/changes/bug24969 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (Linux seccomp2 sandbox): - - Allow the nanosleep() system call, which glibc uses to implement - sleep() and usleep(). Fixes bug 24969; bugfix on 0.2.5.1-alpha. diff --git a/changes/bug24977 b/changes/bug24977 new file mode 100644 index 0000000000..f8127a2a73 --- /dev/null +++ b/changes/bug24977 @@ -0,0 +1,5 @@ + o Minor bugfixes (onion services): + - Recompute some consensus information after clock skews or when we + transition from a non-live consensus to a live consensus. We do this to + avoid having an outdated state which could impact next-generation onion + services. Fixes bug 24977; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug24978 b/changes/bug24978 deleted file mode 100644 index 5dc45c7442..0000000000 --- a/changes/bug24978 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor features (compatibility, OpenSSL): - - Tor will now support TLS1.3 once OpenSSL 1.1.1 is released. - Previous versions of Tor would not have worked with OpenSSL - 1.1.1, since they neither disabled TLS 1.3 nor enabled any of the - ciphersuites it requires. Here we enable the TLS 1.3 ciphersuites. - Closes ticket 24978. - diff --git a/changes/bug24989 b/changes/bug24989 deleted file mode 100644 index a0ea6acf03..0000000000 --- a/changes/bug24989 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (hidden services): - - Re-instate counting pending client HSDir fetch circuits against the - MaxClientCircuitsPending rate limit. Fixes bug 24989; bugfix on - 0.3.3.0-alpha-dev. diff --git a/changes/bug25120 b/changes/bug25120 deleted file mode 100644 index 7215756ef3..0000000000 --- a/changes/bug25120 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (logging): - - Clarify the log messages produced when getrandom() or a related - entropy-generation mechanism gives an error. Closes ticket - 25120. diff --git a/changes/bug25213 b/changes/bug25213 deleted file mode 100644 index bb196ca724..0000000000 --- a/changes/bug25213 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (warnings, ipv6): - - Avoid a bug warning that could occur when trying to connect to - a relay over IPv6 on a Tor instance that downloads router descriptors, - but prefers to use microdescriptors. Fixes bug 25213; bugfix on - 0.3.3.1-alpha. diff --git a/changes/bug25223 b/changes/bug25223 deleted file mode 100644 index fdd5563500..0000000000 --- a/changes/bug25223 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (DoS mitigation): - - Make sure we don't modify consensus parameters if we aren't a public - relay when a new consensus arrives. Fixes bug 25223; bugfix on - 0.3.3.2-alpha. diff --git a/changes/bug25226 b/changes/bug25226 deleted file mode 100644 index b594a7a424..0000000000 --- a/changes/bug25226 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (relay, denial of service): - - Impose a limit on circuit cell queue size. The limit can be controlled by - a consensus parameter. Fixes bug 25226; bugfix on 0.2.4.14-alpha. - diff --git a/changes/bug25249 b/changes/bug25249 deleted file mode 100644 index b4153eeaef..0000000000 --- a/changes/bug25249 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (spec conformance): - - Forbid "-0" as a protocol version. Fixes part of bug 25249; bugfix on - 0.2.9.4-alpha. diff --git a/changes/bug25249.2 b/changes/bug25249.2 deleted file mode 100644 index 9058c11071..0000000000 --- a/changes/bug25249.2 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (spec conformance): - - Forbid UINT32_MAX as a protocol version. Fixes part of bug 25249; - bugfix on 0.2.9.4-alpha. diff --git a/changes/bug25296_032 b/changes/bug25296_032 deleted file mode 100644 index f60048ca66..0000000000 --- a/changes/bug25296_032 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (documentation): - - Document that the PerConnBW{Rate,Burst} options will fall back to their - corresponding consensus parameters only if those parameters are - set. Previously we had claimed that these values would always be - set in the consensus. Fixes bug 25296; bugfix on 0.2.2.7-alpha. diff --git a/changes/bug25306 b/changes/bug25306 deleted file mode 100644 index a2e6306f42..0000000000 --- a/changes/bug25306 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (hidden service v3): - - Avoid asserting when building descriptors in the next rotation time is - out of sync with the consensus valid after time. Instead, log a bug - warning with extra information to hunt down the cause of this assert. - Fixes bug 25306; bugfix on 0.3.2.1-alpha. - diff --git a/changes/bug25372 b/changes/bug25372 deleted file mode 100644 index 4cceab3a45..0000000000 --- a/changes/bug25372 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (log messages): - - Improve log message in the out of memory handler to include information - about memory usage from the different compression backends. Closes ticket 25372. diff --git a/changes/bug25378 b/changes/bug25378 deleted file mode 100644 index 4648a0a313..0000000000 --- a/changes/bug25378 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation: - - Update the documentation for "Log" to include the current list - of logging domains. Closes ticket 25378. - diff --git a/changes/bug25415 b/changes/bug25415 deleted file mode 100644 index ec851aee8d..0000000000 --- a/changes/bug25415 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (directory authority): - - Avoid a crash when testing router reachability on a router that could - have an ed25519 ID, but which does not. Fixes bug 25415; bugfix on - 0.3.3.2-alpha. diff --git a/changes/bug25450 b/changes/bug25450 deleted file mode 100644 index ae14135a72..0000000000 --- a/changes/bug25450 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes (testing): - - Avoid intermittent test failures due to relying on hidden service - introductory point creation within 5 seconds of real clock time. The - time limit for the test has been increased to 500 seconds, which may - still result in intermittent failures (e.g. if the system doing the - testing enters sleep/hibernation or experiences some other clock jump). - However, this should elliminate test failures currently happening on - developer and CI systems. Fixes bug 25450; bugfix on 0.3.1.3-alpha. diff --git a/changes/bug25474 b/changes/bug25474 deleted file mode 100644 index 7d3bd1c5f5..0000000000 --- a/changes/bug25474 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (compilation): - - Fix a c99 compliance issue in our configuration script that was - causing compilation issues when compiling Tor with certain - versions of xtools. Fixes bug 25474; bugfix on 0.3.2.5-alpha. - diff --git a/changes/bug25479 b/changes/bug25479 deleted file mode 100644 index e693b8ec7a..0000000000 --- a/changes/bug25479 +++ /dev/null @@ -1,4 +0,0 @@ - o Major bugfixes (benchmarks): - - Fix a crash when running benchmark tests on win32 systems which - was due to an uninitialised mutex before logging and options - were initialised. Fixes bug 25479; bugfix on 0.3.3.3-alpha. diff --git a/changes/bug25512 b/changes/bug25512 deleted file mode 100644 index 4b6491867e..0000000000 --- a/changes/bug25512 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (restart-in-process): - - When shutting down, Tor now clears all the flags in the control.c - module. This should prevent a bug where authentication cookies - are not generated on restart. Fixes bug 25512; bugfix on 0.3.3.1-alpha. - diff --git a/changes/bug25560 b/changes/bug25560 deleted file mode 100644 index 5947fa03a3..0000000000 --- a/changes/bug25560 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (testing, rust): - - Some of our Rust crates were not having their tests run with `make - test-rust` due to a static string in the `src/test/test_rust.sh` script - specifying which crates to test. Rust crates are not automatically - detected and tested. Fixes bug 25560; bugfix on 0.3.3.3-alpha. diff --git a/changes/bug25581 b/changes/bug25581 deleted file mode 100644 index 86f2491db7..0000000000 --- a/changes/bug25581 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (configuration): - - Remove undescores from the _HSLayer{2,3}Nodes options. This expert-user - configuration can now be enabled as HSLayer{2,3}Nodes. Fixes bug 25581; - bugfix on 0.3.3.1-alpha diff --git a/changes/bug25582 b/changes/bug25582 deleted file mode 100644 index 609b94aac6..0000000000 --- a/changes/bug25582 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (documentation): - - Revert a misformatting issue in the ExitPolicy - documentation. Fixes bug 25582; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug25617 b/changes/bug25617 deleted file mode 100644 index 5de655d69e..0000000000 --- a/changes/bug25617 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (controller): - - Restore the correct operation of the RESOLVE command, which had - been broken since we added the ability to enable/disable DNS - on specific listener ports. Fixes bug 25617; bugfix on 0.2.9.3-alpha. - diff --git a/changes/bug25629 b/changes/bug25629 deleted file mode 100644 index 190928a941..0000000000 --- a/changes/bug25629 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (C correctness): - - Fix a very unlikely null pointer dereference. Fixes bug 25629; - bugfix on 0.2.9.15. Found by Coverity; this is CID 1430932. diff --git a/changes/bug25664 b/changes/bug25664 deleted file mode 100644 index c8b3ca6187..0000000000 --- a/changes/bug25664 +++ /dev/null @@ -1,3 +0,0 @@ - o New system requirements: - - When built with Rust, Tor now depends on version 0.2.39 of the libc - crate. Closes tickets 25310 and 25664. diff --git a/changes/bug25679 b/changes/bug25679 deleted file mode 100644 index 9247a29153..0000000000 --- a/changes/bug25679 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (compilation, rust): - - Build correctly when the rust dependencies submodule is loaded, - but the TOR_RUST_DEPENDENCIES environment variable is not set. - Fixes bug 25679; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug25691 b/changes/bug25691 deleted file mode 100644 index 5f630d5032..0000000000 --- a/changes/bug25691 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (client): - - When using a listed relay as a bridge, and also using - microdescriptors, and considering that relay as a non-bridge in - a circuit, consider its microdescriptor as a valid source of information - about that relay. Fixes bug 25691; bugfix on 0.3.3.4-alpha. - diff --git a/changes/bug25691_again b/changes/bug25691_again deleted file mode 100644 index 3d0d91bfd3..0000000000 --- a/changes/bug25691_again +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (path selection): - - Only select relays when they have the descriptors we prefer to - use for them. This change fixes a bug where we could select - a relay because it had _some_ descriptor, but reject it later with - a nonfatal assertion error because it didn't have the exact one we - wanted. Fixes bugs 25691 and 25692; bugfix on 0.3.3.4-alpha. diff --git a/changes/bug25705 b/changes/bug25705 deleted file mode 100644 index 360d96d4c6..0000000000 --- a/changes/bug25705 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (circuit path selection): - - Don't count path selection failures as circuit build failures. This - should eliminate cases where Tor blames its guard or the network - for situations like insufficient microdescriptors and/or overly - restrictive torrc settings. Fixes bug 25705; bugfix on 0.3.3.1-alpha. diff --git a/changes/bug25732 b/changes/bug25732 deleted file mode 100644 index 49ffae29e5..0000000000 --- a/changes/bug25732 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (distribution, compilation): - - Actually include all of our Rust source in our source - distributions. (Previously, a few of the files were accidentally - omitted.) Fixes bug 25732; bugfix on 0.3.3.2-alpha. diff --git a/changes/bug25733 b/changes/bug25733 deleted file mode 100644 index 775c1ae00e..0000000000 --- a/changes/bug25733 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (Assert crash): - - Avoid an assert in the circuit build timeout code if we fail to - allow any circuits to actually complete. Fixes bug 25733; - bugfix on 0.2.2.2-alpha. diff --git a/changes/bug25761 b/changes/bug25761 deleted file mode 100644 index 096fadcf09..0000000000 --- a/changes/bug25761 +++ /dev/null @@ -1,3 +0,0 @@ - o Major bugfixes (onion service): - - Correctly detect when onion services get disabled after HUP. - Fixes bug 25761; bugfix on 0.3.2.1. diff --git a/changes/bug25787 b/changes/bug25787 new file mode 100644 index 0000000000..3041e8a603 --- /dev/null +++ b/changes/bug25787 @@ -0,0 +1,7 @@ + o Minor bugfixes (testing): + - Instead of trying to read the geoip configuration files from within the + unit tests, instead create our own ersatz files with just enough + geoip data in the format we expect. Trying to read from the source + directory created problems on Windows with mingw, where the + build system's paths are not the same as the platform's paths. + Fixes bug 25787; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug25901 b/changes/bug25901 deleted file mode 100644 index 3ad30c7676..0000000000 --- a/changes/bug25901 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (hidden service v3): - - Fix a memory leak when an hidden service v3 is configured and gets a - SIGHUP signal. Fixes bug 25901; bugfix on 0.3.2.1-alpha. diff --git a/changes/bug26007 b/changes/bug26007 deleted file mode 100644 index efcd15084d..0000000000 --- a/changes/bug26007 +++ /dev/null @@ -1,5 +0,0 @@ - o Major bugfixes (directory authorities, security): - - When directory authorities read a zero-byte bandwidth file, they log - a warning with the contents of an uninitialised buffer. Log a warning - about the empty file instead. - Fixes bug 26007; bugfix on 0.2.2.1-alpha. diff --git a/changes/bug26052 b/changes/bug26052 deleted file mode 100644 index 4721933fa5..0000000000 --- a/changes/bug26052 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (documentation): - - Stop saying in the manual that clients cache ipv4 dns answers - from exit relays. We haven't used them since 0.2.6.3-alpha, and - in ticket 24050 we stopped even caching them as of 0.3.2.6-alpha, - but we forgot to say so in the man page. Fixes bug 26052; bugfix - on 0.3.2.6-alpha. diff --git a/changes/bug26069 b/changes/bug26069 deleted file mode 100644 index 192e97d782..0000000000 --- a/changes/bug26069 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (hidden service v3): - - When parsing the descriptor signature, look for the token plus an extra - white-space at the end. This is more correct but also will allow us to - support new fields that might start with "signature". Fixes bug 26069; - bugfix on 0.3.0.1-alpha. diff --git a/changes/bug26072 b/changes/bug26072 deleted file mode 100644 index 2489e4fbb5..0000000000 --- a/changes/bug26072 +++ /dev/null @@ -1,5 +0,0 @@ - o Minor bugfixes (correctness, client): - - Upon receiving a malformed connected cell, stop processing the cell - immediately. Previously we would mark the connection for close, but - continue processing the cell as if the connection were open. Fixes bug - 26072; bugfix on 0.2.4.7-alpha. diff --git a/changes/bug26116 b/changes/bug26116 deleted file mode 100644 index 3bfde74f77..0000000000 --- a/changes/bug26116 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (compatibility, openssl): - - Work around a change in OpenSSL 1.1.1 where - return values that would previously indicate "no password" now - indicate an empty password. Without this workaround, Tor instances - running with OpenSSL 1.1.1 would accept descriptors that other Tor - instances would reject. Fixes bug 26116; bugfix on 0.2.5.16. - diff --git a/changes/bug26121 b/changes/bug26121 deleted file mode 100644 index 5f734dd56b..0000000000 --- a/changes/bug26121 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (controller): - - Improve accuracy of the BUILDTIMEOUT_SET control port event's - TIMEOUT_RATE and CLOSE_RATE fields. (We were previously miscounting - the total number of circuits for these field values.) Fixes bug - 26121; bugfix on 0.3.3.1-alpha. - diff --git a/changes/bug26196 b/changes/bug26196 index 47fcffa0f8..e63f09a2d6 100644 --- a/changes/bug26196 +++ b/changes/bug26196 @@ -1,4 +1,7 @@ - o Minor bugfixes (hardening): - - Prevent a possible out-of-bounds smartlist read in - protover_compute_vote(). Fixes bug 26196; bugfix on - 0.2.9.4-alpha. + o Minor bugfixes (testing, compilation): + - Refrain from trying to get an item from empty smartlist in + test_bridges_clear_bridge_list. Set DEBUG_SMARTLIST in unit + tests to catch improper smartlist usage. Furthermore, + enable DEBUG_SMARTLIST globally when build is configured + with fragile hardening. Fixes bug 26196; bugfix on + 0.3.4.1-alpha. diff --git a/changes/bug26214 b/changes/bug26214 new file mode 100644 index 0000000000..4277b9c6ec --- /dev/null +++ b/changes/bug26214 @@ -0,0 +1,3 @@ + o Minor bugfixes (correctness, flow control): + - Upon receiving a stream-level SENDME cell, verify that our window has + not grown too large. Fixes bug 26214; bugfix on svn r54 (pre-0.0.1) diff --git a/changes/bug26245 b/changes/bug26245 new file mode 100644 index 0000000000..7a14cea0bc --- /dev/null +++ b/changes/bug26245 @@ -0,0 +1,3 @@ + o Minor features (compilation): + o The --enable-fatal-warnings flag now affects Rust code as well. + Closes ticket 26245. diff --git a/changes/bug26272 b/changes/bug26272 deleted file mode 100644 index 9dcf42f0e1..0000000000 --- a/changes/bug26272 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (compilation): - - Silence unused-const-variable warnings in zstd.h on some gcc versions. - Fixes bug 26272; bugfix on 0.3.1.1-alpha. diff --git a/changes/bug26402 b/changes/bug26402 new file mode 100644 index 0000000000..b21283a2d2 --- /dev/null +++ b/changes/bug26402 @@ -0,0 +1,4 @@ + o Minor bugfixes (compilation): + - When linking the libtor_testing.a library, only include the dirauth + object files once. Previously, they were getting added twice. + Fixes bug 26402; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug26415 b/changes/bug26415 new file mode 100644 index 0000000000..497fbb7365 --- /dev/null +++ b/changes/bug26415 @@ -0,0 +1,3 @@ + o Minor bugfixes (testing): + - Fix compilation of the doctests in the Rust crypto crate. Fixes + bug 26415; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug26568 b/changes/bug26568 new file mode 100644 index 0000000000..0c4c05d043 --- /dev/null +++ b/changes/bug26568 @@ -0,0 +1,3 @@ + o Minor bugfixes (controller): + - Report the port correctly when a port is configured to bind to "auto". + Fixes bug 26568; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug26789 b/changes/bug26789 new file mode 100644 index 0000000000..9b3520543d --- /dev/null +++ b/changes/bug26789 @@ -0,0 +1,4 @@ + o Minor bugfixes (compilation): + - Update build system so that tor builds again with + --disable-unittests after recent refactoring efforts. + Fixes bug 26789; bugfix on 0.3.4.3-alpha. diff --git a/changes/bug26896 b/changes/bug26896 new file mode 100644 index 0000000000..9762dc7fac --- /dev/null +++ b/changes/bug26896 @@ -0,0 +1,3 @@ + o Minor bugfixes (directory connection shutdown): + - Avoid a double-close when shutting down a stalled directory connection. + Fixes bug 26896; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug26979 b/changes/bug26979 new file mode 100644 index 0000000000..e615207b74 --- /dev/null +++ b/changes/bug26979 @@ -0,0 +1,4 @@ + o Minor bugfixes (appveyor ci): + - Improve Appveyor CI IRC logging. Generate correct branches and URLs for + pull requests and tags. Use unambiguous short commits. + Fixes bug 26979; bugfix on master. diff --git a/changes/bug26994 b/changes/bug26994 new file mode 100644 index 0000000000..664894a7fc --- /dev/null +++ b/changes/bug26994 @@ -0,0 +1,3 @@ + o Minor bugfixes (portability): + - Avoid a compilation error in test_bwmgt.c on Solaris 10. + Fixes bug 26994; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27003 b/changes/bug27003 new file mode 100644 index 0000000000..4f2045afc7 --- /dev/null +++ b/changes/bug27003 @@ -0,0 +1,6 @@ + o Major bugfixes (event scheduler): + - When we enable a periodic event, schedule it in the event loop + rather than running it immediately. Previously, we would re-run + periodic events immediately in the middle of (for example) + changing our options, with unpredictable effects. Fixes bug + 27003; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27139 b/changes/bug27139 new file mode 100644 index 0000000000..0d1e3b4329 --- /dev/null +++ b/changes/bug27139 @@ -0,0 +1,14 @@ + o Minor bugfixes (32-bit OSX and iOS, timing): + - Fix an integer overflow bug in our optimized 32-bit millisecond- + difference algorithm for 32-bit Apple platforms. Previously, it + would overflow when calculating the difference between two times + more than 47 days apart. Fixes part of bug 27139; bugfix on + 0.3.4.1-alpha. + - Improve the precision of our 32-bit millisecond difference + algorithm for 32-bit Apple platforms. Fixes part of bug 27139; + bugfix on 0.3.4.1-alpha. + - Relax the tolerance on the mainloop/update_time_jumps test + when running on 32-bit Apple platforms. Fixes part of bug 27139; + bugfix on 0.3.4.1-alpha. + + diff --git a/changes/bug27236 b/changes/bug27236 new file mode 100644 index 0000000000..76d792f4c9 --- /dev/null +++ b/changes/bug27236 @@ -0,0 +1,5 @@ + o Minor bugfixes (testing, bootstrap): + - When calculating bootstrap progress, check exit policies and the exit + flag. Previously, Tor would only check the exit flag, which caused + race conditions in small and fast networks like chutney. + Fixes bug 27236; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug27460 b/changes/bug27460 new file mode 100644 index 0000000000..53c4c7daf3 --- /dev/null +++ b/changes/bug27460 @@ -0,0 +1,6 @@ + o Minor bugfixes (continuous integration): + - Disable gcc hardening in Appveyor Windows 64-bit builds. As of + August 29, 2018, Appveyor images come with gcc 8.2.0 by default. + 64-bit Windows executables compiled with gcc 8.2.0 and + tor's --enable-gcc-hardening crash. + Fixes bug 27460; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27464 b/changes/bug27464 new file mode 100644 index 0000000000..9dedd06f41 --- /dev/null +++ b/changes/bug27464 @@ -0,0 +1,3 @@ + o Minor bugfixes (continuous integration): + - Stop reinstalling identical packages in our Windows CI. + Fixes bug 27464; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27750 b/changes/bug27750 new file mode 100644 index 0000000000..c234788b1c --- /dev/null +++ b/changes/bug27750 @@ -0,0 +1,6 @@ + o Minor bugfixes (connection, relay): + - Avoid a wrong BUG() stacktrace in case a closing connection is being held + open because the write side is rate limited but not the read side. Now, + the connection read side is simply shutdown instead of kept open until tor + is able to flush the connection and then fully close it. Fixes bug 27750; + bugfix on 0.3.4.1-alpha. diff --git a/changes/bug27781 b/changes/bug27781 new file mode 100644 index 0000000000..44d838af8a --- /dev/null +++ b/changes/bug27781 @@ -0,0 +1,4 @@ + o Major bugfixes (compilation): + - Fix compilation on arm (and other less-used CPUs) + when compiling with OpenSSL before 1.1. Fixes bug 27781; + bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28399 b/changes/bug28399 new file mode 100644 index 0000000000..9096db70b0 --- /dev/null +++ b/changes/bug28399 @@ -0,0 +1,4 @@ + o Minor bugfixes (continuous integration, Windows): + - Stop using an external OpenSSL install, and stop installing MSYS2 + packages, when building using mingw on Appveyor Windows CI. + Fixes bug 28399; bugfix on 0.3.4.1-alpha. diff --git a/changes/bug28454 b/changes/bug28454 new file mode 100644 index 0000000000..ca46ae2777 --- /dev/null +++ b/changes/bug28454 @@ -0,0 +1,4 @@ + o Minor bugfixes (continuous integration, Windows): + - Manually configure the zstd compiler options, when building using + mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not + come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha. diff --git a/changes/bugs_25036_25055 b/changes/bugs_25036_25055 deleted file mode 100644 index daa46321c0..0000000000 --- a/changes/bugs_25036_25055 +++ /dev/null @@ -1,7 +0,0 @@ - o Minor bugfixes (networking): - - Tor will not reject IPv6 address strings from TorBrowser when they - are passed as hostnames in SOCKS5 requests. Fixes bug 25036, - bugfix on Tor 0.3.1.2. - - string_is_valid_hostname() will not consider IP strings to be valid - hostnames. Fixes bug 25055; bugfix on Tor 0.2.5.5. - diff --git a/changes/coveralls b/changes/coveralls deleted file mode 100644 index 7fa69bb2b4..0000000000 --- a/changes/coveralls +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (continuous integration): - - Our Travis CI configuration now integrates with the Coveralls coverage - analysis tool. Closes ticket 25818. diff --git a/changes/feature25313 b/changes/feature25313 deleted file mode 100644 index 90f421169f..0000000000 --- a/changes/feature25313 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (sandbox): - - Explicitly permit the poll() system call when the Linux seccomp2-based - sandbox is enabled: apparently, some versions of libc use poll() when - calling getpwnam(). Closes ticket 25313. diff --git a/changes/geoip-2018-03-08 b/changes/geoip-2018-03-08 deleted file mode 100644 index d9696aab53..0000000000 --- a/changes/geoip-2018-03-08 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the March 8 2018 Maxmind GeoLite2 - Country database. Closes ticket 25469. - diff --git a/changes/geoip-2018-04-03 b/changes/geoip-2018-04-03 deleted file mode 100644 index 987cc450bf..0000000000 --- a/changes/geoip-2018-04-03 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the April 3 2018 Maxmind GeoLite2 - Country database. Closes ticket 25718. - diff --git a/changes/geoip-2018-05-01 b/changes/geoip-2018-05-01 deleted file mode 100644 index 1528bb0c31..0000000000 --- a/changes/geoip-2018-05-01 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the May 1 2018 Maxmind GeoLite2 - Country database. Closes ticket 26104. - diff --git a/changes/geoip-2018-06-07 b/changes/geoip-2018-06-07 deleted file mode 100644 index 0f8cff97af..0000000000 --- a/changes/geoip-2018-06-07 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (geoip): - - Update geoip and geoip6 to the June 7 2018 Maxmind GeoLite2 - Country database. Closes ticket 26351. - diff --git a/changes/refactor23814 b/changes/refactor23814 deleted file mode 100644 index a67b6989f8..0000000000 --- a/changes/refactor23814 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Remove the old (deterministic) directory retry logic entirely: - We've used exponential backoff exclusively for some time. - Closes ticket 23814. diff --git a/changes/rust_cross b/changes/rust_cross new file mode 100644 index 0000000000..d490403a28 --- /dev/null +++ b/changes/rust_cross @@ -0,0 +1,2 @@ + o Minor feature (Rust, portability): + - Rust cross-compilation is now supported. Closes ticket 25895. diff --git a/changes/ticket23635 b/changes/ticket23635 deleted file mode 100644 index 54d303e4b8..0000000000 --- a/changes/ticket23635 +++ /dev/null @@ -1,3 +0,0 @@ - o Documentation: - - Improved the documentation of AccountingStart paremeter. - Closes ticket 23635. diff --git a/changes/ticket24343 b/changes/ticket24343 deleted file mode 100644 index e62d65eb54..0000000000 --- a/changes/ticket24343 +++ /dev/null @@ -1,6 +0,0 @@ - o Minor bugfixes (man page, SocksPort): - - Remove dead code about the old "SocksSockets" option. To do so, the - SocksSocketsGroupWritable option has been renamed to - UnixSockssGroupWritable which does the same exact thing. The old option - is still usable but will warn that it is deprecated. Fixes bug 24343; - bugfix on 0.2.6.3. diff --git a/changes/ticket25071 b/changes/ticket25071 deleted file mode 100644 index 5e2917e10b..0000000000 --- a/changes/ticket25071 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (testing): - - Add a "make test-rust" target to run the rust tests only. - Closes ticket 25071. - diff --git a/changes/ticket25202 b/changes/ticket25202 deleted file mode 100644 index ba64abad7b..0000000000 --- a/changes/ticket25202 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor bugfixes (DoS mitigation): - - Add extra safety checks when refilling the circuit creation bucket to - ensure we never set a value that is above the allowed burst. Fixes - bug 25202; bugfix on 0.3.3.2-alpha. diff --git a/changes/ticket25248 b/changes/ticket25248 deleted file mode 100644 index 67ccdb6313..0000000000 --- a/changes/ticket25248 +++ /dev/null @@ -1,4 +0,0 @@ - o Documentation (manpage, denial of service): - - Better detail the denial of service options by listing the different - mitigation in place. Closes ticket 25248. - diff --git a/changes/ticket25323 b/changes/ticket25323 deleted file mode 100644 index 836825de5d..0000000000 --- a/changes/ticket25323 +++ /dev/null @@ -1,4 +0,0 @@ - o Code simplification and refactoring: - - Update the "rust dependencies" submodule to be an project-level - repository, rather than a user repository. Closes ticket 25323. - diff --git a/changes/ticket25573 b/changes/ticket25573 new file mode 100644 index 0000000000..9939601b50 --- /dev/null +++ b/changes/ticket25573 @@ -0,0 +1,5 @@ + o Minor features (controller): + - For purposes of CIRC_BW-based dropped cell detection, track half-closed + stream ids, and allow their ENDs, SENDMEs, DATA and path bias check + cells to arrive without counting it as dropped until either the END arrvies, + or the windows are empty. Closes ticket 25573. diff --git a/changes/ticket25714 b/changes/ticket25714 deleted file mode 100644 index 63823fc6ca..0000000000 --- a/changes/ticket25714 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor feature (continuous integration): - - Update the Travis CI configuration to use the stable Rust - channel, now that we have decided to require that. Closes - ticket 25714. diff --git a/changes/ticket25947 b/changes/ticket25947 new file mode 100644 index 0000000000..68559a73f8 --- /dev/null +++ b/changes/ticket25947 @@ -0,0 +1,4 @@ + o Minor feature (unit tests): + - Test complete bandwidth measurements files and test that incomplete lines + only give warnings when the end of the header has not been + detected. Fixes bug 25947; bugfix on 0.2.2.1-alpha diff --git a/changes/ticket25960 b/changes/ticket25960 new file mode 100644 index 0000000000..0d1be2119b --- /dev/null +++ b/changes/ticket25960 @@ -0,0 +1,5 @@ + o Minor feature (directory authorities): + - Stop warning about incomplete bw lines before the first complete bw line + has been found, so that additional header lines can be ignored. + Fixes bug 25960; bugfix on 0.2.2.1-alpha + diff --git a/changes/ticket26006 b/changes/ticket26006 deleted file mode 100644 index e33e3f1cd2..0000000000 --- a/changes/ticket26006 +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (compilation, portability): - - Avoid some compilation warnings with recent versions - of LibreSSL. Closes ticket 26006. - diff --git a/changes/ticket26062 b/changes/ticket26062 deleted file mode 100644 index 1ee49d8608..0000000000 --- a/changes/ticket26062 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor bugfixes (client): - - Don't consider Tor running as a client if the ControlPort is open. Fixes - bug 26062; bugfix on 0.2.9.4-alpha. diff --git a/changes/ticket26343 b/changes/ticket26343 deleted file mode 100644 index ab5f332ee6..0000000000 --- a/changes/ticket26343 +++ /dev/null @@ -1,3 +0,0 @@ - o Minor features (directory authority): - - Add an IPv6 address for the "dannenberg" directory - authority. Closes ticket 26343. diff --git a/changes/ticket27252-034 b/changes/ticket27252-034 new file mode 100644 index 0000000000..620ad83efe --- /dev/null +++ b/changes/ticket27252-034 @@ -0,0 +1,3 @@ + o Minor features (continuous integration): + - Don't do a distcheck with --disable-module-dirauth in Travis. + Implements ticket 27252. diff --git a/changes/ticket27275 b/changes/ticket27275 new file mode 100644 index 0000000000..d1332e9540 --- /dev/null +++ b/changes/ticket27275 @@ -0,0 +1,3 @@ + o Minor features (continuous integration): + - Only post Appveyor IRC notifications when the build fails. + Implements ticket 27275. diff --git a/changes/ticket27430 b/changes/ticket27430 new file mode 100644 index 0000000000..4e016e91e7 --- /dev/null +++ b/changes/ticket27430 @@ -0,0 +1,4 @@ + o Minor features (continuous integration): + - Show config.log and test-suite.log after failed Appveyor builds. + Also upload the zipped full logs as a build artifact. + Implements ticket 27430. diff --git a/changes/ticket27449 b/changes/ticket27449 new file mode 100644 index 0000000000..2a0984c09c --- /dev/null +++ b/changes/ticket27449 @@ -0,0 +1,3 @@ + o Minor features (continuous integration): + - Log the compiler path and version during Appveyor builds. + Implements ticket 27449. diff --git a/changes/ticket27797 b/changes/ticket27797 new file mode 100644 index 0000000000..f07e35f84c --- /dev/null +++ b/changes/ticket27797 @@ -0,0 +1,5 @@ + o Minor bugfixes (node, hidden service v3): + - When selecting a v3 rendezvous point, not only look at the protover but + also if the curve25519 onion key is present. That way we avoid picking a + node that supports the v3 rendezvous but for which we don't have the + descriptor yet for the key. Fixes bug 27797; bugfix on 0.3.2.1-alpha. diff --git a/changes/ticket27810 b/changes/ticket27810 new file mode 100644 index 0000000000..119f781cb9 --- /dev/null +++ b/changes/ticket27810 @@ -0,0 +1,5 @@ + o Minor bugfixes (test, hidden service v3): + - Make the the hs_service tests uses the same time source when creating + the introduction point and testing it. This helps make test work on very + slow system like ARM or Travis. Fixes bug 27810; bugfix on + 0.3.2.1-alpha. diff --git a/changes/ticket27849 b/changes/ticket27849 new file mode 100644 index 0000000000..c2babeffc7 --- /dev/null +++ b/changes/ticket27849 @@ -0,0 +1,5 @@ + o Major bugfixes (mainloop, bootstrap): + - Make sure Tor bootstraps and works properly if only the ControlPort is + set. Prior to this fix, Tor would only bootstrap with at least a client + port being set (Socks, Trans, NATD, DNS or HTTPTunnel port). Fixes bug + 27849; bugfix on 0.3.4.1-alpha. diff --git a/changes/ticket28089 b/changes/ticket28089 new file mode 100644 index 0000000000..a900ec18f5 --- /dev/null +++ b/changes/ticket28089 @@ -0,0 +1,6 @@ + o Major bugfixes (relay): + - When our write bandwidth limit is exhausted, stop writing on the + connection. Previously, we had a typo in the code that would make us stop + reading leading to relay connections being stuck indefinitely. Fixes bug + 28089; bugfix on 0.3.4.1-alpha. + diff --git a/changes/ticket28459 b/changes/ticket28459 new file mode 100644 index 0000000000..6b5839b52b --- /dev/null +++ b/changes/ticket28459 @@ -0,0 +1,4 @@ + o Minor features (continuous integration, Windows): + - Always show the configure and test logs, and upload them as build + artifacts, when building for Windows using Appveyor CI. + Implements 28459. diff --git a/changes/ticket28574 b/changes/ticket28574 new file mode 100644 index 0000000000..562810f511 --- /dev/null +++ b/changes/ticket28574 @@ -0,0 +1,4 @@ + o Minor bugfixes (continuous integration, Windows): + - Explicitly specify the path to the OpenSSL library and do not download + OpenSSL from Pacman, but instead use the library that is already provided + by AppVeyor. Fixes bug 28574; bugfix on master. diff --git a/changes/ticket28912 b/changes/ticket28912 new file mode 100644 index 0000000000..4119b778bc --- /dev/null +++ b/changes/ticket28912 @@ -0,0 +1,6 @@ + o Major bugfixes (relay, directory): + - A connection serving directory information wouldn't get reactivated after + the first chunk of data was sent (usually 32KB). Tor now always activate + the main loop event that goes through these connections as long as at + least one connection is still active. Fixes bug 28912; bugfix on + 0.3.4.1-alpha. Patch by "cypherpunks3". diff --git a/changes/tickets_27765_27943 b/changes/tickets_27765_27943 new file mode 100644 index 0000000000..e8468545f1 --- /dev/null +++ b/changes/tickets_27765_27943 @@ -0,0 +1,5 @@ + o Minor bugfixes (CI, appveyor): + - Only install the necessary mingw packages during our appveyor + builds. This change makes the build a little faster, and prevents a + conflict with a preinstalled mingw openssl that appveyor now ships. + Fixes bugs 27943 and 27765; bugfix on 0.3.4.2-alpha. diff --git a/changes/travis_distcheck b/changes/travis_distcheck deleted file mode 100644 index 0f278fe7ee..0000000000 --- a/changes/travis_distcheck +++ /dev/null @@ -1,4 +0,0 @@ - o Minor features (continuous integration): - - Our .travis.yml configuration now includes support for testing - the results of "make distcheck". (It's not uncommon for "make check" to - pass but "make distcheck" to fail.) Closes ticket 25814. diff --git a/changes/trove-2018-001.1 b/changes/trove-2018-001.1 deleted file mode 100644 index f0ee92f409..0000000000 --- a/changes/trove-2018-001.1 +++ /dev/null @@ -1,6 +0,0 @@ - o Major bugfixes (denial-of-service, directory authority): - - Fix a protocol-list handling bug that could be used to remotely crash - directory authorities with a null-pointer exception. Fixes bug 25074; - bugfix on 0.2.9.4-alpha. Also tracked as TROVE-2018-001. - - diff --git a/changes/trove-2018-004 b/changes/trove-2018-004 deleted file mode 100644 index 37e0a89b0d..0000000000 --- a/changes/trove-2018-004 +++ /dev/null @@ -1,8 +0,0 @@ - o Minor bugfixes (denial-of-service): - - Fix a possible crash on malformed consensus. If a consensus had - contained an unparseable protocol line, it could have made clients - and relays crash with a null-pointer exception. To exploit this - issue, however, an attacker would need to be able to subvert the - directory-authority system. Fixes bug 25251; bugfix on - 0.2.9.4-alpha. Also tracked as TROVE-2018-004. - diff --git a/config.rust.in b/config.rust.in new file mode 100644 index 0000000000..4ca5351aec --- /dev/null +++ b/config.rust.in @@ -0,0 +1,22 @@ +# Used by our cargo build.rs script to get variables from autoconf. +# +# The "configure" script will generate "config.rust" from "config.rust.in", +# and then build.rs will read "config.rust". + +BUILDDIR=@BUILDDIR@ +TOR_LDFLAGS_zlib=@TOR_LDFLAGS_zlib@ +TOR_LDFLAGS_openssl=@TOR_LDFLAGS_openssl@ +TOR_LDFLAGS_libevent=@TOR_LDFLAGS_libevent@ +TOR_ZLIB_LIBS=@TOR_ZLIB_LIBS@ +TOR_LIB_MATH=@TOR_LIB_MATH@ +TOR_LIBEVENT_LIBS=@TOR_LIBEVENT_LIBS@ +TOR_OPENSSL_LIBS=@TOR_OPENSSL_LIBS@ +TOR_LIB_WS32=@TOR_LIB_WS32@ +TOR_LIB_GDI=@TOR_LIB_GDI@ +TOR_LIB_USERENV=@TOR_LIB_USERENV@ +CURVE25519_LIBS=@CURVE25519_LIBS@ +TOR_SYSTEMD_LIBS=@TOR_SYSTEMD_LIBS@ +TOR_LZMA_LIBS=@TOR_LZMA_LIBS@ +TOR_ZSTD_LIBS=@TOR_ZSTD_LIBS@ +LIBS=@LIBS@ +LDFLAGS=@LDFLAGS@ diff --git a/configure.ac b/configure.ac index a651a7b5f3..14c87cd1bb 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ dnl Copyright (c) 2007-2017, The Tor Project, Inc. dnl See LICENSE for licensing information AC_PREREQ([2.63]) -AC_INIT([tor],[0.3.3.10-dev]) +AC_INIT([tor],[0.3.4.9-dev]) AC_CONFIG_SRCDIR([src/or/main.c]) AC_CONFIG_MACRO_DIR([m4]) @@ -61,6 +61,8 @@ AC_ARG_ENABLE(cargo-online-mode, AS_HELP_STRING(--enable-cargo-online-mode, [Allow cargo to make network requests to fetch crates. For builds with rust only.])) AC_ARG_ENABLE(restart-debugging, AS_HELP_STRING(--enable-restart-debugging, [Build Tor with support for debugging in-process restart. Developers only.])) +AC_ARG_ENABLE(zstd-advanced-apis, + AS_HELP_STRING(--disable-zstd-advanced-apis, [Build without support for zstd's "static-only" APIs.])) if test "x$enable_coverage" != "xyes" -a "x$enable_asserts_in_tests" = "xno" ; then AC_MSG_ERROR([Can't disable assertions outside of coverage build]) @@ -114,6 +116,10 @@ if test "$enable_restart_debugging" = "yes"; then [Defined if we're building with support for in-process restart debugging.]) fi +if test "$enable_zstd_advanced_apis" != "no"; then + AC_DEFINE(ENABLE_ZSTD_ADVANCED_APIS, 1, + [Defined if we're going to try to use zstd's "static-only" APIs.]) +fi # systemd support if test "x$enable_systemd" = "xno"; then @@ -170,6 +176,7 @@ AC_ARG_ENABLE(fragile-hardening, AS_HELP_STRING(--enable-fragile-hardening, [enable more fragile and expensive compiler hardening; makes Tor slower])) if test "x$enable_expensive_hardening" = "xyes" || test "x$enable_fragile_hardening" = "xyes"; then fragile_hardening="yes" + AC_DEFINE(DEBUG_SMARTLIST, 1, [Enable smartlist debugging]) fi dnl Linker hardening options @@ -224,6 +231,31 @@ if test "x$enable_android" = "xyes"; then fi +dnl --- +dnl Tor modules options. These options are namespaced with --disable-module-XXX +dnl --- + +dnl All our modules. +m4_define(MODULES, dirauth) + +dnl Directory Authority module. +AC_ARG_ENABLE([module-dirauth], + AS_HELP_STRING([--disable-module-dirauth], + [Do not build tor with the dirauth module]), + [], dnl Action if-given + AC_DEFINE([HAVE_MODULE_DIRAUTH], [1], + [Compile with Directory Authority feature support])) +AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"]) + +dnl Helper variables. +TOR_MODULES_ALL_ENABLED= +AC_DEFUN([ADD_MODULE], [ + MODULE=m4_toupper($1) + TOR_MODULES_ALL_ENABLED="${TOR_MODULES_ALL_ENABLED} -DHAVE_MODULE_${MODULE}=1" +]) +m4_foreach_w([module], MODULES, [ADD_MODULE([module])]) +AC_SUBST(TOR_MODULES_ALL_ENABLED) + dnl check for the correct "ar" when cross-compiling. dnl (AM_PROG_AR was new in automake 1.11.2, which we do not yet require, dnl so kludge up a replacement for the case where it isn't there yet.) @@ -279,7 +311,10 @@ fi AM_CONDITIONAL(USEPYTHON, [test "x$PYTHON" != "x"]) dnl List all external rust crates we depend on here. Include the version -rust_crates="libc-0.2.39" +rust_crates=" \ + digest-0.7.2 \ + libc-0.2.39 \ +" AC_SUBST(rust_crates) ifdef([AC_C_FLEXIBLE_ARRAY_MEMBER], [ @@ -418,6 +453,8 @@ fi AC_C_BIGENDIAN +AC_ARG_VAR([TOR_RUST_TARGET], [Rust target, must be specified when cross-compiling (HOST != BUILD). example: i686-pc-windows-gnu]) + if test "x$enable_rust" = "xyes"; then AC_ARG_VAR([RUSTC], [path to the rustc binary]) AC_CHECK_PROG([RUSTC], [rustc], [rustc],[no]) @@ -432,6 +469,11 @@ if test "x$enable_rust" = "xyes"; then fi AC_DEFINE([HAVE_RUST], 1, [have Rust]) + if test "x$enable_fatal_warnings" = "xyes"; then + RUST_WARN= + else + RUST_WARN=# + fi if test "x$enable_cargo_online_mode" = "xyes"; then CARGO_ONLINE= RUST_DL=# @@ -478,13 +520,31 @@ if test "x$enable_rust" = "xyes"; then dnl For now both MSVC and MinGW rust libraries will output static libs with dnl the MSVC naming convention. if test "$bwin32" = "true"; then - TOR_RUST_STATIC_NAME=tor_rust.lib + tor_rust_static_name=tor_rust.lib else - TOR_RUST_STATIC_NAME=libtor_rust.a + tor_rust_static_name=libtor_rust.a fi - AC_SUBST(TOR_RUST_STATIC_NAME) + AC_CANONICAL_BUILD + + if test -n "$TOR_RUST_TARGET"; then + if test "$host" = "$build"; then + AC_MSG_ERROR([HOST = BUILD is invalid if TOR_RUST_TARGET is specified, see configure --help for more information.]) + fi + RUST_TARGET_PROP="target = '$TOR_RUST_TARGET'" + TOR_RUST_LIB_PATH="src/rust/target/$TOR_RUST_TARGET/release/$tor_rust_static_name" + else + if test "$host" != "$build"; then + AC_MSG_ERROR([TOR_RUST_TARGET must be specified when cross-compiling with Rust enabled.]) + fi + RUST_TARGET_PROP= + TOR_RUST_LIB_PATH="src/rust/target/release/$tor_rust_static_name" + fi + + AC_SUBST(RUST_TARGET_PROP) + AC_SUBST(TOR_RUST_LIB_PATH) AC_SUBST(CARGO_ONLINE) + AC_SUBST(RUST_WARN) AC_SUBST(RUST_DL) dnl Let's check the rustc version, too @@ -552,6 +612,7 @@ AC_CHECK_FUNCS( lround \ memmem \ memset_s \ + mmap \ pipe \ pipe2 \ prctl \ @@ -647,6 +708,21 @@ fi AM_CONDITIONAL(BUILD_READPASSPHRASE_C, test "x$ac_cv_func_readpassphrase" = "xno" && test "$bwin32" = "false") +AC_MSG_CHECKING([whether free(NULL) works]) +AC_RUN_IFELSE([AC_LANG_PROGRAM([ + #include <stdlib.h> +], [ +char *p = NULL; +free(p); +])], +[free_null_ok=true; AC_MSG_RESULT(yes)], +[free_null_ok=false; AC_MSG_RESULT(no)], +[free_null_ok=cross; AC_MSG_RESULT(cross)]) + +if test "$free_null_ok" = "false"; then + AC_MSG_ERROR([Your libc implementation doesn't allow free(NULL), as required by C99.]) +fi + dnl ------------------------------------------------------ dnl Where do you live, libevent? And how do we call you? @@ -798,9 +874,18 @@ AC_ARG_WITH(ssl-dir, AC_MSG_NOTICE([Now, we'll look for OpenSSL >= 1.0.1]) TOR_SEARCH_LIBRARY(openssl, $tryssldir, [-lssl -lcrypto $TOR_LIB_GDI $TOR_LIB_WS32], - [#include <openssl/ssl.h>], - [struct ssl_method_st; const struct ssl_method_st *TLSv1_1_method(void);], - [TLSv1_1_method();], [], + [#include <openssl/ssl.h> + char *getenv(const char *);], + [struct ssl_cipher_st; + unsigned SSL_CIPHER_get_id(const struct ssl_cipher_st *); + char *getenv(const char *);], + dnl This funny-looking test program calls getenv, so that the compiler + dnl will neither make code that call SSL_CIPHER_get_id(NULL) [producing + dnl a crash], nor optimize out the call to SSL_CIPHER_get_id(). + dnl We look for SSL_cipher_get_id() because it is present in + dnl OpenSSL >=1.0.1, because it is not deprecated, and because Tor + dnl depends on it. + [if (getenv("THIS_SHOULDNT_BE_SET_X201803")) SSL_CIPHER_get_id((void *)0);], [], [/usr/local/opt/openssl /usr/local/openssl /usr/lib/openssl /usr/local/ssl /usr/lib/ssl /usr/local /opt/openssl]) dnl XXXX check for OPENSSL_VERSION_NUMBER == SSLeay() @@ -869,6 +954,10 @@ AC_CHECK_MEMBERS([SSL.state], , , [#include <openssl/ssl.h> ]) +AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT() +#include <openssl/sha.h> +]) + dnl Define the set of checks for KIST scheduler support. AC_DEFUN([CHECK_KIST_SUPPORT],[ dnl KIST needs struct tcp_info and for certain members to exist. @@ -987,6 +1076,16 @@ if test "x$have_zstd" = "xyes"; then AC_DEFINE(HAVE_ZSTD,1,[Have Zstd]) TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}" TOR_ZSTD_LIBS="${ZSTD_LIBS}" + + dnl now check for zstd functions + save_LIBS="$LIBS" + save_CFLAGS="$CFLAGS" + LIBS="$LIBS $ZSTD_LIBS" + CFLAGS="$CFLAGS $ZSTD_CFLAGS" + AC_CHECK_FUNCS(ZSTD_estimateCStreamSize \ + ZSTD_estimateDCtxSize) + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" fi AC_SUBST(TOR_ZSTD_CFLAGS) AC_SUBST(TOR_ZSTD_LIBS) @@ -1072,6 +1171,33 @@ if test "$fragile_hardening" = "yes"; then TOR_CHECK_CFLAGS([-fno-omit-frame-pointer]) fi +dnl Find the correct libraries to add in order to use the sanitizers. +dnl +dnl When building Rust, Cargo will run the linker with the -nodefaultlibs +dnl option, which will prevent the compiler from linking the sanitizer +dnl libraries it needs. We need to specify them manually. +dnl +dnl What's more, we need to specify them in a linker script rather than +dnl from build.rs: these options aren't allowed in the cargo:rustc-flags +dnl variable. +RUST_LINKER_OPTIONS="" +if test "x$have_clang" = "xyes"; then + if test "x$CFLAGS_ASAN" != "x"; then + RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS $CFLAGS_ASAN" + fi + if test "x$CFLAGS_UBSAN" != "x"; then + RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS $CFLAGS_UBSAN" + fi +else + if test "x$CFLAGS_ASAN" != "x"; then + RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -lasan" + fi + if test "x$CFLAGS_UBSAN" != "x"; then + RUST_LINKER_OPTIONS="$RUST_LINKER_OPTIONS -lubsan" + fi +fi +AC_SUBST(RUST_LINKER_OPTIONS) + CFLAGS_BUGTRAP="$CFLAGS_FTRAPV $CFLAGS_ASAN $CFLAGS_UBSAN" CFLAGS_CONSTTIME="$CFLAGS_FWRAPV" @@ -1763,14 +1889,6 @@ AC_CHECK_DECLS([mlockall], , , [ #include <sys/mman.h> #endif]) -# Some MinGW environments don't have getpagesize in unistd.h. We don't use -# AC_CHECK_FUNCS(getpagesize), because other environments rename getpagesize -# using macros -AC_CHECK_DECLS([getpagesize], , , [ -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif]) - # Allow user to specify an alternate syslog facility AC_ARG_WITH(syslog-facility, AS_HELP_STRING(--with-syslog-facility=LOG, [syslog facility to use (default=LOG_DAEMON)]), @@ -1908,6 +2026,12 @@ AC_SUBST(BUILDDIR) AH_TEMPLATE([BUILDDIR],[tor's build directory]) AC_DEFINE_UNQUOTED(BUILDDIR,"$BUILDDIR") +if test "x$SRCDIR" = "x"; then + SRCDIR=$(cd "$srcdir"; pwd) +fi +AH_TEMPLATE([SRCDIR],[tor's sourcedir directory]) +AC_DEFINE_UNQUOTED(SRCDIR,"$SRCDIR") + if test "x$CONFDIR" = "x"; then CONFDIR=`eval echo $sysconfdir/tor` fi @@ -2250,6 +2374,8 @@ CPPFLAGS="$CPPFLAGS $TOR_CPPFLAGS_libevent $TOR_CPPFLAGS_openssl $TOR_CPPFLAGS_z AC_CONFIG_FILES([ Doxyfile Makefile + config.rust + link_rust.sh contrib/dist/suse/tor.sh contrib/operator-tools/tor.logrotate contrib/dist/tor.sh diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in index fbf7e9afe4..92de557781 100644 --- a/contrib/win32build/tor-mingw.nsi.in +++ b/contrib/win32build/tor-mingw.nsi.in @@ -8,7 +8,7 @@ !include "LogicLib.nsh" !include "FileFunc.nsh" !insertmacro GetParameters -!define VERSION "0.3.3.10-dev" +!define VERSION "0.3.4.9-dev" !define INSTALLER "tor-${VERSION}-win32.exe" !define WEBSITE "https://www.torproject.org/" !define LICENSE "LICENSE" diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md index 79a6a9f0ce..3711f70198 100644 --- a/doc/HACKING/CodingStandards.md +++ b/doc/HACKING/CodingStandards.md @@ -42,6 +42,23 @@ If you have changed build system components: - For example, if you have changed Makefiles, autoconf files, or anything else that affects the build system. +License issues +============== + +Tor is distributed under the license terms in the LICENSE -- in +brief, the "3-clause BSD license". If you send us code to +distribute with Tor, it needs to be code that we can distribute +under those terms. Please don't send us patches unless you agree +to allow this. + +Some compatible licenses include: + + - 3-clause BSD + - 2-clause BSD + - CC0 Public Domain Dedication + + + How we use Git branches ======================= @@ -417,3 +434,4 @@ the functions that call your function rely on it doing something, then your function should mention that it does that something in the documentation. If you rely on a function doing something beyond what is in its documentation, then you should watch out, or it might do something else later. + diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md index 7c6405e624..d9496c08f7 100644 --- a/doc/HACKING/CodingStandardsRust.md +++ b/doc/HACKING/CodingStandardsRust.md @@ -324,12 +324,26 @@ Here are some additional bits of advice and rules: } } -3. Pass only integer types and bytes over the boundary +3. Pass only C-compatible primitive types and bytes over the boundary - The only non-integer type which may cross the FFI boundary is + Rust's C-compatible primitive types are integers and floats. + These types are declared in the [libc crate](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types). + Most Rust objects have different [representations](https://doc.rust-lang.org/libc/x86_64-unknown-linux-gnu/libc/index.html#types) + in C and Rust, so they can't be passed using FFI. + + Tor currently uses the following Rust primitive types from libc for FFI: + * defined-size integers: `uint32_t` + * native-sized integers: `c_int` + * native-sized floats: `c_double` + * native-sized raw pointers: `* c_void`, `* c_char`, `** c_char` + + TODO: C smartlist to Stringlist conversion using FFI + + The only non-primitive type which may cross the FFI boundary is bytes, e.g. `&[u8]`. This SHOULD be done on the Rust side by - passing a pointer (`*mut libc::c_char`) and a length - (`libc::size_t`). + passing a pointer (`*mut libc::c_char`). The length can be passed + explicitly (`libc::size_t`), or the string can be NUL-byte terminated + C string. One might be tempted to do this via doing `CString::new("blah").unwrap().into_raw()`. This has several problems: diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md index f919d08ec1..a0795076e0 100644 --- a/doc/HACKING/HelpfulTools.md +++ b/doc/HACKING/HelpfulTools.md @@ -4,6 +4,22 @@ Useful tools These aren't strictly necessary for hacking on Tor, but they can help track down bugs. +Travis CI +--------- +It's CI. Looks like this: https://travis-ci.org/torproject/tor. + +Runs automatically on Pull Requests sent to torproject/tor. You can set it up +for your fork to build commits outside of PRs too: + +1. sign up for GitHub: https://github.com/join +2. fork https://github.com/torproject/tor: + https://help.github.com/articles/fork-a-repo/ +3. follow https://docs.travis-ci.com/user/getting-started/#To-get-started-with-Travis-CI. + skip steps involving `.travis.yml` (we already have one). + +Builds should show up on the web at travis-ci.com and on IRC at #tor-ci on +OFTC. If they don't, ask #tor-dev (also on OFTC). + Jenkins ------- diff --git a/doc/HACKING/Module.md b/doc/HACKING/Module.md new file mode 100644 index 0000000000..1028a029d9 --- /dev/null +++ b/doc/HACKING/Module.md @@ -0,0 +1,111 @@ +# Modules in Tor # + +This document describes the build system and coding standards when writing a +module in Tor. + +## What is a module? ## + +In the context of the tor code base, a module is a subsystem that we can +selectively enable or disable, at `configure` time. + +Currently, there is only one module: + + - Directory Authority subsystem (dirauth) + +It is located in its own directory in `src/or/dirauth/`. To disable it, one +need to pass `--disable-module-dirauth` at configure time. All modules are +currently enabled by default. + +## Build System ## + +The changes to the build system are pretty straightforward. + +1. Locate in the `configure.ac` file this define: `m4_define(MODULES`. It + contains a list (white-space separated) of the module in tor. Add yours to + the list. + +2. Use the `AC_ARG_ENABLE([module-dirauth]` template for your new module. We + use the "disable module" approach instead of enabling them one by one. So, + by default, tor will build all the modules. + + This will define the `HAVE_MODULE_<name>` statement which can be used in + the C code to conditionally compile things for your module. And the + `BUILD_MODULE_<name>` is also defined for automake files (e.g: include.am). + +3. In the `src/or/include.am` file, locate the `MODULE_DIRAUTH_SOURCES` value. + You need to create your own `_SOURCES` variable for your module and then + conditionally add the it to `LIBTOR_A_SOURCES` if you should build the + module. + + It is then **very** important to add your SOURCES variable to + `src_or_libtor_testing_a_SOURCES` so the tests can build it. + +4. Do the same for header files, locate `ORHEADERS +=` which always add all + headers of all modules so the symbol can be found for the module entry + points. + +Finally, your module will automatically be included in the +`TOR_MODULES_ALL_ENABLED` variable which is used to build the unit tests. They +always build everything in order to tests everything. + +## Coding ## + +As mentioned above, a module must be isolated in its own directory (name of +the module) in `src/or/`. + +There are couples of "rules" you want to follow: + +* Minimize as much as you can the number of entry points into your module. + Less is always better but of course that doesn't work out for every use + case. However, it is a good thing to always keep that in mind. + +* Do **not** use the `HAVE_MODULE_<name>` define outside of the module code + base. Every entry point should have a second definition if the module is + disabled. For instance: + + ``` + #ifdef HAVE_MODULE_DIRAUTH + + int sr_init(int save_to_disk); + + #else /* HAVE_MODULE_DIRAUTH */ + + static inline int + sr_init(int save_to_disk) + { + (void) save_to_disk; + return 0; + } + + #endif /* HAVE_MODULE_DIRAUTH */ + + ``` + + The main reason for this approach is to avoid having conditional code + everywhere in the code base. It should be centralized as much as possible + which helps maintainability but also avoids conditional spaghetti code + making the code much more difficult to follow/understand. + +* It is possible that you end up with code that needs to be used by the rest + of the code base but is still part of your module. As a good example, if you + look at `src/or/shared_random_client.c`: it contains code needed by the hidden + service subsystem but mainly related to the shared random subsystem very + specific to the dirauth module. + + This is fine but try to keep it as lean as possible and never use the same + filename as the one in the module. For example, this is a bad idea and + should never be done: + + - `src/or/shared_random.c` + - `src/or/dirauth/shared_random.c` + +* When you include headers from the module, **always** use the full module + path in your statement. Example: + + `#include "dirauth/dirvote.h"` + + The main reason is that we do **not** add the module include path by default + so it needs to be specified. But also, it helps our human brain understand + which part comes from a module or not. + + Even **in** the module itself, use the full include path like above. diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md index 6c8fa1331f..e70416c354 100644 --- a/doc/HACKING/ReleasingTor.md +++ b/doc/HACKING/ReleasingTor.md @@ -34,7 +34,7 @@ new Tor release: What about Coverity Scan? - What about clan scan-build? + What about clang scan-build? Does 'make distcheck' complain? diff --git a/doc/include.am b/doc/include.am index 0e8de231e1..7942188eaf 100644 --- a/doc/include.am +++ b/doc/include.am @@ -35,10 +35,15 @@ EXTRA_DIST+= doc/asciidoc-helper.sh \ doc/TUNING \ doc/HACKING/README.1st.md \ doc/HACKING/CodingStandards.md \ + doc/HACKING/CodingStandardsRust.md \ + doc/HACKING/Fuzzing.md \ doc/HACKING/GettingStarted.md \ + doc/HACKING/GettingStartedRust.md \ doc/HACKING/HelpfulTools.md \ doc/HACKING/HowToReview.md \ + doc/HACKING/Module.md \ doc/HACKING/ReleasingTor.md \ + doc/HACKING/Tracing.md \ doc/HACKING/WritingTests.md docdir = @docdir@ diff --git a/doc/tor.1.txt b/doc/tor.1.txt index c089bffbb0..6dd1b60b23 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -311,7 +311,9 @@ GENERAL OPTIONS other than controller connections, and we close (and don't reattempt) any outbound connections. Controllers sometimes use this option to avoid using - the network until Tor is fully configured. (Default: 0) + the network until Tor is fully configured. Tor will make still certain + network-related calls (like DNS lookups) as a part of its configuration + process, even if DisableNetwork is set. (Default: 0) [[ConstrainedSockets]] **ConstrainedSockets** **0**|**1**:: If set, Tor will tell the kernel to attempt to shrink the buffers for all @@ -366,7 +368,8 @@ GENERAL OPTIONS [[ControlSocket]] **ControlSocket** __Path__:: Like ControlPort, but listens on a Unix domain socket, rather than a TCP - socket. '0' disables ControlSocket (Unix and Unix-like systems only.) + socket. '0' disables ControlSocket. (Unix and Unix-like systems only.) + (Default: 0) [[ControlSocketsGroupWritable]] **ControlSocketsGroupWritable** **0**|**1**:: If this option is set to 0, don't allow the filesystem group to read and @@ -788,17 +791,15 @@ GENERAL OPTIONS This is useful when running on flash memory or other media that support only a limited number of writes. (Default: 0) -[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM1__:: +[[CircuitPriorityHalflife]] **CircuitPriorityHalflife** __NUM__:: If this value is set, we override the default algorithm for choosing which - circuit's cell to deliver or relay next. When the value is 0, we - round-robin between the active circuits on a connection, delivering one - cell from each in turn. When the value is positive, we prefer delivering - cells from whichever connection has the lowest weighted cell count, where - cells are weighted exponentially according to the supplied - CircuitPriorityHalflife value (in seconds). If this option is not set at - all, we use the behavior recommended in the current consensus - networkstatus. This is an advanced option; you generally shouldn't have - to mess with it. (Default: not set) + circuit's cell to deliver or relay next. It is delivered first to the + circuit that has the lowest weighted cell count, where cells are weighted + exponentially according to this value (in seconds). If the value is -1, it + is taken from the consensus if possible else it will fallback to the + default value of 30. Minimum: 1, Maximum: 2147483647. This can be defined + as a float value. This is an advanced option; you generally shouldn't have + to mess with it. (Default: -1) [[CountPrivateBandwidth]] **CountPrivateBandwidth** **0**|**1**:: If this option is set, then Tor's rate-limiting applies not only to @@ -815,10 +816,9 @@ GENERAL OPTIONS [[NoExec]] **NoExec** **0**|**1**:: If this option is set to 1, then Tor will never launch another - executable, regardless of the settings of PortForwardingHelper, - ClientTransportPlugin, or ServerTransportPlugin. Once this - option has been set to 1, it cannot be set back to 0 without - restarting Tor. (Default: 0) + executable, regardless of the settings of ClientTransportPlugin + or ServerTransportPlugin. Once this option has been set to 1, + it cannot be set back to 0 without restarting Tor. (Default: 0) [[Schedulers]] **Schedulers** **KIST**|**KISTLite**|**Vanilla**:: Specify the scheduler type that tor should use. The scheduler is @@ -1296,9 +1296,11 @@ The following options are useful only for clients (that is, if 2 minutes) [[TokenBucketRefillInterval]] **TokenBucketRefillInterval** __NUM__ [**msec**|**second**]:: - Set the refill interval of Tor's token bucket to NUM milliseconds. - NUM must be between 1 and 1000, inclusive. Note that the configured - bandwidth limits are still expressed in bytes per second: this + Set the refill delay interval of Tor's token bucket to NUM milliseconds. + NUM must be between 1 and 1000, inclusive. When Tor is out of bandwidth, + on a connection or globally, it will wait up to this long before it tries + to use that connection again. + Note that bandwidth limits are still expressed in bytes per second: this option only affects the frequency with which Tor checks to see whether previously exhausted connections may read again. Can not be changed while tor is running. (Default: 100 msec) @@ -1355,6 +1357,13 @@ The following options are useful only for clients (that is, if number from the guard-n-primary-guards-to-use consensus parameter, and default to 1 if the consensus parameter isn't set. (Default: 0) +[[NumPrimaryGuards]] **NumPrimaryGuards** __NUM__:: + If UseEntryGuards is set to 1, we will try to pick NUM routers for our + primary guard list, which is the set of routers we strongly prefer when + connecting to the Tor network. If NUM is 0, we try to learn the number from + the guard-n-primary-guards consensus parameter, and default to 3 if the + consensus parameter isn't set. (Default: 0) + [[NumDirectoryGuards]] **NumDirectoryGuards** __NUM__:: If UseEntryGuards is set to 1, we try to make sure we have at least NUM routers to use as directory guards. If this option is set to 0, use the @@ -1408,7 +1417,7 @@ The following options are useful only for clients (that is, if [[HTTPTunnelPort]] **HTTPTunnelPort** \['address':]__port__|**auto** [_isolation flags_]:: Open this port to listen for proxy connections using the "HTTP CONNECT" - protocol instead of SOCKS. Set this to 0 + protocol instead of SOCKS. Set this to 0 if you don't want to allow "HTTP CONNECT" connections. Set the port to "auto" to have Tor pick a port for you. This directive can be specified multiple times to bind to multiple addresses/ports. See @@ -1448,7 +1457,7 @@ The following options are useful only for clients (that is, if Set this to "default", or leave it unconfigured, to use regular IPTables on Linux, or to use pf +rdr-to+ rules on *BSD systems. + + - (Default: "default".) + (Default: "default") [[NATDPort]] **NATDPort** \['address':]__port__|**auto** [_isolation flags_]:: Open this port to listen for connections from old versions of ipfw (as @@ -1585,6 +1594,14 @@ The following options are useful only for clients (that is, if which means that nodes specified in ExcludeNodes will not be picked. + + When either this option or HSLayer3Nodes are set, the /16 subnet + and node family restrictions are removed for hidden service + circuits. Additionally, we allow the guard node to be present + as the Rend, HSDir, and IP node, and as the hop before it. This + is done to prevent the adversary from inferring information + about our guard, layer2, and layer3 node choices at later points + in the path. + + This option is meant to be managed by a Tor controller such as https://github.com/mikeperry-tor/vanguards that selects and updates this set of nodes for you. Hence it does not do load @@ -1630,6 +1647,14 @@ The following options are useful only for clients (that is, if ExcludeNodes have higher priority than HSLayer3Nodes, which means that nodes specified in ExcludeNodes will not be picked. + + + When either this option or HSLayer2Nodes are set, the /16 subnet + and node family restrictions are removed for hidden service + circuits. Additionally, we allow the guard node to be present + as the Rend, HSDir, and IP node, and as the hop before it. This + is done to prevent the adversary from inferring information + about our guard, layer2, and layer3 node choices at later points + in the path. + This option is meant to be managed by a Tor controller such as https://github.com/mikeperry-tor/vanguards that selects and @@ -1740,34 +1765,31 @@ The following options are useful only for clients (that is, if prevent your Tor client from bootstrapping. If this option is negative, Tor will use a default value chosen by the directory authorities. If the directory authorities do not choose a value, Tor will default to 0.6. - (Default: -1.) + (Default: -1) -[[ClientBootstrapConsensusAuthorityDownloadSchedule]] **ClientBootstrapConsensusAuthorityDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download consensuses from authorities +[[ClientBootstrapConsensusAuthorityDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download consensuses from authorities if they are bootstrapping (that is, they don't have a usable, reasonably live consensus). Only used by clients fetching from a list of fallback directory mirrors. This schedule is advanced by (potentially concurrent) connection attempts, unlike other schedules, which are advanced by - connection failures. (Default: 6, 11, 3600, 10800, 25200, 54000, 111600, - 262800) + connection failures. (Default: 6) -[[ClientBootstrapConsensusFallbackDownloadSchedule]] **ClientBootstrapConsensusFallbackDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download consensuses from fallback +[[ClientBootstrapConsensusFallbackDownloadInitialDelay]] **ClientBootstrapConsensusFallbackDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download consensuses from fallback directory mirrors if they are bootstrapping (that is, they don't have a usable, reasonably live consensus). Only used by clients fetching from a list of fallback directory mirrors. This schedule is advanced by (potentially concurrent) connection attempts, unlike other schedules, - which are advanced by connection failures. (Default: 0, 1, 4, 11, 3600, - 10800, 25200, 54000, 111600, 262800) + which are advanced by connection failures. (Default: 0) -[[ClientBootstrapConsensusAuthorityOnlyDownloadSchedule]] **ClientBootstrapConsensusAuthorityOnlyDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download consensuses from authorities +[[ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay]] **ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download consensuses from authorities if they are bootstrapping (that is, they don't have a usable, reasonably live consensus). Only used by clients which don't have or won't fetch from a list of fallback directory mirrors. This schedule is advanced by (potentially concurrent) connection attempts, unlike other schedules, - which are advanced by connection failures. (Default: 0, 3, 7, 3600, - 10800, 25200, 54000, 111600, 262800) + which are advanced by connection failures. (Default: 0) [[ClientBootstrapConsensusMaxInProgressTries]] **ClientBootstrapConsensusMaxInProgressTries** __NUM__:: Try this many simultaneous connections to download a consensus before @@ -1898,7 +1920,7 @@ is non-zero): If you want to use a reduced exit policy rather than the default exit policy, set "ReducedExitPolicy 1". If you want to _replace_ the default exit policy with your custom exit policy, end your exit policy with either - a reject *:* or an accept *:*. Otherwise, you’re _augmenting_ (prepending + a reject *:* or an accept *:*. Otherwise, you're _augmenting_ (prepending to) the default or reduced exit policy. + + The default exit policy is: @@ -2061,6 +2083,8 @@ is non-zero): [[Nickname]] **Nickname** __name__:: Set the server's nickname to \'name'. Nicknames must be between 1 and 19 characters inclusive, and must contain only the characters [a-zA-Z0-9]. + If not set, **Unnamed** will be used. Relays can always be uniquely identified + by their identity fingerprints. [[NumCPUs]] **NumCPUs** __num__:: How many processes to use at once for decrypting onionskins and other @@ -2096,18 +2120,6 @@ is non-zero): For obvious reasons, NoAdvertise and NoListen are mutually exclusive, and IPv4Only and IPv6Only are mutually exclusive. -[[PortForwarding]] **PortForwarding** **0**|**1**:: - Attempt to automatically forward the DirPort and ORPort on a NAT router - connecting this Tor server to the Internet. If set, Tor will try both - NAT-PMP (common on Apple routers) and UPnP (common on routers from other - manufacturers). (Default: 0) - -[[PortForwardingHelper]] **PortForwardingHelper** __filename__|__pathname__:: - If PortForwarding is set, use this executable to configure the forwarding. - If set to a filename, the system path will be searched for the executable. - If set to a path, only the specified path will be executed. - (Default: tor-fw-helper) - [[PublishServerDescriptor]] **PublishServerDescriptor** **0**|**1**|**v3**|**bridge**,**...**:: This option specifies which descriptors Tor will publish when acting as a relay. You can @@ -2271,7 +2283,8 @@ is non-zero): sent and received by this relay, in addition to total cell counts. These statistics are rounded, and omitted if traffic is low. This information is important for load balancing decisions related to padding. - (Default: 1) + If ExtraInfoStatistics is enabled, it will be published + as a part of extra-info document. (Default: 1) [[DirReqStatistics]] **DirReqStatistics** **0**|**1**:: Relays and bridges only. @@ -2370,6 +2383,11 @@ is non-zero): KeywDirectory. If the option is set to 1, make the KeyDirectory readable by the default GID. (Default: 0) +[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**:: + Tells an authority, or other node tracking node reliability and history, + that fine-grained information about nodes can be discarded when it hasn't + changed for a given amount of time. (Default: 24 hours) + DIRECTORY SERVER OPTIONS ------------------------ @@ -2739,11 +2757,6 @@ on the public Tor network. different identity. This feature is used to migrate directory authority keys in the event of a compromise. (Default: 0) -[[RephistTrackTime]] **RephistTrackTime** __N__ **seconds**|**minutes**|**hours**|**days**|**weeks**:: - Tells an authority, or other node tracking node reliability and history, - that fine-grained information about nodes can be discarded when it hasn't - changed for a given amount of time. (Default: 24 hours) - [[AuthDirHasIPv6Connectivity]] **AuthDirHasIPv6Connectivity** **0**|**1**:: Authoritative directories only. When set to 0, OR ports with an IPv6 address are not included in the authority's votes. When set to 1, @@ -2911,12 +2924,9 @@ The following options are used for running a testing Tor network. AssumeReachable 1 AuthDirMaxServersPerAddr 0 AuthDirMaxServersPerAuthAddr 0 - ClientBootstrapConsensusAuthorityDownloadSchedule 0, 2, - 4 (for 40 seconds), 8, 16, 32, 60 - ClientBootstrapConsensusFallbackDownloadSchedule 0, 1, - 4 (for 40 seconds), 8, 16, 32, 60 - ClientBootstrapConsensusAuthorityOnlyDownloadSchedule 0, 1, - 4 (for 40 seconds), 8, 16, 32, 60 + ClientBootstrapConsensusAuthorityDownloadInitialDelay 0 + ClientBootstrapConsensusFallbackDownloadInitialDelay 0 + ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay 0 ClientDNSRejectInternalAddresses 0 ClientRejectInternalAddresses 0 CountPrivateBandwidth 1 @@ -2931,17 +2941,16 @@ The following options are used for running a testing Tor network. TestingV3AuthInitialDistDelay 20 seconds TestingAuthDirTimeToLearnReachability 0 minutes TestingEstimatedDescriptorPropagationTime 0 minutes - TestingServerDownloadSchedule 0, 0, 0, 5, 10, 15, 20, 30, 60 - TestingClientDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60 - TestingServerConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60 - TestingClientConsensusDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60 - TestingBridgeDownloadSchedule 10, 30, 60 - TestingBridgeBootstrapDownloadSchedule 0, 0, 5, 10, 15, 20, 30, 60 + TestingServerDownloadInitialDelay 0 + TestingClientDownloadInitialDelay 0 + TestingServerConsensusDownloadInitialDelay 0 + TestingClientConsensusDownloadInitialDelay 0 + TestingBridgeDownloadInitialDelay 10 + TestingBridgeBootstrapDownloadInitialDelay 0 TestingClientMaxIntervalWithoutRequest 5 seconds TestingDirConnectionMaxStall 30 seconds TestingEnableConnBwEvent 1 TestingEnableCellStatsEvent 1 - TestingEnableTbEmptyEvent 1 [[TestingV3AuthInitialVotingInterval]] **TestingV3AuthInitialVotingInterval** __N__ **minutes**|**hours**:: Like V3AuthVotingInterval, but for initial voting interval before the first @@ -2976,37 +2985,31 @@ The following options are used for running a testing Tor network. Minimum value for the Fast flag. Overrides the ordinary minimum taken from the consensus when TestingTorNetwork is set. (Default: 0.) -[[TestingServerDownloadSchedule]] **TestingServerDownloadSchedule** __N__,__N__,__...__:: - Schedule for when servers should download things in general. Changing this - requires that **TestingTorNetwork** is set. (Default: 0, 0, 0, 60, 60, 120, - 300, 900, 2147483647) +[[TestingServerDownloadInitialDelay]] **TestingServerDownloadInitialDelay** __N__:: + Initial delay in seconds for when servers should download things in general. Changing this + requires that **TestingTorNetwork** is set. (Default: 0) -[[TestingClientDownloadSchedule]] **TestingClientDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download things in general. Changing this - requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600, - 2147483647) +[[TestingClientDownloadInitialDelay]] **TestingClientDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download things in general. Changing this + requires that **TestingTorNetwork** is set. (Default: 0) -[[TestingServerConsensusDownloadSchedule]] **TestingServerConsensusDownloadSchedule** __N__,__N__,__...__:: - Schedule for when servers should download consensuses. Changing this - requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600, - 1800, 1800, 1800, 1800, 1800, 3600, 7200) +[[TestingServerConsensusDownloadInitialDelay]] **TestingServerConsensusDownloadInitialDelay** __N__:: + Initial delay in seconds for when servers should download consensuses. Changing this + requires that **TestingTorNetwork** is set. (Default: 0) -[[TestingClientConsensusDownloadSchedule]] **TestingClientConsensusDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download consensuses. Changing this - requires that **TestingTorNetwork** is set. (Default: 0, 0, 60, 300, 600, - 1800, 3600, 3600, 3600, 10800, 21600, 43200) +[[TestingClientConsensusDownloadInitialDelay]] **TestingClientConsensusDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download consensuses. Changing this + requires that **TestingTorNetwork** is set. (Default: 0) -[[TestingBridgeDownloadSchedule]] **TestingBridgeDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download each bridge descriptor when they +[[TestingBridgeDownloadInitialDelay]] **TestingBridgeDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download each bridge descriptor when they know that one or more of their configured bridges are running. Changing - this requires that **TestingTorNetwork** is set. (Default: 10800, 25200, - 54000, 111600, 262800) + this requires that **TestingTorNetwork** is set. (Default: 10800) -[[TestingBridgeBootstrapDownloadSchedule]] **TestingBridgeBootstrapDownloadSchedule** __N__,__N__,__...__:: - Schedule for when clients should download each bridge descriptor when they +[[TestingBridgeBootstrapDownloadInitialDelay]] **TestingBridgeBootstrapDownloadInitialDelay** __N__:: + Initial delay in seconds for when clients should download each bridge descriptor when they have just started, or when they can not contact any of their bridges. - Changing this requires that **TestingTorNetwork** is set. (Default: 0, 30, - 90, 600, 3600, 10800, 25200, 54000, 111600, 262800) + Changing this requires that **TestingTorNetwork** is set. (Default: 0) [[TestingClientMaxIntervalWithoutRequest]] **TestingClientMaxIntervalWithoutRequest** __N__ **seconds**|**minutes**:: When directory clients have only a few descriptors to request, they batch @@ -3079,11 +3082,6 @@ The following options are used for running a testing Tor network. events. Changing this requires that **TestingTorNetwork** is set. (Default: 0) -[[TestingEnableTbEmptyEvent]] **TestingEnableTbEmptyEvent** **0**|**1**:: - If this option is set, then Tor controllers may register for TB_EMPTY - events. Changing this requires that **TestingTorNetwork** is set. - (Default: 0) - [[TestingMinExitFlagThreshold]] **TestingMinExitFlagThreshold** __N__ **KBytes**|**MBytes**|**GBytes**|**TBytes**|**KBits**|**MBits**|**GBits**|**TBits**:: Sets a lower-bound for assigning an exit flag when running as an authority on a testing network. Overrides the usual default lower bound diff --git a/link_rust.sh.in b/link_rust.sh.in new file mode 100644 index 0000000000..59f4142baa --- /dev/null +++ b/link_rust.sh.in @@ -0,0 +1,10 @@ +#!/bin/sh +# +# A linker script used when building Rust tests. Autoconf makes link_rust.sh +# from link_rust_sh.in, and uses it to pass extra options to the linker +# when linking Rust stuff. +# +# We'd like to remove the need for this, but build.rs doesn't let us pass +# -static-libasan and -static-libubsan to the linker. + +$CCLD @RUST_LINKER_OPTIONS@ "$@" diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index fda57d2ae8..5bef22d900 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -98,6 +98,7 @@ def get_id_name(s): for fuzzer in FUZZERS: idname = get_id_name(fuzzer) print("""\ +if UNITTESTS_ENABLED src_test_fuzz_fuzz_{name}_SOURCES = \\ src/test/fuzz/fuzzing_common.c \\ src/test/fuzz/fuzz_{name}.c @@ -105,11 +106,14 @@ src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS) +endif """.format(name=idname)) +print("if UNITTESTS_ENABLED") print("FUZZERS = \\") print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer) for fuzzer in FUZZERS)) +print("endif") print("\n# ===== libfuzzer") print("\nif LIBFUZZER_ENABLED") @@ -117,12 +121,14 @@ print("\nif LIBFUZZER_ENABLED") for fuzzer in FUZZERS: idname = get_id_name(fuzzer) print("""\ +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_{name}_SOURCES = \\ $(src_test_fuzz_fuzz_{name}_SOURCES) src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS) +endif """.format(name=idname)) print("LIBFUZZER_FUZZERS = \\") @@ -140,10 +146,12 @@ print("if OSS_FUZZ_ENABLED") for fuzzer in FUZZERS: idname = get_id_name(fuzzer) print("""\ +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\ $(src_test_fuzz_fuzz_{name}_SOURCES) src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif """.format(name=idname)) print("OSS_FUZZ_FUZZERS = \\") diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py index c854830e18..b093463e08 100755 --- a/scripts/maint/updateFallbackDirs.py +++ b/scripts/maint/updateFallbackDirs.py @@ -173,12 +173,6 @@ MAX_LIST_FILE_SIZE = 1024 * 1024 # meant that we had to rebuild the list more often. We want fallbacks to be # stable for 2 years, so we set it to a few months. # -# There was a bug in Tor 0.2.8.1-alpha and earlier where a relay temporarily -# submits a 0 DirPort when restarted. -# This causes OnionOO to (correctly) reset its stability timer. -# Affected relays should upgrade to Tor 0.2.9 or later, which has a fix -# for this issue. -# # If a relay changes address or port, that's it, it's not useful any more, # because clients can't find it ADDRESS_AND_PORT_STABLE_DAYS = 90 diff --git a/scripts/test/appveyor-irc-notify.py b/scripts/test/appveyor-irc-notify.py new file mode 100644 index 0000000000..cfe0afe7ae --- /dev/null +++ b/scripts/test/appveyor-irc-notify.py @@ -0,0 +1,219 @@ +# coding=utf8 +# Copyright (C) 2015-2016 Christopher R. Wood +# Copyright (c) 2018 The Tor Project +# Copyright (c) 2018 isis agora lovecruft +# +# From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py +# and: https://github.com/gridsync/gridsync +# +# Modified by nexB on October 2016: +# - rework the handling of environment variables. +# - made the script use functions +# - support only Appveyor loading its environment variable to craft IRC notices. +# +# Modified by isis agora lovecruft <isis@torproject.org> in 2018: +# - Make IRC server configurable. +# - Make bot IRC nick deterministic. +# - Make bot join the channel rather than sending NOTICE messages externally. +# - Fix a bug which always caused sys.exit() to be logged as a traceback. +# - Actually reset the IRC colour codes after printing. +# +# Modified by Marcin Cieślak in 2018: +# - Accept UTF-8 +# - only guess github URLs +# - stop using ANSI colors +# +# Modified by teor in 2018: +# - fix github provider detection ('gitHub' or 'gitHubEnterprise', apparently) +# - make short commits 10 hexdigits long (that's what git does for tor) +# - generate correct branches and URLs for pull requests and tags +# - switch to one URL per line + +# This program is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation; +# either version 2 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this +# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, +# Fifth Floor, Boston, MA 02110-1301 USA. + +"""Simple AppVeyor IRC notification script. + +The first argument is an IRC server and port; the second is the channel. Other +arguments passed to the script will be sent as notice messages content and any +{var}-formatted environment variables will be expanded automatically, replaced +with a corresponding Appveyor environment variable value. Use commas to +delineate multiple messages. + + +Example: +export APPVEYOR_ACCOUNT_NAME=isislovecruft +export APPVEYOR_BUILD_VERSION=1 +export APPVEYOR_PROJECT_NAME=tor +export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number +export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title +export APPVEYOR_REPO_BRANCH=repo_branch +export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8 +export APPVEYOR_REPO_COMMIT_AUTHOR=isislovecruft +export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test" +export APPVEYOR_REPO_COMMIT_TIMESTAMP=2018-04-23 +export APPVEYOR_REPO_NAME=isislovecruft/tor +export APPVEYOR_REPO_PROVIDER=github +export APPVEYOR_URL=https://ci.appveyor.com +python ./appveyor-irc-notify.py irc.oftc.net:6697 tor-ci '{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}','Build #{build_version} passed. Details: {build_url} | Commit: {commit_url} + +See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples +in Appveyor's YAML: + + on_success: + - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success + on_failure: + - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure +""" + +from __future__ import print_function +from __future__ import absolute_import + +import os +import random +import socket +import ssl +import sys +import time + + +def appveyor_vars(): + """ + Return a dict of key value crafted from appveyor environment variables. + """ + + vars = dict([ + ( + v.replace('APPVEYOR_', '').lower(), + os.getenv(v, '').decode('utf-8') + ) for v in [ + 'APPVEYOR_ACCOUNT_NAME', + 'APPVEYOR_BUILD_VERSION', + 'APPVEYOR_PROJECT_NAME', + 'APPVEYOR_PULL_REQUEST_HEAD_COMMIT', + 'APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH', + 'APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME', + 'APPVEYOR_PULL_REQUEST_NUMBER', + 'APPVEYOR_PULL_REQUEST_TITLE', + 'APPVEYOR_REPO_BRANCH', + 'APPVEYOR_REPO_COMMIT', + 'APPVEYOR_REPO_COMMIT_AUTHOR', + 'APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL', + 'APPVEYOR_REPO_COMMIT_MESSAGE', + 'APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED', + 'APPVEYOR_REPO_COMMIT_TIMESTAMP', + 'APPVEYOR_REPO_NAME', + 'APPVEYOR_REPO_PROVIDER', + 'APPVEYOR_REPO_TAG_NAME', + 'APPVEYOR_URL', + ] + ]) + + BUILD_FMT = u'{url}/project/{account_name}/{project_name}/build/{build_version}' + + if vars["repo_tag_name"]: + BRANCH_FMT = u'{repo_name} {repo_tag_name} {short_commit}' + else: + BRANCH_FMT = u'{repo_name} {repo_branch} {short_commit}' + + vars.update(head_commit=vars["repo_commit"]) + + if vars["repo_provider"].lower().startswith('github'): + COMMIT_FMT = u'https://github.com/{repo_name}/commit/{repo_commit}' + if vars["pull_request_number"]: + vars.update(head_commit=vars["pull_request_head_commit"]) + BRANCH_FMT = u'{repo_name} {repo_branch} pull {pull_request_head_repo_name} {pull_request_head_repo_branch} {short_commit}' + COMMIT_FMT = u'https://github.com/{pull_request_head_repo_name}/commit/{pull_request_head_commit}' + PULL_FMT = u'https://github.com/{repo_name}/pull/{pull_request_number}' + vars.update(pull_url=PULL_FMT.format(**vars)) + vars.update(commit_url=COMMIT_FMT.format(**vars)) + + vars.update(short_commit=vars["head_commit"][:10]) + + vars.update( + build_url=BUILD_FMT.format(**vars), + branch_detail=BRANCH_FMT.format(**vars), + ) + return vars + + +def notify(): + """ + Send IRC notification + """ + apvy_vars = appveyor_vars() + + server, port = sys.argv[1].rsplit(":", 1) + channel = sys.argv[2] + success = sys.argv[3] == "success" + failure = sys.argv[3] == "failure" + + if success or failure: + messages = [] + messages.append(u"{branch_detail} - {repo_commit_author}: {repo_commit_message}") + + if success: + messages.append(u"Build #{build_version} passed. Details: {build_url}") + if failure: + messages.append(u"Build #{build_version} failed. Details: {build_url}") + + if "commit_url" in apvy_vars: + messages.append(u"Commit: {commit_url}") + + if "pull_url" in apvy_vars: + messages.append(u"Pull: {pull_url}") + + else: + messages = sys.argv[3:] + messages = ' '.join(messages) + messages = messages.decode("utf-8").split(',') + + print(repr(apvy_vars)) + messages = [msg.format(**apvy_vars).strip() for msg in messages] + + irc_username = 'appveyor-ci' + irc_nick = irc_username + + # establish connection + irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + irc_sock.connect((socket.gethostbyname(server), int(port))) + irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode()) + irc_sock.send('JOIN #{0}\r\n'.format(channel).encode()) + irc_file = irc_sock.makefile() + + while irc_file: + line = irc_file.readline() + print(line.rstrip()) + response = line.split() + + if response[0] == 'PING': + irc_file.send('PONG {}\r\n'.format(response[1]).encode()) + + elif response[1] == '433': + irc_sock.send('NICK {}\r\n'.format(irc_nick).encode()) + + elif response[1] == '001': + time.sleep(5) + # send notification + for msg in messages: + print(u'PRIVMSG #{} :{}'.format(channel, msg).encode("utf-8")) + irc_sock.send(u'PRIVMSG #{} :{}\r\n'.format(channel, msg).encode("utf-8")) + time.sleep(5) + return + + +if __name__ == '__main__': + try: + notify() + except: + import traceback + print('ERROR: Failed to send notification: \n' + traceback.format_exc()) diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index ed8874d2d3..6179dff63e 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -10,12 +10,12 @@ DIRB="$2" for B in $DIRB/*; do A=$DIRA/`basename $B` if [ -f $A ]; then - perl -pe 's/^\s*\!*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" + perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" else cat /dev/null > "$A.tmp" fi - perl -pe 's/^\s*\!*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" - diff -u "$A.tmp" "$B.tmp" + perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" + diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;' rm "$A.tmp" "$B.tmp" done diff --git a/src/common/address.c b/src/common/address.c index 5074c1ccf0..9446675712 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1700,7 +1700,7 @@ get_interface_address6_via_udp_socket_hack,(int severity, sa_family_t family, tor_addr_t *addr)) { - struct sockaddr_storage my_addr, target_addr; + struct sockaddr_storage target_addr; int sock=-1, r=-1; socklen_t addr_len; @@ -1743,21 +1743,19 @@ get_interface_address6_via_udp_socket_hack,(int severity, goto err; } - if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) { + if (tor_addr_from_getsockname(addr, sock) < 0) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s", tor_socket_strerror(e)); goto err; } - if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) { - if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) { - log_fn(severity, LD_NET, "Address that we determined via UDP socket" - " magic is unsuitable for public comms."); - } else { - r=0; - } - } + if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) { + log_fn(severity, LD_NET, "Address that we determined via UDP socket" + " magic is unsuitable for public comms."); + } else { + r=0; + } err: if (sock >= 0) diff --git a/src/common/address_set.c b/src/common/address_set.c index f61fa294e0..b2f4bb4c95 100644 --- a/src/common/address_set.c +++ b/src/common/address_set.c @@ -15,7 +15,7 @@ #include "address.h" #include "compat.h" #include "container.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "siphash.h" diff --git a/src/common/aes.c b/src/common/aes.c index 4d4a2d773a..0c4f9b9bca 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -16,8 +16,8 @@ #include <ws2tcpip.h> #endif +#include "compat_openssl.h" #include <openssl/opensslv.h> -#include "crypto.h" #include "crypto_openssl_mgt.h" #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0) @@ -41,6 +41,7 @@ ENABLE_GCC_WARNING(redundant-decls) #include "util.h" #include "torlog.h" #include "di_ops.h" +#include "crypto_util.h" #ifdef ANDROID /* Android's OpenSSL seems to have removed all of its Engine support. */ @@ -116,7 +117,11 @@ aes_cipher_free_(aes_cnt_cipher_t *cipher_) if (!cipher_) return; EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_; +#ifdef OPENSSL_1_1_API + EVP_CIPHER_CTX_reset(cipher); +#else EVP_CIPHER_CTX_cleanup(cipher); +#endif EVP_CIPHER_CTX_free(cipher); } void diff --git a/src/common/buffers.h b/src/common/buffers.h index 22a5f7bfa3..4275152de2 100644 --- a/src/common/buffers.h +++ b/src/common/buffers.h @@ -13,7 +13,6 @@ #define TOR_BUFFERS_H #include "compat.h" -#include "compat.h" #include "torint.h" #include "testsupport.h" diff --git a/src/common/compat.c b/src/common/compat.c index 7d9add50b2..6fdd6ecf00 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -100,7 +100,6 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) /* Only use the linux prctl; the IRIX prctl is totally different */ #include <sys/prctl.h> #elif defined(__APPLE__) -#include <sys/types.h> #include <sys/ptrace.h> #endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) || ... */ @@ -116,7 +115,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #ifdef HAVE_SIGNAL_H #include <signal.h> #endif -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MMAP #include <sys/mman.h> #endif #ifdef HAVE_SYS_SYSLIMITS_H @@ -204,25 +203,17 @@ tor_rename(const char *path_old, const char *path_new) sandbox_intern_string(path_new)); } -/* Some MinGW builds have sys/mman.h, but not the corresponding symbols. - * Other configs rename the symbols using macros (including getpagesize). - * So check for sys/mman.h and unistd.h, and a getpagesize declaration. */ -#if (defined(HAVE_SYS_MMAN_H) && defined(HAVE_UNISTD_H) && \ - defined(HAVE_DECL_GETPAGESIZE)) -#define COMPAT_HAS_MMAN_AND_PAGESIZE -#endif - -#if defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || \ - defined(RUNNING_DOXYGEN) +#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) /** Try to create a memory mapping for <b>filename</b> and return it. On - * failure, return NULL. Sets errno properly, using ERANGE to mean - * "empty file". */ + * failure, return NULL. Sets errno properly, using ERANGE to mean + * "empty file". Must only be called on trusted Tor-owned files, as changing + * the underlying file's size causes unspecified behavior. */ tor_mmap_t * tor_mmap_file(const char *filename) { int fd; /* router file */ char *string; - int page_size, result; + int result; tor_mmap_t *res; size_t size, filesize; struct stat st; @@ -251,13 +242,6 @@ tor_mmap_file(const char *filename) return NULL; } size = filesize = (size_t)(st.st_size); - /* - * Should we check for weird crap like mmapping a named pipe here, - * or just wait for if (!size) below to fail? - */ - /* ensure page alignment */ - page_size = getpagesize(); - size += (size%page_size) ? page_size-(size%page_size) : 0; if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) { log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename); @@ -418,40 +402,8 @@ tor_munmap_file(tor_mmap_t *handle) return 0; } #else -tor_mmap_t * -tor_mmap_file(const char *filename) -{ - struct stat st; - char *res = read_file_to_str(filename, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - tor_mmap_t *handle; - if (! res) - return NULL; - handle = tor_malloc_zero(sizeof(tor_mmap_t)); - handle->data = res; - handle->size = st.st_size; - return handle; -} - -/** Unmap the file mapped with tor_mmap_file(), and return 0 for success - * or -1 for failure. - */ - -int -tor_munmap_file(tor_mmap_t *handle) -{ - char *d = NULL; - if (handle == NULL) - return 0; - - d = (char*)handle->data; - tor_free(d); - memwipe(handle, 0, sizeof(tor_mmap_t)); - tor_free(handle); - - /* Can't fail in this mmap()/munmap()-free case */ - return 0; -} -#endif /* defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || ... || ... */ +#error "cannot implement tor_mmap_file" +#endif /* defined(HAVE_MMAP) || ... || ... */ /** Replacement for snprintf. Differs from platform snprintf in two * ways: First, always NUL-terminates its output. Second, always @@ -1392,6 +1344,24 @@ tor_getsockname,(tor_socket_t sock, struct sockaddr *address, return getsockname(sock, address, address_len); } +/** + * Find the local address associated with the socket <b>sock</b>, and + * place it in *<b>addr_out</b>. Return 0 on success, -1 on failure. + * + * (As tor_getsockname, but instead places the result in a tor_addr_t.) */ +int +tor_addr_from_getsockname(tor_addr_t *addr_out, tor_socket_t sock) +{ + struct sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + memset(&ss, 0, sizeof(ss)); + + if (tor_getsockname(sock, (struct sockaddr *) &ss, &ss_len) < 0) + return -1; + + return tor_addr_from_sockaddr(addr_out, (struct sockaddr *)&ss, NULL); +} + /** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1 * on failure. */ diff --git a/src/common/compat.h b/src/common/compat.h index 3088e68355..c7e7f8d9ef 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -318,12 +318,12 @@ typedef struct tor_mmap_t { size_t size; /**< Size of the file. */ /* None of the fields below should be accessed from outside compat.c */ -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MMAP size_t mapping_size; /**< Size of the actual mapping. (This is this file * size, rounded up to the nearest page.) */ #elif defined _WIN32 HANDLE mmap_handle; -#endif /* defined(HAVE_SYS_MMAN_H) || ... */ +#endif /* defined(HAVE_MMAP) || ... */ } tor_mmap_t; @@ -510,6 +510,8 @@ int get_n_open_sockets(void); MOCK_DECL(int, tor_getsockname,(tor_socket_t socket, struct sockaddr *address, socklen_t *address_len)); +struct tor_addr_t; +int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags) #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 735385557c..e60eb148d8 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -11,7 +11,7 @@ #define COMPAT_LIBEVENT_PRIVATE #include "compat_libevent.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "torlog.h" @@ -79,6 +79,43 @@ tor_event_free_(struct event *ev) /** Global event base for use by the main thread. */ static struct event_base *the_event_base = NULL; +/** + * @defgroup postloop post-loop event helpers + * + * If we're not careful, Libevent can susceptible to infinite event chains: + * one event can activate another, whose callback activates another, whose + * callback activates another, ad infinitum. While this is happening, + * Libevent won't be checking timeouts, socket-based events, signals, and so + * on. + * + * We solve this problem by marking some events as "post-loop". A post-loop + * event behaves like any ordinary event, but any events that _it_ activates + * cannot run until Libevent has checked for other events at least once. + * + * @{ */ + +/** + * An event that stops Libevent from running any more events on the current + * iteration of its loop, until it has re-checked for socket events, signal + * events, timeouts, etc. + */ +static struct event *rescan_mainloop_ev = NULL; + +/** + * Callback to implement rescan_mainloop_ev: it simply exits the mainloop, + * and relies on Tor to re-enter the mainloop since no error has occurred. + */ +static void +rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg) +{ + (void)fd; + (void)events; + struct event_base *the_base = arg; + event_base_loopbreak(the_base); +} + +/** @} */ + /* This is what passes for version detection on OSX. We set * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before * 10.4.0 (aka 1040). */ @@ -130,6 +167,15 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg) /* LCOV_EXCL_STOP */ } + rescan_mainloop_ev = event_new(the_event_base, -1, 0, + rescan_mainloop_cb, the_event_base); + if (!rescan_mainloop_ev) { + /* LCOV_EXCL_START */ + log_err(LD_GENERAL, "Unable to create rescan event: cannot continue."); + exit(1); // exit ok: libevent is broken. + /* LCOV_EXCL_STOP */ + } + log_info(LD_GENERAL, "Initialized libevent version %s using method %s. Good.", event_get_version(), tor_libevent_get_method()); @@ -207,10 +253,39 @@ periodic_timer_new(struct event_base *base, } timer->cb = cb; timer->data = data; - event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/ + periodic_timer_launch(timer, tv); return timer; } +/** + * Launch the timer <b>timer</b> to run at <b>tv</b> from now, and every + * <b>tv</b> thereafter. + * + * If the timer is already enabled, this function does nothing. + */ +void +periodic_timer_launch(periodic_timer_t *timer, const struct timeval *tv) +{ + tor_assert(timer); + if (event_pending(timer->ev, EV_TIMEOUT, NULL)) + return; + event_add(timer->ev, tv); +} + +/** + * Disable the provided <b>timer</b>, but do not free it. + * + * You can reenable the same timer later with periodic_timer_launch. + * + * If the timer is already disabled, this function does nothing. + */ +void +periodic_timer_disable(periodic_timer_t *timer) +{ + tor_assert(timer); + (void) event_del(timer->ev); +} + /** Stop and free a periodic timer */ void periodic_timer_free_(periodic_timer_t *timer) @@ -221,6 +296,173 @@ periodic_timer_free_(periodic_timer_t *timer) tor_free(timer); } +/** + * Type used to represent events that run directly from the main loop, + * either because they are activated from elsewhere in the code, or + * because they have a simple timeout. + * + * We use this type to avoid exposing Libevent's API throughout the rest + * of the codebase. + * + * This type can't be used for all events: it doesn't handle events that + * are triggered by signals or by sockets. + */ +struct mainloop_event_t { + struct event *ev; + void (*cb)(mainloop_event_t *, void *); + void *userdata; +}; + +/** + * Internal: Implements mainloop event using a libevent event. + */ +static void +mainloop_event_cb(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + mainloop_event_t *mev = arg; + mev->cb(mev, mev->userdata); +} + +/** + * As mainloop_event_cb, but implements a post-loop event. + */ +static void +mainloop_event_postloop_cb(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + + /* Note that if rescan_mainloop_ev is already activated, + * event_active() will do nothing: only the first post-loop event that + * happens each time through the event loop will cause it to be + * activated. + * + * Because event_active() puts events on a FIFO queue, every event + * that is made active _after_ rescan_mainloop_ev will get its + * callback run after rescan_mainloop_cb is called -- that is, on the + * next iteration of the loop. + */ + event_active(rescan_mainloop_ev, EV_READ, 1); + + mainloop_event_t *mev = arg; + mev->cb(mev, mev->userdata); +} + +/** + * Helper for mainloop_event_new() and mainloop_event_postloop_new(). + */ +static mainloop_event_t * +mainloop_event_new_impl(int postloop, + void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + tor_assert(cb); + + struct event_base *base = tor_libevent_get_base(); + mainloop_event_t *mev = tor_malloc_zero(sizeof(mainloop_event_t)); + mev->ev = tor_event_new(base, -1, 0, + postloop ? mainloop_event_postloop_cb : mainloop_event_cb, + mev); + tor_assert(mev->ev); + mev->cb = cb; + mev->userdata = userdata; + return mev; +} + +/** + * Create and return a new mainloop_event_t to run the function <b>cb</b>. + * + * When run, the callback function will be passed the mainloop_event_t + * and <b>userdata</b> as its arguments. The <b>userdata</b> pointer + * must remain valid for as long as the mainloop_event_t event exists: + * it is your responsibility to free it. + * + * The event is not scheduled by default: Use mainloop_event_activate() + * or mainloop_event_schedule() to make it run. + */ +mainloop_event_t * +mainloop_event_new(void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + return mainloop_event_new_impl(0, cb, userdata); +} + +/** + * As mainloop_event_new(), but create a post-loop event. + * + * A post-loop event behaves like any ordinary event, but any events + * that _it_ activates cannot run until Libevent has checked for other + * events at least once. + */ +mainloop_event_t * +mainloop_event_postloop_new(void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + return mainloop_event_new_impl(1, cb, userdata); +} + +/** + * Schedule <b>event</b> to run in the main loop, immediately. If it is + * not scheduled, it will run anyway. If it is already scheduled to run + * later, it will run now instead. This function will have no effect if + * the event is already scheduled to run. + * + * This function may only be called from the main thread. + */ +void +mainloop_event_activate(mainloop_event_t *event) +{ + tor_assert(event); + event_active(event->ev, EV_READ, 1); +} + +/** Schedule <b>event</b> to run in the main loop, after a delay of <b>tv</b>. + * + * If the event is scheduled for a different time, cancel it and run + * after this delay instead. If the event is currently pending to run + * <em>now</b>, has no effect. + * + * Do not call this function with <b>tv</b> == NULL -- use + * mainloop_event_activate() instead. + * + * This function may only be called from the main thread. + */ +int +mainloop_event_schedule(mainloop_event_t *event, const struct timeval *tv) +{ + tor_assert(event); + if (BUG(tv == NULL)) { + // LCOV_EXCL_START + mainloop_event_activate(event); + return 0; + // LCOV_EXCL_STOP + } + return event_add(event->ev, tv); +} + +/** Cancel <b>event</b> if it is currently active or pending. (Do nothing if + * the event is not currently active or pending.) */ +void +mainloop_event_cancel(mainloop_event_t *event) +{ + if (!event) + return; + (void) event_del(event->ev); +} + +/** Cancel <b>event</b> and release all storage associated with it. */ +void +mainloop_event_free_(mainloop_event_t *event) +{ + if (!event) + return; + tor_event_free(event->ev); + memset(event, 0xb8, sizeof(*event)); + tor_free(event); +} + int tor_init_libevent_rng(void) { @@ -243,56 +485,45 @@ tor_init_libevent_rng(void) void tor_libevent_free_all(void) { + tor_event_free(rescan_mainloop_ev); if (the_event_base) event_base_free(the_event_base); the_event_base = NULL; } -#if defined(LIBEVENT_VERSION_NUMBER) && \ - LIBEVENT_VERSION_NUMBER >= V(2,1,1) && \ - !defined(TOR_UNIT_TESTS) -void -tor_gettimeofday_cached(struct timeval *tv) -{ - event_base_gettimeofday_cached(the_event_base, tv); -} -void -tor_gettimeofday_cache_clear(void) -{ - event_base_update_cache_time(the_event_base); -} -#else /* !(defined(LIBEVENT_VERSION_NUMBER) && ...) */ -/** Cache the current hi-res time; the cache gets reset when libevent - * calls us. */ -static struct timeval cached_time_hires = {0, 0}; - -/** Return a fairly recent view of the current time. */ -void -tor_gettimeofday_cached(struct timeval *tv) +/** + * Run the event loop for the provided event_base, handling events until + * something stops it. If <b>once</b> is set, then just poll-and-run + * once, then exit. Return 0 on success, -1 if an error occurred, or 1 + * if we exited because no events were pending or active. + * + * This isn't reentrant or multithreaded. + */ +int +tor_libevent_run_event_loop(struct event_base *base, int once) { - if (cached_time_hires.tv_sec == 0) { - tor_gettimeofday(&cached_time_hires); - } - *tv = cached_time_hires; + const int flags = once ? EVLOOP_ONCE : 0; + return event_base_loop(base, flags); } -/** Reset the cached view of the current time, so that the next time we try - * to learn it, we will get an up-to-date value. */ +/** Tell the event loop to exit after <b>delay</b>. If <b>delay</b> is NULL, + * instead exit after we're done running the currently active events. */ void -tor_gettimeofday_cache_clear(void) +tor_libevent_exit_loop_after_delay(struct event_base *base, + const struct timeval *delay) { - cached_time_hires.tv_sec = 0; + event_base_loopexit(base, delay); } -#ifdef TOR_UNIT_TESTS -/** For testing: force-update the cached time to a given value. */ +/** Tell the event loop to exit after running whichever callback is currently + * active. */ void -tor_gettimeofday_cache_set(const struct timeval *tv) +tor_libevent_exit_loop_after_callback(struct event_base *base) { - tor_assert(tv); - memcpy(&cached_time_hires, tv, sizeof(*tv)); + event_base_loopbreak(base); } +#if defined(TOR_UNIT_TESTS) /** For testing: called post-fork to make libevent reinitialize * kernel structures. */ void @@ -302,5 +533,4 @@ tor_libevent_postfork(void) tor_assert(r == 0); } #endif /* defined(TOR_UNIT_TESTS) */ -#endif /* defined(LIBEVENT_VERSION_NUMBER) && ... */ diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index 1853e50917..286a268122 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -7,8 +7,6 @@ #include "orconfig.h" #include "testsupport.h" -#include <event2/event.h> - void configure_libevent_logging(void); void suppress_libevent_log_msg(const char *msg); @@ -19,6 +17,9 @@ void suppress_libevent_log_msg(const char *msg); evdns_add_server_port_with_base(tor_libevent_get_base(), \ (sock),(tcp),(cb),(data)); +struct event; +struct event_base; + void tor_event_free_(struct event *ev); #define tor_event_free(ev) \ FREE_AND_NULL(struct event, tor_event_free_, (ev)) @@ -30,11 +31,24 @@ periodic_timer_t *periodic_timer_new(struct event_base *base, void (*cb)(periodic_timer_t *timer, void *data), void *data); void periodic_timer_free_(periodic_timer_t *); +void periodic_timer_launch(periodic_timer_t *, const struct timeval *tv); +void periodic_timer_disable(periodic_timer_t *); #define periodic_timer_free(t) \ FREE_AND_NULL(periodic_timer_t, periodic_timer_free_, (t)) -#define tor_event_base_loopexit event_base_loopexit -#define tor_event_base_loopbreak event_base_loopbreak +typedef struct mainloop_event_t mainloop_event_t; +mainloop_event_t *mainloop_event_new(void (*cb)(mainloop_event_t *, void *), + void *userdata); +mainloop_event_t * mainloop_event_postloop_new( + void (*cb)(mainloop_event_t *, void *), + void *userdata); +void mainloop_event_activate(mainloop_event_t *event); +int mainloop_event_schedule(mainloop_event_t *event, + const struct timeval *delay); +void mainloop_event_cancel(mainloop_event_t *event); +void mainloop_event_free_(mainloop_event_t *event); +#define mainloop_event_free(event) \ + FREE_AND_NULL(mainloop_event_t, mainloop_event_free_, (event)) /** Defines a configuration for using libevent with Tor: passed as an argument * to tor_libevent_initialize() to describe how we want to set up. */ @@ -56,13 +70,15 @@ void tor_libevent_free_all(void); int tor_init_libevent_rng(void); -void tor_gettimeofday_cached(struct timeval *tv); -void tor_gettimeofday_cache_clear(void); #ifdef TOR_UNIT_TESTS -void tor_gettimeofday_cache_set(const struct timeval *tv); void tor_libevent_postfork(void); #endif +int tor_libevent_run_event_loop(struct event_base *base, int once); +void tor_libevent_exit_loop_after_delay(struct event_base *base, + const struct timeval *delay); +void tor_libevent_exit_loop_after_callback(struct event_base *base); + #ifdef COMPAT_LIBEVENT_PRIVATE /** Macro: returns the number of a Libevent version as a 4-byte number, diff --git a/src/common/compat_time.c b/src/common/compat_time.c index 183a60a480..93b527def0 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -71,8 +71,8 @@ tor_sleep_msec(int msec) /** Set *timeval to the current time of day. On error, log and terminate. * (Same as gettimeofday(timeval,NULL), but never returns -1.) */ -void -tor_gettimeofday(struct timeval *timeval) +MOCK_IMPL(void, +tor_gettimeofday, (struct timeval *timeval)) { #ifdef _WIN32 /* Epoch bias copied from perl: number of units between windows epoch and @@ -279,6 +279,8 @@ monotime_reset_ratchets_for_testing(void) * nanoseconds. */ static struct mach_timebase_info mach_time_info; +static struct mach_timebase_info mach_time_info_msec_cvt; +static int32_t mach_time_msec_cvt_threshold; static int monotime_shift = 0; static void @@ -296,6 +298,18 @@ monotime_init_internal(void) // requires that tor_log2(0) == 0. monotime_shift = tor_log2(ms_per_tick); } + { + // For converting ticks to milliseconds in a 32-bit-friendly way, we + // will first right-shift by 20, and then multiply by 2048/1953, since + // (1<<20) * 1953/2048 is about 1e6. We precompute a new numerator and + // denominator here to avoid multiple multiplies. + mach_time_info_msec_cvt.numer = mach_time_info.numer * 2048; + mach_time_info_msec_cvt.denom = mach_time_info.denom * 1953; + // For any value above this amount, we should divide before multiplying, + // to avoid overflow. For a value below this, we should multiply + // before dividing, to improve accuracy. + mach_time_msec_cvt_threshold = INT32_MAX / mach_time_info_msec_cvt.numer; + } } /** @@ -345,6 +359,27 @@ monotime_diff_nsec(const monotime_t *start, return diff_nsec; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + if (BUG(mach_time_info.denom == 0)) { + monotime_init(); + } + const int64_t diff_ticks = end->abstime_ - start->abstime_; + + /* We already require in di_ops.c that right-shift performs a sign-extend. */ + const int32_t diff_microticks = (int32_t)(diff_ticks >> 20); + + if (diff_microticks >= mach_time_msec_cvt_threshold) { + return (diff_microticks / mach_time_info_msec_cvt.denom) * + mach_time_info_msec_cvt.numer; + } else { + return (diff_microticks * mach_time_info_msec_cvt.numer) / + mach_time_info_msec_cvt.denom; + } +} + uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t) { @@ -443,6 +478,15 @@ monotime_diff_nsec(const monotime_t *start, return diff_nsec; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + const int32_t diff_sec = (int32_t)(end->ts_.tv_sec - start->ts_.tv_sec); + const int32_t diff_nsec = (int32_t)(end->ts_.tv_nsec - start->ts_.tv_nsec); + return diff_sec * 1000 + diff_nsec / ONE_MILLION; +} + /* This value is ONE_BILLION >> 20. */ static const uint32_t STAMP_TICKS_PER_SECOND = 953; @@ -592,6 +636,13 @@ monotime_coarse_diff_msec(const monotime_coarse_t *start, return diff_ticks; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + return (int32_t)monotime_coarse_diff_msec(start, end); +} + int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end) @@ -677,6 +728,15 @@ monotime_diff_nsec(const monotime_t *start, return (diff.tv_sec * ONE_BILLION + diff.tv_usec * 1000); } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + struct timeval diff; + timersub(&end->tv_, &start->tv_, &diff); + return diff.tv_sec * 1000 + diff.tv_usec / 1000; +} + /* This value is ONE_MILLION >> 10. */ static const uint32_t STAMP_TICKS_PER_SECOND = 976; @@ -830,11 +890,24 @@ monotime_coarse_stamp_units_to_approx_msec(uint64_t units) return (abstime_diff * mach_time_info.numer) / (mach_time_info.denom * ONE_MILLION); } +uint64_t +monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) +{ + uint64_t abstime_val = + (((uint64_t)msec) * ONE_MILLION * mach_time_info.denom) / + mach_time_info.numer; + return abstime_val >> monotime_shift; +} #else uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { return (units * 1000) / STAMP_TICKS_PER_SECOND; } +uint64_t +monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) +{ + return (msec * STAMP_TICKS_PER_SECOND) / 1000; +} #endif diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 6ddd11883d..f241aa5eba 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -150,6 +150,7 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t); * into an approximate number of milliseconds. */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units); +uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec); uint32_t monotime_coarse_get_stamp(void); #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) @@ -172,7 +173,35 @@ void monotime_coarse_add_msec(monotime_coarse_t *out, #define monotime_coarse_add_msec monotime_add_msec #endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ -void tor_gettimeofday(struct timeval *timeval); +/** + * As monotime_coarse_diff_msec, but avoid 64-bit division. + * + * Requires that the difference fit into an int32_t; not for use with + * large time differences. + */ +int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end); + +/** + * As monotime_coarse_diff_msec, but avoid 64-bit division if it is expensive. + * + * Requires that the difference fit into an int32_t; not for use with + * large time differences. + */ +static inline int32_t +monotime_coarse_diff_msec32(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ +#if SIZEOF_VOID_P == 8 + // on a 64-bit platform, let's assume 64/64 division is cheap. + return (int32_t) monotime_coarse_diff_msec(start, end); +#else +#define USING_32BIT_MSEC_HACK + return monotime_coarse_diff_msec32_(start, end); +#endif +} + +MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval)); #ifdef TOR_UNIT_TESTS void tor_sleep_msec(int msec); diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 5f7ec94c23..7021344f6e 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -18,7 +18,6 @@ #include "util.h" #include "container.h" #include "torlog.h" -#include <process.h> /* This value is more or less total cargo-cult */ #define SPIN_COUNT 2000 diff --git a/src/common/compress.c b/src/common/compress.c index 47c93cf6a9..cb1549f1aa 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -663,3 +663,13 @@ tor_compress_init(void) tor_zstd_init(); } +/** Warn if we had any problems while setting up our compression libraries. + * + * (This isn't part of tor_compress_init, since the logs aren't set up yet.) + */ +void +tor_compress_log_init_warnings(void) +{ + tor_zstd_warn_if_version_mismatched(); +} + diff --git a/src/common/compress.h b/src/common/compress.h index 952102bf97..65d63a4386 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -87,6 +87,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); void tor_compress_init(void); +void tor_compress_log_init_warnings(void); #endif /* !defined(TOR_COMPRESS_H) */ diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index b9f9f1f076..dc8b4d621d 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -18,6 +18,13 @@ #include "compress.h" #include "compress_zstd.h" +#ifdef ENABLE_ZSTD_ADVANCED_APIS +/* This is a lie, but we make sure it doesn't get us in trouble by wrapping + * all invocations of zstd's static-only functions in a check to make sure + * that the compile-time version matches the run-time version. */ +#define ZSTD_STATIC_LINKING_ONLY +#endif + #ifdef HAVE_ZSTD #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE DISABLE_GCC_WARNING(unused-const-variable) @@ -57,21 +64,31 @@ tor_zstd_method_supported(void) #endif } +#ifdef HAVE_ZSTD +/** Format a zstd version number as a string in <b>buf</b>. */ +static void +tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) +{ + tor_snprintf(buf, buflen, + "%u.%u.%u", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); +} +#endif + +#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */ + /** Return a string representation of the version of the currently running * version of libzstd. Returns NULL if Zstandard is unsupported. */ const char * tor_zstd_get_version_str(void) { #ifdef HAVE_ZSTD - static char version_str[16]; - size_t version_number; + static char version_str[VERSION_STR_MAX_LEN]; - version_number = ZSTD_versionNumber(); - tor_snprintf(version_str, sizeof(version_str), - "%d.%d.%d", - (int) version_number / 10000 % 100, - (int) version_number / 100 % 100, - (int) version_number % 100); + tor_zstd_format_version(version_str, sizeof(version_str), + ZSTD_versionNumber()); return version_str; #else /* !(defined(HAVE_ZSTD)) */ @@ -91,6 +108,26 @@ tor_zstd_get_header_version_str(void) #endif } +#ifdef TOR_UNIT_TESTS +static int static_apis_disable_for_testing = 0; +#endif + +/** Return true iff we can use the "static-only" APIs. */ +int +tor_zstd_can_use_static_apis(void) +{ +#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) +#ifdef TOR_UNIT_TESTS + if (static_apis_disable_for_testing) { + return 0; + } +#endif + return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber()); +#else + return 0; +#endif +} + /** Internal Zstandard state for incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_zstd_compress_state_t { @@ -118,9 +155,11 @@ struct tor_zstd_compress_state_t { #ifdef HAVE_ZSTD /** Return an approximate number of bytes stored in memory to hold the - * Zstandard compression/decompression state. */ + * Zstandard compression/decompression state. This is a fake estimate + * based on inspecting the zstd source: tor_zstd_state_size_precalc() is + * more accurate when it's allowed to use "static-only" functions */ static size_t -tor_zstd_state_size_precalc(int compress, int preset) +tor_zstd_state_size_precalc_fake(int compress, int preset) { tor_assert(preset > 0); @@ -177,6 +216,28 @@ tor_zstd_state_size_precalc(int compress, int preset) return memory_usage; } + +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ +#ifdef ZSTD_STATIC_LINKING_ONLY + if (tor_zstd_can_use_static_apis()) { + if (compress) { +#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE + return ZSTD_estimateCStreamSize(preset); +#endif + } else { +#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE + /* Could use DStream, but that takes a windowSize. */ + return ZSTD_estimateDCtxSize(); +#endif + } + } +#endif + return tor_zstd_state_size_precalc_fake(compress, preset); +} #endif /* defined(HAVE_ZSTD) */ /** Construct and return a tor_zstd_compress_state_t object using @@ -446,3 +507,34 @@ tor_zstd_init(void) atomic_counter_init(&total_zstd_allocation); } +/** Warn if the header and library versions don't match. */ +void +tor_zstd_warn_if_version_mismatched(void) +{ +#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) + if (! tor_zstd_can_use_static_apis()) { + char header_version[VERSION_STR_MAX_LEN]; + char runtime_version[VERSION_STR_MAX_LEN]; + tor_zstd_format_version(header_version, sizeof(header_version), + ZSTD_VERSION_NUMBER); + tor_zstd_format_version(runtime_version, sizeof(runtime_version), + ZSTD_versionNumber()); + + log_warn(LD_GENERAL, + "Tor was compiled with zstd %s, but is running with zstd %s. " + "For safety, we'll avoid using advanced zstd functionality.", + header_version, runtime_version); + } +#endif +} + +#ifdef TOR_UNIT_TESTS +/** Testing only: disable usage of static-only APIs, so we can make sure that + * we still work without them. */ +void +tor_zstd_set_static_apis_disabled_for_testing(int disabled) +{ + static_apis_disable_for_testing = disabled; +} +#endif + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index 9bca24ded7..bd42cf65ce 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -17,6 +17,8 @@ const char *tor_zstd_get_version_str(void); const char *tor_zstd_get_header_version_str(void); +int tor_zstd_can_use_static_apis(void); + /** Internal state for an incremental Zstandard compression/decompression. */ typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; @@ -41,6 +43,11 @@ size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); size_t tor_zstd_get_total_allocation(void); void tor_zstd_init(void); +void tor_zstd_warn_if_version_mismatched(void); + +#ifdef TOR_UNIT_TESTS +void tor_zstd_set_static_apis_disabled_for_testing(int disabled); +#endif #endif /* !defined(TOR_COMPRESS_ZSTD_H) */ diff --git a/src/common/container.c b/src/common/container.c index 54b0b2028f..72ad3a9258 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -15,7 +15,7 @@ #include "util.h" #include "torlog.h" #include "container.h" -#include "crypto.h" +#include "crypto_digest.h" #include <stdlib.h> #include <string.h> @@ -628,7 +628,7 @@ smartlist_uniq(smartlist_t *sl, * less than member, and greater than 0 if key is greater then member. */ void * -smartlist_bsearch(smartlist_t *sl, const void *key, +smartlist_bsearch(const smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)) { int found, idx; diff --git a/src/common/container.h b/src/common/container.h index 5d2dce5416..7457c1e918 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -120,7 +120,7 @@ const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl); void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); void smartlist_uniq_digests256(smartlist_t *sl); -void *smartlist_bsearch(smartlist_t *sl, const void *key, +void *smartlist_bsearch(const smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member)); int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, int (*compare)(const void *key, const void **member), diff --git a/src/common/crypto.c b/src/common/crypto.c index d85aca4004..d5b7c96916 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -23,25 +23,26 @@ #endif /* defined(_WIN32) */ #define CRYPTO_PRIVATE -#include "crypto.h" #include "compat_openssl.h" +#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_rand.h" #include "crypto_rsa.h" +#include "crypto_util.h" DISABLE_GCC_WARNING(redundant-decls) #include <openssl/err.h> -#include <openssl/rsa.h> -#include <openssl/pem.h> #include <openssl/evp.h> #include <openssl/engine.h> -#include <openssl/rand.h> #include <openssl/bn.h> #include <openssl/dh.h> #include <openssl/conf.h> #include <openssl/hmac.h> +#include <openssl/ssl.h> ENABLE_GCC_WARNING(redundant-decls) @@ -59,18 +60,6 @@ ENABLE_GCC_WARNING(redundant-decls) #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_FCNTL_H -#include <sys/fcntl.h> -#endif -#ifdef HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#ifdef HAVE_SYS_RANDOM_H -#include <sys/random.h> -#endif #include "torlog.h" #include "torint.h" @@ -83,12 +72,6 @@ ENABLE_GCC_WARNING(redundant-decls) #include "keccak-tiny/keccak-tiny.h" -/** Longest recognized */ -#define MAX_DNS_LABEL_SIZE 63 - -/** Largest strong entropy request */ -#define MAX_STRONGEST_RAND_SIZE 256 - /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake * while we're waiting for the second.*/ struct crypto_dh_t { @@ -161,23 +144,6 @@ try_load_engine(const char *path, const char *engine) } #endif /* !defined(DISABLE_ENGINES) */ -/** Make sure that openssl is using its default PRNG. Return 1 if we had to - * adjust it; 0 otherwise. */ -STATIC int -crypto_force_rand_ssleay(void) -{ - RAND_METHOD *default_method; - default_method = RAND_OpenSSL(); - if (RAND_get_rand_method() != default_method) { - log_notice(LD_CRYPTO, "It appears that one of our engines has provided " - "a replacement the OpenSSL RNG. Resetting it to the default " - "implementation."); - RAND_set_rand_method(default_method); - return 1; - } - return 0; -} - static int have_seeded_siphash = 0; /** Set up the siphash key if we haven't already done so. */ @@ -203,8 +169,15 @@ crypto_early_init(void) crypto_early_initialized_ = 1; +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS | + OPENSSL_INIT_LOAD_CRYPTO_STRINGS | + OPENSSL_INIT_ADD_ALL_CIPHERS | + OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); +#else ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); +#endif setup_openssl_threading(); @@ -397,266 +370,6 @@ crypto_cipher_free_(crypto_cipher_t *env) aes_cipher_free(env); } -/* public key crypto */ - -/** Check a siglen-byte long signature at <b>sig</b> against - * <b>datalen</b> bytes of data at <b>data</b>, using the public key - * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for - * SHA1(data). Else return -1. - */ -MOCK_IMPL(int, -crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, - size_t siglen)) -{ - char digest[DIGEST_LEN]; - char *buf; - size_t buflen; - int r; - - tor_assert(env); - tor_assert(data); - tor_assert(sig); - tor_assert(datalen < SIZE_T_CEILING); - tor_assert(siglen < SIZE_T_CEILING); - - if (crypto_digest(digest,data,datalen)<0) { - log_warn(LD_BUG, "couldn't compute digest"); - return -1; - } - buflen = crypto_pk_keysize(env); - buf = tor_malloc(buflen); - r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen); - if (r != DIGEST_LEN) { - log_warn(LD_CRYPTO, "Invalid signature"); - tor_free(buf); - return -1; - } - if (tor_memneq(buf, digest, DIGEST_LEN)) { - log_warn(LD_CRYPTO, "Signature mismatched with digest."); - tor_free(buf); - return -1; - } - tor_free(buf); - - return 0; -} - -/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at - * <b>from</b>; sign the data with the private key in <b>env</b>, and - * store it in <b>to</b>. Return the number of bytes written on - * success, and -1 on failure. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen) -{ - int r; - char digest[DIGEST_LEN]; - if (crypto_digest(digest,from,fromlen)<0) - return -1; - r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN); - memwipe(digest, 0, sizeof(digest)); - return r; -} - -/** Perform a hybrid (public/secret) encryption on <b>fromlen</b> - * bytes of data from <b>from</b>, with padding type 'padding', - * storing the results on <b>to</b>. - * - * Returns the number of bytes written on success, -1 on failure. - * - * The encrypted data consists of: - * - The source data, padded and encrypted with the public key, if the - * padded source data is no longer than the public key, and <b>force</b> - * is false, OR - * - The beginning of the source data prefixed with a 16-byte symmetric key, - * padded and encrypted with the public key; followed by the rest of - * the source data encrypted in AES-CTR mode with the symmetric key. - * - * NOTE that this format does not authenticate the symmetrically encrypted - * part of the data, and SHOULD NOT BE USED for new protocols. - */ -int -crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, - char *to, size_t tolen, - const char *from, - size_t fromlen, - int padding, int force) -{ - int overhead, outlen, r; - size_t pkeylen, symlen; - crypto_cipher_t *cipher = NULL; - char *buf = NULL; - - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(fromlen < SIZE_T_CEILING); - - overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding)); - pkeylen = crypto_pk_keysize(env); - - if (!force && fromlen+overhead <= pkeylen) { - /* It all fits in a single encrypt. */ - return crypto_pk_public_encrypt(env,to, - tolen, - from,fromlen,padding); - } - tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN); - tor_assert(tolen >= pkeylen); - - char key[CIPHER_KEY_LEN]; - crypto_rand(key, sizeof(key)); /* generate a new key. */ - cipher = crypto_cipher_new(key); - - buf = tor_malloc(pkeylen+1); - memcpy(buf, key, CIPHER_KEY_LEN); - memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN); - - /* Length of symmetrically encrypted data. */ - symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN); - - outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding); - if (outlen!=(int)pkeylen) { - goto err; - } - r = crypto_cipher_encrypt(cipher, to+outlen, - from+pkeylen-overhead-CIPHER_KEY_LEN, symlen); - - if (r<0) goto err; - memwipe(buf, 0, pkeylen); - memwipe(key, 0, sizeof(key)); - tor_free(buf); - crypto_cipher_free(cipher); - tor_assert(outlen+symlen < INT_MAX); - return (int)(outlen + symlen); - err: - - memwipe(buf, 0, pkeylen); - memwipe(key, 0, sizeof(key)); - tor_free(buf); - crypto_cipher_free(cipher); - return -1; -} - -/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of - * bytes written on success, -1 on failure. - * - * NOTE that this format does not authenticate the symmetrically encrypted - * part of the data, and SHOULD NOT BE USED for new protocols. - */ -int -crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, - char *to, - size_t tolen, - const char *from, - size_t fromlen, - int padding, int warnOnFailure) -{ - int outlen, r; - size_t pkeylen; - crypto_cipher_t *cipher = NULL; - char *buf = NULL; - - tor_assert(fromlen < SIZE_T_CEILING); - pkeylen = crypto_pk_keysize(env); - - if (fromlen <= pkeylen) { - return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding, - warnOnFailure); - } - - buf = tor_malloc(pkeylen); - outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding, - warnOnFailure); - if (outlen<0) { - log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO, - "Error decrypting public-key data"); - goto err; - } - if (outlen < CIPHER_KEY_LEN) { - log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO, - "No room for a symmetric key"); - goto err; - } - cipher = crypto_cipher_new(buf); - if (!cipher) { - goto err; - } - memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN); - outlen -= CIPHER_KEY_LEN; - tor_assert(tolen - outlen >= fromlen - pkeylen); - r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen); - if (r<0) - goto err; - memwipe(buf,0,pkeylen); - tor_free(buf); - crypto_cipher_free(cipher); - tor_assert(outlen + fromlen < INT_MAX); - return (int)(outlen + (fromlen-pkeylen)); - err: - memwipe(buf,0,pkeylen); - tor_free(buf); - crypto_cipher_free(cipher); - return -1; -} - -/** Given a private or public key <b>pk</b>, put a SHA1 hash of the - * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space). - * Return 0 on success, -1 on failure. - */ -int -crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out) -{ - char *buf; - size_t buflen; - int len; - int rv = -1; - - buflen = crypto_pk_keysize(pk)*2; - buf = tor_malloc(buflen); - len = crypto_pk_asn1_encode(pk, buf, buflen); - if (len < 0) - goto done; - - if (crypto_digest(digest_out, buf, len) < 0) - goto done; - - rv = 0; - done: - tor_free(buf); - return rv; -} - -/** Compute all digests of the DER encoding of <b>pk</b>, and store them - * in <b>digests_out</b>. Return 0 on success, -1 on failure. */ -int -crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out) -{ - char *buf; - size_t buflen; - int len; - int rv = -1; - - buflen = crypto_pk_keysize(pk)*2; - buf = tor_malloc(buflen); - len = crypto_pk_asn1_encode(pk, buf, buflen); - if (len < 0) - goto done; - - if (crypto_common_digests(digests_out, (char*)buf, len) < 0) - goto done; - - rv = 0; - done: - tor_free(buf); - return rv; -} - /** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces * every four characters. */ void @@ -788,524 +501,6 @@ crypto_cipher_decrypt_with_iv(const char *key, return (int)(fromlen - CIPHER_IV_LEN); } -/* SHA-1 */ - -/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in - * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, -1 on failure. - */ -int -crypto_digest(char *digest, const char *m, size_t len) -{ - tor_assert(m); - tor_assert(digest); - if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) - return -1; - return 0; -} - -/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - - int ret = 0; - if (algorithm == DIGEST_SHA256) - ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); - else - ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - > -1); - - if (!ret) - return -1; - return 0; -} - -/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - - int ret = 0; - if (algorithm == DIGEST_SHA512) - ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - != NULL); - else - ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - > -1); - - if (!ret) - return -1; - return 0; -} - -/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the - * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on - * success, -1 on failure. */ -int -crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len) -{ - tor_assert(ds_out); - memset(ds_out, 0, sizeof(*ds_out)); - if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) - return -1; - if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0) - return -1; - - return 0; -} - -/** Return the name of an algorithm, as used in directory documents. */ -const char * -crypto_digest_algorithm_get_name(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: - return "sha1"; - case DIGEST_SHA256: - return "sha256"; - case DIGEST_SHA512: - return "sha512"; - case DIGEST_SHA3_256: - return "sha3-256"; - case DIGEST_SHA3_512: - return "sha3-512"; - // LCOV_EXCL_START - default: - tor_fragile_assert(); - return "??unknown_digest??"; - // LCOV_EXCL_STOP - } -} - -/** Given the name of a digest algorithm, return its integer value, or -1 if - * the name is not recognized. */ -int -crypto_digest_algorithm_parse_name(const char *name) -{ - if (!strcmp(name, "sha1")) - return DIGEST_SHA1; - else if (!strcmp(name, "sha256")) - return DIGEST_SHA256; - else if (!strcmp(name, "sha512")) - return DIGEST_SHA512; - else if (!strcmp(name, "sha3-256")) - return DIGEST_SHA3_256; - else if (!strcmp(name, "sha3-512")) - return DIGEST_SHA3_512; - else - return -1; -} - -/** Given an algorithm, return the digest length in bytes. */ -size_t -crypto_digest_algorithm_get_length(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: - return DIGEST_LEN; - case DIGEST_SHA256: - return DIGEST256_LEN; - case DIGEST_SHA512: - return DIGEST512_LEN; - case DIGEST_SHA3_256: - return DIGEST256_LEN; - case DIGEST_SHA3_512: - return DIGEST512_LEN; - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; /* Unreachable */ // LCOV_EXCL_LINE - } -} - -/** Intermediate information about the digest of a stream of data. */ -struct crypto_digest_t { - digest_algorithm_t algorithm; /**< Which algorithm is in use? */ - /** State for the digest we're using. Only one member of the - * union is usable, depending on the value of <b>algorithm</b>. Note also - * that space for other members might not even be allocated! - */ - union { - SHA_CTX sha1; /**< state for SHA1 */ - SHA256_CTX sha2; /**< state for SHA256 */ - SHA512_CTX sha512; /**< state for SHA512 */ - keccak_state sha3; /**< state for SHA3-[256,512] */ - } d; -}; - -#ifdef TOR_UNIT_TESTS - -digest_algorithm_t -crypto_digest_get_algorithm(crypto_digest_t *digest) -{ - tor_assert(digest); - - return digest->algorithm; -} - -#endif /* defined(TOR_UNIT_TESTS) */ - -/** - * Return the number of bytes we need to malloc in order to get a - * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe - * when we free one. - */ -static size_t -crypto_digest_alloc_bytes(digest_algorithm_t alg) -{ - /* Helper: returns the number of bytes in the 'f' field of 'st' */ -#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) - /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ - STRUCT_FIELD_SIZE(crypto_digest_t, f)) - switch (alg) { - case DIGEST_SHA1: - return END_OF_FIELD(d.sha1); - case DIGEST_SHA256: - return END_OF_FIELD(d.sha2); - case DIGEST_SHA512: - return END_OF_FIELD(d.sha512); - case DIGEST_SHA3_256: - case DIGEST_SHA3_512: - return END_OF_FIELD(d.sha3); - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; // LCOV_EXCL_LINE - } -#undef END_OF_FIELD -#undef STRUCT_FIELD_SIZE -} - -/** - * Internal function: create and return a new digest object for 'algorithm'. - * Does not typecheck the algorithm. - */ -static crypto_digest_t * -crypto_digest_new_internal(digest_algorithm_t algorithm) -{ - crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - r->algorithm = algorithm; - - switch (algorithm) - { - case DIGEST_SHA1: - SHA1_Init(&r->d.sha1); - break; - case DIGEST_SHA256: - SHA256_Init(&r->d.sha2); - break; - case DIGEST_SHA512: - SHA512_Init(&r->d.sha512); - break; - case DIGEST_SHA3_256: - keccak_digest_init(&r->d.sha3, 256); - break; - case DIGEST_SHA3_512: - keccak_digest_init(&r->d.sha3, 512); - break; - default: - tor_assert_unreached(); - } - - return r; -} - -/** Allocate and return a new digest object to compute SHA1 digests. - */ -crypto_digest_t * -crypto_digest_new(void) -{ - return crypto_digest_new_internal(DIGEST_SHA1); -} - -/** Allocate and return a new digest object to compute 256-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest256_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - return crypto_digest_new_internal(algorithm); -} - -/** Allocate and return a new digest object to compute 512-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest512_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - return crypto_digest_new_internal(algorithm); -} - -/** Deallocate a digest object. - */ -void -crypto_digest_free_(crypto_digest_t *digest) -{ - if (!digest) - return; - size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - memwipe(digest, 0, bytes); - tor_free(digest); -} - -/** Add <b>len</b> bytes from <b>data</b> to the digest object. - */ -void -crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len) -{ - tor_assert(digest); - tor_assert(data); - /* Using the SHA*_*() calls directly means we don't support doing - * SHA in hardware. But so far the delay of getting the question - * to the hardware, and hearing the answer, is likely higher than - * just doing it ourselves. Hashes are fast. - */ - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Update(&digest->d.sha1, (void*)data, len); - break; - case DIGEST_SHA256: - SHA256_Update(&digest->d.sha2, (void*)data, len); - break; - case DIGEST_SHA512: - SHA512_Update(&digest->d.sha512, (void*)data, len); - break; - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); - break; - default: - /* LCOV_EXCL_START */ - tor_fragile_assert(); - break; - /* LCOV_EXCL_STOP */ - } -} - -/** Compute the hash of the data that has been passed to the digest - * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. - */ -void -crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len) -{ - unsigned char r[DIGEST512_LEN]; - crypto_digest_t tmpenv; - tor_assert(digest); - tor_assert(out); - tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - - /* The SHA-3 code handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); - return; - } - - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - /* memcpy into a temporary ctx, since SHA*_Final clears the context */ - memcpy(&tmpenv, digest, alloc_bytes); - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Final(r, &tmpenv.d.sha1); - break; - case DIGEST_SHA256: - SHA256_Final(r, &tmpenv.d.sha2); - break; - case DIGEST_SHA512: - SHA512_Final(r, &tmpenv.d.sha512); - break; -//LCOV_EXCL_START - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - default: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - /* This is fatal, because it should never happen. */ - tor_assert_unreached(); - break; -//LCOV_EXCL_STOP - } - memcpy(out, r, out_len); - memwipe(r, 0, sizeof(r)); -} - -/** Allocate and return a new digest object with the same state as - * <b>digest</b> - */ -crypto_digest_t * -crypto_digest_dup(const crypto_digest_t *digest) -{ - tor_assert(digest); - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - return tor_memdup(digest, alloc_bytes); -} - -/** Replace the state of the digest object <b>into</b> with the state - * of the digest object <b>from</b>. Requires that 'into' and 'from' - * have the same digest type. - */ -void -crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from) -{ - tor_assert(into); - tor_assert(from); - tor_assert(into->algorithm == from->algorithm); - const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); - memcpy(into,from,alloc_bytes); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of those strings, - * plus the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist(char *digest_out, size_t len_out, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of: the - * optional string <b>prepend</b>, those strings, - * and the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>len_out</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_t *d = crypto_digest_new_internal(alg); - if (prepend) - crypto_digest_add_bytes(d, prepend, strlen(prepend)); - SMARTLIST_FOREACH(lst, const char *, cp, - crypto_digest_add_bytes(d, cp, strlen(cp))); - if (append) - crypto_digest_add_bytes(d, append, strlen(append)); - crypto_digest_get_digest(d, digest_out, len_out); - crypto_digest_free(d); -} - -/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using - * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte - * result in <b>hmac_out</b>. Asserts on failure. - */ -void -crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - unsigned char *rv = NULL; - /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - tor_assert(hmac_out); - rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); - tor_assert(rv); -} - -/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a - * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length - * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in - * <b>mac_out</b>. This function can't fail. */ -void -crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, - const uint8_t *key, size_t key_len, - const uint8_t *msg, size_t msg_len) -{ - crypto_digest_t *digest; - - const uint64_t key_len_netorder = tor_htonll(key_len); - - tor_assert(mac_out); - tor_assert(key); - tor_assert(msg); - - digest = crypto_digest256_new(DIGEST_SHA3_256); - - /* Order matters here that is any subsystem using this function should - * expect this very precise ordering in the MAC construction. */ - crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, - sizeof(key_len_netorder)); - crypto_digest_add_bytes(digest, (const char *) key, key_len); - crypto_digest_add_bytes(digest, (const char *) msg, msg_len); - crypto_digest_get_digest(digest, (char *) mac_out, len_out); - crypto_digest_free(digest); -} - -/** Internal state for a eXtendable-Output Function (XOF). */ -struct crypto_xof_t { - keccak_state s; -}; - -/** Allocate a new XOF object backed by SHAKE-256. The security level - * provided is a function of the length of the output used. Read and - * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output - * Functions" before using this construct. - */ -crypto_xof_t * -crypto_xof_new(void) -{ - crypto_xof_t *xof; - xof = tor_malloc(sizeof(crypto_xof_t)); - keccak_xof_init(&xof->s, 256); - return xof; -} - -/** Absorb bytes into a XOF object. Must not be called after a call to - * crypto_xof_squeeze_bytes() for the same instance, and will assert - * if attempted. - */ -void -crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) -{ - int i = keccak_xof_absorb(&xof->s, data, len); - tor_assert(i == 0); -} - -/** Squeeze bytes out of a XOF object. Calling this routine will render - * the XOF instance ineligible to absorb further data. - */ -void -crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) -{ - int i = keccak_xof_squeeze(&xof->s, out, len); - tor_assert(i == 0); -} - -/** Cleanse and deallocate a XOF object. */ -void -crypto_xof_free_(crypto_xof_t *xof) -{ - if (!xof) - return; - memwipe(xof, 0, sizeof(crypto_xof_t)); - tor_free(xof); -} - /* DH */ /** Our DH 'g' parameter */ @@ -1860,576 +1055,6 @@ crypto_dh_free_(crypto_dh_t *dh) tor_free(dh); } -/* random numbers */ - -/** How many bytes of entropy we add at once. - * - * This is how much entropy OpenSSL likes to add right now, so maybe it will - * work for us too. */ -#define ADD_ENTROPY 32 - -/** Set the seed of the weak RNG to a random value. */ -void -crypto_seed_weak_rng(tor_weak_rng_t *rng) -{ - unsigned seed; - crypto_rand((void*)&seed, sizeof(seed)); - tor_init_weak_random(rng, seed); -} - -#ifdef TOR_UNIT_TESTS -int break_strongest_rng_syscall = 0; -int break_strongest_rng_fallback = 0; -#endif - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on - * failure. A maximum request size of 256 bytes is imposed. - */ -static int -crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) -{ - tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); - - /* We only log at notice-level here because in the case that this function - * fails the crypto_strongest_rand_raw() caller will log with a warning-level - * message and let crypto_strongest_rand() error out and finally terminating - * Tor with an assertion error. - */ - -#ifdef TOR_UNIT_TESTS - if (break_strongest_rng_syscall) - return -1; -#endif - -#if defined(_WIN32) - static int provider_set = 0; - static HCRYPTPROV provider; - - if (!provider_set) { - if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT)) { - log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1]."); - return -1; - } - provider_set = 1; - } - if (!CryptGenRandom(provider, out_len, out)) { - log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI."); - return -1; - } - - return 0; -#elif defined(__linux__) && defined(SYS_getrandom) - static int getrandom_works = 1; /* Be optimistic about our chances... */ - - /* getrandom() isn't as straightforward as getentropy(), and has - * no glibc wrapper. - * - * As far as I can tell from getrandom(2) and the source code, the - * requests we issue will always succeed (though it will block on the - * call if /dev/urandom isn't seeded yet), since we are NOT specifying - * GRND_NONBLOCK and the request is <= 256 bytes. - * - * The manpage is unclear on what happens if a signal interrupts the call - * while the request is blocked due to lack of entropy.... - * - * We optimistically assume that getrandom() is available and functional - * because it is the way of the future, and 2 branch mispredicts pale in - * comparison to the overheads involved with failing to open - * /dev/srandom followed by opening and reading from /dev/urandom. - */ - if (PREDICT_LIKELY(getrandom_works)) { - long ret; - /* A flag of '0' here means to read from '/dev/urandom', and to - * block if insufficient entropy is available to service the - * request. - */ - const unsigned int flags = 0; - do { - ret = syscall(SYS_getrandom, out, out_len, flags); - } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); - - if (PREDICT_UNLIKELY(ret == -1)) { - /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ - tor_assert(errno != EAGAIN); - tor_assert(errno != EINTR); - - /* Useful log message for errno. */ - if (errno == ENOSYS) { - log_notice(LD_CRYPTO, "Can't get entropy from getrandom()." - " You are running a version of Tor built to support" - " getrandom(), but the kernel doesn't implement this" - " function--probably because it is too old?" - " Trying fallback method instead."); - } else { - log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s." - " Trying fallback method instead.", - strerror(errno)); - } - - getrandom_works = 0; /* Don't bother trying again. */ - return -1; - /* LCOV_EXCL_STOP */ - } - - tor_assert(ret == (long)out_len); - return 0; - } - - return -1; /* getrandom() previously failed unexpectedly. */ -#elif defined(HAVE_GETENTROPY) - /* getentropy() is what Linux's getrandom() wants to be when it grows up. - * the only gotcha is that requests are limited to 256 bytes. - */ - return getentropy(out, out_len); -#else - (void) out; -#endif /* defined(_WIN32) || ... */ - - /* This platform doesn't have a supported syscall based random. */ - return -1; -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * via the per-platform fallback mechanism, storing it into <b>out</b>. - * Return 0 on success, -1 on failure. A maximum request size of 256 bytes - * is imposed. - */ -static int -crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) -{ -#ifdef TOR_UNIT_TESTS - if (break_strongest_rng_fallback) - return -1; -#endif - -#ifdef _WIN32 - /* Windows exclusively uses crypto_strongest_rand_syscall(). */ - (void)out; - (void)out_len; - return -1; -#else /* !(defined(_WIN32)) */ - static const char *filenames[] = { - "/dev/srandom", "/dev/urandom", "/dev/random", NULL - }; - int fd, i; - size_t n; - - for (i = 0; filenames[i]; ++i) { - log_debug(LD_FS, "Considering %s as entropy source", filenames[i]); - fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); - if (fd<0) continue; - log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); - n = read_all(fd, (char*)out, out_len, 0); - close(fd); - if (n != out_len) { - /* LCOV_EXCL_START - * We can't make /dev/foorandom actually fail. */ - log_notice(LD_CRYPTO, - "Error reading from entropy source %s (read only %lu bytes).", - filenames[i], - (unsigned long)n); - return -1; - /* LCOV_EXCL_STOP */ - } - - return 0; - } - - return -1; -#endif /* defined(_WIN32) */ -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum - * request size of 256 bytes is imposed. - */ -STATIC int -crypto_strongest_rand_raw(uint8_t *out, size_t out_len) -{ - static const size_t sanity_min_size = 16; - static const int max_attempts = 3; - tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); - - /* For buffers >= 16 bytes (128 bits), we sanity check the output by - * zero filling the buffer and ensuring that it actually was at least - * partially modified. - * - * Checking that any individual byte is non-zero seems like it would - * fail too often (p = out_len * 1/256) for comfort, but this is an - * "adjust according to taste" sort of check. - */ - memwipe(out, 0, out_len); - for (int i = 0; i < max_attempts; i++) { - /* Try to use the syscall/OS favored mechanism to get strong entropy. */ - if (crypto_strongest_rand_syscall(out, out_len) != 0) { - /* Try to use the less-favored mechanism to get strong entropy. */ - if (crypto_strongest_rand_fallback(out, out_len) != 0) { - /* Welp, we tried. Hopefully the calling code terminates the process - * since we're basically boned without good entropy. - */ - log_warn(LD_CRYPTO, - "Cannot get strong entropy: no entropy source found."); - return -1; - } - } - - if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) - return 0; - } - - /* LCOV_EXCL_START - * - * We tried max_attempts times to fill a buffer >= 128 bits long, - * and each time it returned all '0's. Either the system entropy - * source is busted, or the user should go out and buy a ticket to - * every lottery on the planet. - */ - log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); - - return -1; - /* LCOV_EXCL_STOP */ -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * storing it into <b>out</b>. - */ -void -crypto_strongest_rand(uint8_t *out, size_t out_len) -{ -#define DLEN SHA512_DIGEST_LENGTH - /* We're going to hash DLEN bytes from the system RNG together with some - * bytes from the openssl PRNG, in order to yield DLEN bytes. - */ - uint8_t inp[DLEN*2]; - uint8_t tmp[DLEN]; - tor_assert(out); - while (out_len) { - crypto_rand((char*) inp, DLEN); - if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { - // LCOV_EXCL_START - log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " - "important key. Exiting."); - /* Die with an assertion so we get a stack trace. */ - tor_assert(0); - // LCOV_EXCL_STOP - } - if (out_len >= DLEN) { - SHA512(inp, sizeof(inp), out); - out += DLEN; - out_len -= DLEN; - } else { - SHA512(inp, sizeof(inp), tmp); - memcpy(out, tmp, out_len); - break; - } - } - memwipe(tmp, 0, sizeof(tmp)); - memwipe(inp, 0, sizeof(inp)); -#undef DLEN -} - -/** Seed OpenSSL's random number generator with bytes from the operating - * system. Return 0 on success, -1 on failure. - */ -int -crypto_seed_rng(void) -{ - int rand_poll_ok = 0, load_entropy_ok = 0; - uint8_t buf[ADD_ENTROPY]; - - /* OpenSSL has a RAND_poll function that knows about more kinds of - * entropy than we do. We'll try calling that, *and* calling our own entropy - * functions. If one succeeds, we'll accept the RNG as seeded. */ - rand_poll_ok = RAND_poll(); - if (rand_poll_ok == 0) - log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE - - load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); - if (load_entropy_ok) { - RAND_seed(buf, sizeof(buf)); - } - - memwipe(buf, 0, sizeof(buf)); - - if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1) - return 0; - else - return -1; -} - -/** Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking - * for unit tests. - * - * This function is not allowed to fail; if it would fail to generate strong - * entropy, it must terminate the process instead. - */ -MOCK_IMPL(void, -crypto_rand, (char *to, size_t n)) -{ - crypto_rand_unmocked(to, n); -} - -/** Write <b>n</b> bytes of strong random data to <b>to</b>. Most callers - * will want crypto_rand instead. - * - * This function is not allowed to fail; if it would fail to generate strong - * entropy, it must terminate the process instead. - */ -void -crypto_rand_unmocked(char *to, size_t n) -{ - int r; - if (n == 0) - return; - - tor_assert(n < INT_MAX); - tor_assert(to); - r = RAND_bytes((unsigned char*)to, (int)n); - /* We consider a PRNG failure non-survivable. Let's assert so that we get a - * stack trace about where it happened. - */ - tor_assert(r >= 0); -} - -/** Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - */ -int -crypto_rand_int_range(unsigned int min, unsigned int max) -{ - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** As crypto_rand_int_range, but supports uint64_t. */ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) -{ - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** As crypto_rand_int_range, but supports time_t. */ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. */ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - */ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; -} - -/** Generate and return a new random hostname starting with <b>prefix</b>, - * ending with <b>suffix</b>, and containing no fewer than - * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 - * characters. Does not check for failure. - * - * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE. - **/ -char * -crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, - const char *suffix) -{ - char *result, *rand_bytes; - int randlen, rand_bytes_len; - size_t resultlen, prefixlen; - - if (max_rand_len > MAX_DNS_LABEL_SIZE) - max_rand_len = MAX_DNS_LABEL_SIZE; - if (min_rand_len > max_rand_len) - min_rand_len = max_rand_len; - - randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1); - - prefixlen = strlen(prefix); - resultlen = prefixlen + strlen(suffix) + randlen + 16; - - rand_bytes_len = ((randlen*5)+7)/8; - if (rand_bytes_len % 5) - rand_bytes_len += 5 - (rand_bytes_len%5); - rand_bytes = tor_malloc(rand_bytes_len); - crypto_rand(rand_bytes, rand_bytes_len); - - result = tor_malloc(resultlen); - memcpy(result, prefix, prefixlen); - base32_encode(result+prefixlen, resultlen-prefixlen, - rand_bytes, rand_bytes_len); - tor_free(rand_bytes); - strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen)); - - return result; -} - -/** Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b> - * is empty. */ -void * -smartlist_choose(const smartlist_t *sl) -{ - int len = smartlist_len(sl); - if (len) - return smartlist_get(sl,crypto_rand_int(len)); - return NULL; /* no elements to choose from */ -} - -/** Scramble the elements of <b>sl</b> into a random order. */ -void -smartlist_shuffle(smartlist_t *sl) -{ - int i; - /* From the end of the list to the front, choose at random from the - positions we haven't looked at yet, and swap that position into the - current position. Remember to give "no swap" the same probability as - any other swap. */ - for (i = smartlist_len(sl)-1; i > 0; --i) { - int j = crypto_rand_int(i+1); - smartlist_swap(sl, i, j); - } -} - -/** - * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to - * the value <b>byte</b>. - * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens. - * - * This function is preferable to memset, since many compilers will happily - * optimize out memset() when they can convince themselves that the data being - * cleared will never be read. - * - * Right now, our convention is to use this function when we are wiping data - * that's about to become inaccessible, such as stack buffers that are about - * to go out of scope or structures that are about to get freed. (In - * practice, it appears that the compilers we're currently using will optimize - * out the memset()s for stack-allocated buffers, but not those for - * about-to-be-freed structures. That could change, though, so we're being - * wary.) If there are live reads for the data, then you can just use - * memset(). - */ -void -memwipe(void *mem, uint8_t byte, size_t sz) -{ - if (sz == 0) { - return; - } - /* If sz is nonzero, then mem must not be NULL. */ - tor_assert(mem != NULL); - - /* Data this large is likely to be an underflow. */ - tor_assert(sz < SIZE_T_CEILING); - - /* Because whole-program-optimization exists, we may not be able to just - * have this function call "memset". A smart compiler could inline it, then - * eliminate dead memsets, and declare itself to be clever. */ - -#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) - /* Here's what you do on windows. */ - SecureZeroMemory(mem,sz); -#elif defined(HAVE_RTLSECUREZEROMEMORY) - RtlSecureZeroMemory(mem,sz); -#elif defined(HAVE_EXPLICIT_BZERO) - /* The BSDs provide this. */ - explicit_bzero(mem, sz); -#elif defined(HAVE_MEMSET_S) - /* This is in the C99 standard. */ - memset_s(mem, sz, 0, sz); -#else - /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk - * based on the pointer value, then uses that junk to update a global - * variable. It's an elaborate ruse to trick the compiler into not - * optimizing out the "wipe this memory" code. Read it if you like zany - * programming tricks! In later versions of Tor, we should look for better - * not-optimized-out memory wiping stuff... - * - * ...or maybe not. In practice, there are pure-asm implementations of - * OPENSSL_cleanse() on most platforms, which ought to do the job. - **/ - - OPENSSL_cleanse(mem, sz); -#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */ - - /* Just in case some caller of memwipe() is relying on getting a buffer - * filled with a particular value, fill the buffer. - * - * If this function gets inlined, this memset might get eliminated, but - * that's okay: We only care about this particular memset in the case where - * the caller should have been using memset(), and the memset() wouldn't get - * eliminated. In other words, this is here so that we won't break anything - * if somebody accidentally calls memwipe() instead of memset(). - **/ - memset(mem, byte, sz); -} - /** @{ */ /** Uninitialize the crypto library. Return 0 on success. Does not detect * failure. @@ -2437,11 +1062,15 @@ memwipe(void *mem, uint8_t byte, size_t sz) int crypto_global_cleanup(void) { +#ifndef OPENSSL_1_1_API EVP_cleanup(); +#endif #ifndef NEW_THREAD_API ERR_remove_thread_state(NULL); #endif +#ifndef OPENSSL_1_1_API ERR_free_strings(); +#endif if (dh_param_p) BN_clear_free(dh_param_p); @@ -2453,11 +1082,15 @@ crypto_global_cleanup(void) dh_param_p = dh_param_p_tls = dh_param_g = NULL; #ifndef DISABLE_ENGINES +#ifndef OPENSSL_1_1_API ENGINE_cleanup(); #endif +#endif CONF_modules_unload(1); +#ifndef OPENSSL_1_1_API CRYPTO_cleanup_all_ex_data(); +#endif crypto_openssl_free_all(); diff --git a/src/common/crypto.h b/src/common/crypto.h index a9c8837b9e..c773557310 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -17,20 +17,10 @@ #include <stdio.h> #include "torint.h" -#include "testsupport.h" #include "compat.h" #include "util.h" #include "crypto_rsa.h" -#include "keccak-tiny/keccak-tiny.h" - -/** Length of the output of our message digest. */ -#define DIGEST_LEN 20 -/** Length of the output of our second (improved) message digests. (For now - * this is just sha256, but it could be any other 256-bit digest.) */ -#define DIGEST256_LEN 32 -/** Length of the output of our 64-bit optimized message digests (SHA512). */ -#define DIGEST512_LEN 64 /** Length of our symmetric cipher's keys of 128-bit. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV of 128-bit. */ @@ -40,57 +30,15 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a sha1 message digest when encoded in base32 with trailing = - * signs removed. */ -#define BASE32_DIGEST_LEN 32 -/** Length of a sha1 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST_LEN 27 -/** Length of a sha256 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST256_LEN 43 -/** Length of a sha512 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST512_LEN 86 - /** Length of encoded public key fingerprints, including space; but not * including terminating NUL. */ #define FINGERPRINT_LEN 49 -/** Length of hex encoding of SHA1 digest, not including final NUL. */ -#define HEX_DIGEST_LEN 40 -/** Length of hex encoding of SHA256 digest, not including final NUL. */ -#define HEX_DIGEST256_LEN 64 -/** Length of hex encoding of SHA512 digest, not including final NUL. */ -#define HEX_DIGEST512_LEN 128 - -typedef enum { - DIGEST_SHA1 = 0, - DIGEST_SHA256 = 1, - DIGEST_SHA512 = 2, - DIGEST_SHA3_256 = 3, - DIGEST_SHA3_512 = 4, -} digest_algorithm_t; -#define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) -#define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) - -/** A set of all the digests we commonly compute, taken on a single - * string. Any digests that are shorter than 512 bits are right-padded - * with 0 bits. - * - * Note that this representation wastes 44 bytes for the SHA1 case, so - * don't use it for anything where we need to allocate a whole bunch at - * once. - **/ -typedef struct { - char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; -} common_digests_t; typedef struct aes_cnt_cipher crypto_cipher_t; -typedef struct crypto_digest_t crypto_digest_t; -typedef struct crypto_xof_t crypto_xof_t; typedef struct crypto_dh_t crypto_dh_t; /* global state */ +int crypto_init_siphash_key(void); int crypto_early_init(void) ATTR_WUR; int crypto_global_init(int hardwareAccel, const char *accelName, @@ -114,24 +62,6 @@ void crypto_cipher_free_(crypto_cipher_t *env); #define crypto_cipher_free(c) \ FREE_AND_NULL(crypto_cipher_t, crypto_cipher_free_, (c)) -/* public key crypto */ -MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, - const char *data, size_t datalen, - const char *sig, size_t siglen)); -int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int force); -int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int warnOnFailure); -int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out); -int crypto_pk_get_common_digests(crypto_pk_t *pk, - common_digests_t *digests_out); - /* symmetric crypto */ const char *crypto_cipher_get_key(crypto_cipher_t *env); @@ -148,52 +78,6 @@ int crypto_cipher_decrypt_with_iv(const char *key, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 and other digests. */ -int crypto_digest(char *digest, const char *m, size_t len); -int crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm); -int crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm); -int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len); -struct smartlist_t; -void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const struct smartlist_t *lst, - const char *append, - digest_algorithm_t alg); -void crypto_digest_smartlist(char *digest_out, size_t len_out, - const struct smartlist_t *lst, const char *append, - digest_algorithm_t alg); -const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); -size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); -int crypto_digest_algorithm_parse_name(const char *name); -crypto_digest_t *crypto_digest_new(void); -crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); -crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); -void crypto_digest_free_(crypto_digest_t *digest); -#define crypto_digest_free(d) \ - FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d)) -void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len); -void crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len); -crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); -void crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from); -void crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len); -void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, - const uint8_t *key, size_t key_len, - const uint8_t *msg, size_t msg_len); - -crypto_xof_t *crypto_xof_new(void); -void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); -void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); -void crypto_xof_free_(crypto_xof_t *xof); -#define crypto_xof_free(xof) \ - FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) - /* Key negotiation */ #define DH_TYPE_CIRCUIT 1 #define DH_TYPE_REND 2 @@ -219,31 +103,6 @@ int crypto_expand_key_material_rfc5869_sha256( const uint8_t *info_in, size_t info_in_len, uint8_t *key_out, size_t key_out_len); -/* random numbers */ -int crypto_seed_rng(void) ATTR_WUR; -MOCK_DECL(void,crypto_rand,(char *to, size_t n)); -void crypto_rand_unmocked(char *to, size_t n); -void crypto_strongest_rand(uint8_t *out, size_t out_len); -int crypto_rand_int(unsigned int max); -int crypto_rand_int_range(unsigned int min, unsigned int max); -uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); -time_t crypto_rand_time_range(time_t min, time_t max); -uint64_t crypto_rand_uint64(uint64_t max); -double crypto_rand_double(void); -struct tor_weak_rng_t; -void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); -int crypto_init_siphash_key(void); - -char *crypto_random_hostname(int min_rand_len, int max_rand_len, - const char *prefix, const char *suffix); - -struct smartlist_t; -void *smartlist_choose(const struct smartlist_t *sl); -void smartlist_shuffle(struct smartlist_t *sl); - -/** OpenSSL-based utility functions. */ -void memwipe(void *mem, uint8_t byte, size_t sz); - /* Prototypes for private functions only used by tortls.c, crypto.c, and the * unit tests. */ struct dh_st; @@ -251,20 +110,5 @@ struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh); void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); -#ifdef CRYPTO_PRIVATE - -STATIC int crypto_force_rand_ssleay(void); -STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); - -#ifdef TOR_UNIT_TESTS -extern int break_strongest_rng_syscall; -extern int break_strongest_rng_fallback; -#endif -#endif /* defined(CRYPTO_PRIVATE) */ - -#ifdef TOR_UNIT_TESTS -digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); -#endif - #endif /* !defined(TOR_CRYPTO_H) */ diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 8793fa6274..996d94c6e2 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -21,9 +21,11 @@ #include <sys/stat.h> #endif #include "container.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "util.h" #include "torlog.h" diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 11f7423b07..4834fa0836 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -6,6 +6,7 @@ #include "testsupport.h" #include "torint.h" +#include "crypto_digest.h" #include "crypto_openssl_mgt.h" /** Length of a curve25519 public key when encoded. */ diff --git a/src/common/crypto_digest.c b/src/common/crypto_digest.c new file mode 100644 index 0000000000..9f9a1a1e2c --- /dev/null +++ b/src/common/crypto_digest.c @@ -0,0 +1,583 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest.c + * \brief Block of functions related with digest and xof utilities and + * operations. + **/ + +#include "container.h" +#include "crypto_digest.h" +#include "crypto_openssl_mgt.h" +#include "crypto_util.h" +#include "torlog.h" + +#include "keccak-tiny/keccak-tiny.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/hmac.h> +#include <openssl/sha.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +int +crypto_digest(char *digest, const char *m, size_t len) +{ + tor_assert(m); + tor_assert(digest); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) + return -1; + return 0; +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); + else + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); + else + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + + if (!ret) + return -1; + return 0; +} + +/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len) +{ + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0) + return -1; + + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + case DIGEST_SHA512: + return "sha512"; + case DIGEST_SHA3_256: + return "sha3-256"; + case DIGEST_SHA3_512: + return "sha3-512"; + // LCOV_EXCL_START + default: + tor_fragile_assert(); + return "??unknown_digest??"; + // LCOV_EXCL_STOP + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else if (!strcmp(name, "sha512")) + return DIGEST_SHA512; + else if (!strcmp(name, "sha3-256")) + return DIGEST_SHA3_256; + else if (!strcmp(name, "sha3-512")) + return DIGEST_SHA3_512; + else + return -1; +} + +/** Given an algorithm, return the digest length in bytes. */ +size_t +crypto_digest_algorithm_get_length(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return DIGEST_LEN; + case DIGEST_SHA256: + return DIGEST256_LEN; + case DIGEST_SHA512: + return DIGEST512_LEN; + case DIGEST_SHA3_256: + return DIGEST256_LEN; + case DIGEST_SHA3_512: + return DIGEST512_LEN; + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; /* Unreachable */ // LCOV_EXCL_LINE + } +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + SHA512_CTX sha512; /**< state for SHA512 */ + keccak_state sha3; /**< state for SHA3-[256,512] */ + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: + return END_OF_FIELD(d.sha1); + case DIGEST_SHA256: + return END_OF_FIELD(d.sha2); + case DIGEST_SHA512: + return END_OF_FIELD(d.sha512); + case DIGEST_SHA3_256: + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + case DIGEST_SHA512: + SHA512_Update(&digest->d.sha512, (void*)data, len); + break; + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + crypto_digest_t tmpenv; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; + } + + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, alloc_bytes); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + SHA256_Final(r, &tmpenv.d.sha2); + break; + case DIGEST_SHA512: + SHA512_Final(r, &tmpenv.d.sha512); + break; +//LCOV_EXCL_START + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + default: + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); + break; +//LCOV_EXCL_STOP + } + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + return tor_memdup(digest, alloc_bytes); +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + unsigned char *rv = NULL; + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, + (unsigned char*)hmac_out, NULL); + tor_assert(rv); +} + +/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a + * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length + * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in + * <b>mac_out</b>. This function can't fail. */ +void +crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len) +{ + crypto_digest_t *digest; + + const uint64_t key_len_netorder = tor_htonll(key_len); + + tor_assert(mac_out); + tor_assert(key); + tor_assert(msg); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + + /* Order matters here that is any subsystem using this function should + * expect this very precise ordering in the MAC construction. */ + crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, + sizeof(key_len_netorder)); + crypto_digest_add_bytes(digest, (const char *) key, key_len); + crypto_digest_add_bytes(digest, (const char *) msg, msg_len); + crypto_digest_get_digest(digest, (char *) mac_out, len_out); + crypto_digest_free(digest); +} + +/* xof functions */ + +/** Internal state for a eXtendable-Output Function (XOF). */ +struct crypto_xof_t { + keccak_state s; +}; + +/** Allocate a new XOF object backed by SHAKE-256. The security level + * provided is a function of the length of the output used. Read and + * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output + * Functions" before using this construct. + */ +crypto_xof_t * +crypto_xof_new(void) +{ + crypto_xof_t *xof; + xof = tor_malloc(sizeof(crypto_xof_t)); + keccak_xof_init(&xof->s, 256); + return xof; +} + +/** Absorb bytes into a XOF object. Must not be called after a call to + * crypto_xof_squeeze_bytes() for the same instance, and will assert + * if attempted. + */ +void +crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) +{ + int i = keccak_xof_absorb(&xof->s, data, len); + tor_assert(i == 0); +} + +/** Squeeze bytes out of a XOF object. Calling this routine will render + * the XOF instance ineligible to absorb further data. + */ +void +crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) +{ + int i = keccak_xof_squeeze(&xof->s, out, len); + tor_assert(i == 0); +} + +/** Cleanse and deallocate a XOF object. */ +void +crypto_xof_free_(crypto_xof_t *xof) +{ + if (!xof) + return; + memwipe(xof, 0, sizeof(crypto_xof_t)); + tor_free(xof); +} + diff --git a/src/common/crypto_digest.h b/src/common/crypto_digest.h new file mode 100644 index 0000000000..3bd74acdfa --- /dev/null +++ b/src/common/crypto_digest.h @@ -0,0 +1,136 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest.h + * + * \brief Headers for crypto_digest.c + **/ + +#ifndef TOR_CRYPTO_DIGEST_H +#define TOR_CRYPTO_DIGEST_H + +#include <stdio.h> + +#include "container.h" +#include "torint.h" + +/** Length of the output of our message digest. */ +#define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but it could be any other 256-bit digest.) */ +#define DIGEST256_LEN 32 +/** Length of the output of our 64-bit optimized message digests (SHA512). */ +#define DIGEST512_LEN 64 + +/** Length of a sha1 message digest when encoded in base32 with trailing = + * signs removed. */ +#define BASE32_DIGEST_LEN 32 +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 +/** Length of a sha512 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST512_LEN 86 + +/** Length of hex encoding of SHA1 digest, not including final NUL. */ +#define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 +/** Length of hex encoding of SHA512 digest, not including final NUL. */ +#define HEX_DIGEST512_LEN 128 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, + DIGEST_SHA512 = 2, + DIGEST_SHA3_256 = 3, + DIGEST_SHA3_512 = 4, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) +#define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +#define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512) +/** Structure used to temporarily save the a digest object. Only implemented + * for SHA1 digest for now. */ +typedef struct crypto_digest_checkpoint_t { + uint8_t mem[DIGEST_CHECKPOINT_BYTES]; +} crypto_digest_checkpoint_t; + +/** A set of all the digests we commonly compute, taken on a single + * string. Any digests that are shorter than 512 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 44 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} common_digests_t; + +typedef struct crypto_digest_t crypto_digest_t; +typedef struct crypto_xof_t crypto_xof_t; + +/* SHA-1 and other digests */ +int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len); +void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const struct smartlist_t *lst, + const char *append, + digest_algorithm_t alg); +void crypto_digest_smartlist(char *digest_out, size_t len_out, + const struct smartlist_t *lst, const char *append, + digest_algorithm_t alg); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); +crypto_digest_t *crypto_digest_new(void); +crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); +crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); +void crypto_digest_free_(crypto_digest_t *digest); +#define crypto_digest_free(d) \ + FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d)) +void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len); +void crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len); +crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); +void crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest); +void crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint); +void crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from); +void crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len); +void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len); + +/* xof functions*/ +crypto_xof_t *crypto_xof_new(void); +void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); +void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); +void crypto_xof_free_(crypto_xof_t *xof); +#define crypto_xof_free(xof) \ + FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) + +#ifdef TOR_UNIT_TESTS +digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); +#endif + +#endif /* !defined(TOR_CRYPTO_DIGEST_H) */ + diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index b962a59de1..9c13e3bdf0 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -21,11 +21,12 @@ #include <sys/stat.h> #endif -#include "crypto.h" - #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "torlog.h" #include "util.h" #include "util_format.h" diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index 1d090a8770..460e85bac1 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -15,10 +15,11 @@ #include <sys/stat.h> #endif #include "container.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_util.h" #include "util.h" #include "util_format.h" #include "torlog.h" diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 12acc9331c..c2bd1d26cb 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -9,8 +9,11 @@ */ #include "crypto.h" -#include "crypto_s2k.h" +#include "crypto_digest.h" #include "crypto_pwbox.h" +#include "crypto_rand.h" +#include "crypto_s2k.h" +#include "crypto_util.h" #include "di_ops.h" #include "util.h" #include "pwbox.h" diff --git a/src/common/crypto_rand.c b/src/common/crypto_rand.c new file mode 100644 index 0000000000..df2e2f65d3 --- /dev/null +++ b/src/common/crypto_rand.c @@ -0,0 +1,615 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand.c + * + * \brief Functions for initialising and seeding (pseudo-)random + * number generators, and working with randomness. + **/ + +#ifndef CRYPTO_RAND_PRIVATE +#define CRYPTO_RAND_PRIVATE + +#include "crypto_rand.h" + +#ifdef _WIN32 +#include <windows.h> +#include <wincrypt.h> +#endif /* defined(_WIN32) */ + +#include "container.h" +#include "compat.h" +#include "compat_openssl.h" +#include "crypto_util.h" +#include "sandbox.h" +#include "testsupport.h" +#include "torlog.h" +#include "util.h" +#include "util_format.h" + +DISABLE_GCC_WARNING(redundant-decls) +#include <openssl/rand.h> +ENABLE_GCC_WARNING(redundant-decls) + +#if __GNUC__ && GCC_VERSION >= 402 +#if GCC_VERSION >= 406 +#pragma GCC diagnostic pop +#else +#pragma GCC diagnostic warning "-Wredundant-decls" +#endif +#endif /* __GNUC__ && GCC_VERSION >= 402 */ + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_SYS_FCNTL_H +#include <sys/fcntl.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_SYS_SYSCALL_H +#include <sys/syscall.h> +#endif +#ifdef HAVE_SYS_RANDOM_H +#include <sys/random.h> +#endif + +/** + * How many bytes of entropy we add at once. + * + * This is how much entropy OpenSSL likes to add right now, so maybe it will + * work for us too. + **/ +#define ADD_ENTROPY 32 + +/** + * Longest recognized DNS query. + **/ +#define MAX_DNS_LABEL_SIZE 63 + +/** + * Largest strong entropy request permitted. + **/ +#define MAX_STRONGEST_RAND_SIZE 256 + +/** + * Set the seed of the weak RNG to a random value. + **/ +void +crypto_seed_weak_rng(tor_weak_rng_t *rng) +{ + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(rng, seed); +} + +#ifdef TOR_UNIT_TESTS +int break_strongest_rng_syscall = 0; +int break_strongest_rng_fallback = 0; +#endif + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on + * failure. A maximum request size of 256 bytes is imposed. + **/ +static int +crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) +{ + tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); + + /* We only log at notice-level here because in the case that this function + * fails the crypto_strongest_rand_raw() caller will log with a warning-level + * message and let crypto_strongest_rand() error out and finally terminating + * Tor with an assertion error. + */ + +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_syscall) + return -1; +#endif + +#if defined(_WIN32) + static int provider_set = 0; + static HCRYPTPROV provider; + + if (!provider_set) { + if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT)) { + log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1]."); + return -1; + } + provider_set = 1; + } + if (!CryptGenRandom(provider, out_len, out)) { + log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI."); + return -1; + } + + return 0; +#elif defined(__linux__) && defined(SYS_getrandom) + static int getrandom_works = 1; /* Be optimistic about our chances... */ + + /* getrandom() isn't as straightforward as getentropy(), and has + * no glibc wrapper. + * + * As far as I can tell from getrandom(2) and the source code, the + * requests we issue will always succeed (though it will block on the + * call if /dev/urandom isn't seeded yet), since we are NOT specifying + * GRND_NONBLOCK and the request is <= 256 bytes. + * + * The manpage is unclear on what happens if a signal interrupts the call + * while the request is blocked due to lack of entropy.... + * + * We optimistically assume that getrandom() is available and functional + * because it is the way of the future, and 2 branch mispredicts pale in + * comparison to the overheads involved with failing to open + * /dev/srandom followed by opening and reading from /dev/urandom. + */ + if (PREDICT_LIKELY(getrandom_works)) { + long ret; + /* A flag of '0' here means to read from '/dev/urandom', and to + * block if insufficient entropy is available to service the + * request. + */ + const unsigned int flags = 0; + do { + ret = syscall(SYS_getrandom, out, out_len, flags); + } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); + + if (PREDICT_UNLIKELY(ret == -1)) { + /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ + tor_assert(errno != EAGAIN); + tor_assert(errno != EINTR); + + /* Useful log message for errno. */ + if (errno == ENOSYS) { + log_notice(LD_CRYPTO, "Can't get entropy from getrandom()." + " You are running a version of Tor built to support" + " getrandom(), but the kernel doesn't implement this" + " function--probably because it is too old?" + " Trying fallback method instead."); + } else { + log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s." + " Trying fallback method instead.", + strerror(errno)); + } + + getrandom_works = 0; /* Don't bother trying again. */ + return -1; + /* LCOV_EXCL_STOP */ + } + + tor_assert(ret == (long)out_len); + return 0; + } + + return -1; /* getrandom() previously failed unexpectedly. */ +#elif defined(HAVE_GETENTROPY) + /* getentropy() is what Linux's getrandom() wants to be when it grows up. + * the only gotcha is that requests are limited to 256 bytes. + */ + return getentropy(out, out_len); +#else + (void) out; +#endif /* defined(_WIN32) || ... */ + + /* This platform doesn't have a supported syscall based random. */ + return -1; +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * via the per-platform fallback mechanism, storing it into <b>out</b>. + * Return 0 on success, -1 on failure. A maximum request size of 256 bytes + * is imposed. + **/ +static int +crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) +{ +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_fallback) + return -1; +#endif + +#ifdef _WIN32 + /* Windows exclusively uses crypto_strongest_rand_syscall(). */ + (void)out; + (void)out_len; + return -1; +#else /* !(defined(_WIN32)) */ + static const char *filenames[] = { + "/dev/srandom", "/dev/urandom", "/dev/random", NULL + }; + int fd, i; + size_t n; + + for (i = 0; filenames[i]; ++i) { + log_debug(LD_FS, "Considering %s as entropy source", filenames[i]); + fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); + if (fd<0) continue; + log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); + n = read_all(fd, (char*)out, out_len, 0); + close(fd); + if (n != out_len) { + /* LCOV_EXCL_START + * We can't make /dev/foorandom actually fail. */ + log_notice(LD_CRYPTO, + "Error reading from entropy source %s (read only %lu bytes).", + filenames[i], + (unsigned long)n); + return -1; + /* LCOV_EXCL_STOP */ + } + + return 0; + } + + return -1; +#endif /* defined(_WIN32) */ +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum + * request size of 256 bytes is imposed. + **/ +STATIC int +crypto_strongest_rand_raw(uint8_t *out, size_t out_len) +{ + static const size_t sanity_min_size = 16; + static const int max_attempts = 3; + tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); + + /* For buffers >= 16 bytes (128 bits), we sanity check the output by + * zero filling the buffer and ensuring that it actually was at least + * partially modified. + * + * Checking that any individual byte is non-zero seems like it would + * fail too often (p = out_len * 1/256) for comfort, but this is an + * "adjust according to taste" sort of check. + */ + memwipe(out, 0, out_len); + for (int i = 0; i < max_attempts; i++) { + /* Try to use the syscall/OS favored mechanism to get strong entropy. */ + if (crypto_strongest_rand_syscall(out, out_len) != 0) { + /* Try to use the less-favored mechanism to get strong entropy. */ + if (crypto_strongest_rand_fallback(out, out_len) != 0) { + /* Welp, we tried. Hopefully the calling code terminates the process + * since we're basically boned without good entropy. + */ + log_warn(LD_CRYPTO, + "Cannot get strong entropy: no entropy source found."); + return -1; + } + } + + if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) + return 0; + } + + /* LCOV_EXCL_START + * + * We tried max_attempts times to fill a buffer >= 128 bits long, + * and each time it returned all '0's. Either the system entropy + * source is busted, or the user should go out and buy a ticket to + * every lottery on the planet. + */ + log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); + + return -1; + /* LCOV_EXCL_STOP */ +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * storing it into <b>out</b>. + **/ +void +crypto_strongest_rand(uint8_t *out, size_t out_len) +{ +#define DLEN SHA512_DIGEST_LENGTH + /* We're going to hash DLEN bytes from the system RNG together with some + * bytes from the openssl PRNG, in order to yield DLEN bytes. + */ + uint8_t inp[DLEN*2]; + uint8_t tmp[DLEN]; + tor_assert(out); + while (out_len) { + crypto_rand((char*) inp, DLEN); + if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { + // LCOV_EXCL_START + log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " + "important key. Exiting."); + /* Die with an assertion so we get a stack trace. */ + tor_assert(0); + // LCOV_EXCL_STOP + } + if (out_len >= DLEN) { + SHA512(inp, sizeof(inp), out); + out += DLEN; + out_len -= DLEN; + } else { + SHA512(inp, sizeof(inp), tmp); + memcpy(out, tmp, out_len); + break; + } + } + memwipe(tmp, 0, sizeof(tmp)); + memwipe(inp, 0, sizeof(inp)); +#undef DLEN +} + +/** + * Seed OpenSSL's random number generator with bytes from the operating + * system. Return 0 on success, -1 on failure. + **/ +int +crypto_seed_rng(void) +{ + int rand_poll_ok = 0, load_entropy_ok = 0; + uint8_t buf[ADD_ENTROPY]; + + /* OpenSSL has a RAND_poll function that knows about more kinds of + * entropy than we do. We'll try calling that, *and* calling our own entropy + * functions. If one succeeds, we'll accept the RNG as seeded. */ + rand_poll_ok = RAND_poll(); + if (rand_poll_ok == 0) + log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE + + load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); + if (load_entropy_ok) { + RAND_seed(buf, sizeof(buf)); + } + + memwipe(buf, 0, sizeof(buf)); + + if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1) + return 0; + else + return -1; +} + +/** + * Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking + * for unit tests. + * + * This function is not allowed to fail; if it would fail to generate strong + * entropy, it must terminate the process instead. + **/ +MOCK_IMPL(void, +crypto_rand, (char *to, size_t n)) +{ + crypto_rand_unmocked(to, n); +} + +/** + * Write <b>n</b> bytes of strong random data to <b>to</b>. Most callers + * will want crypto_rand instead. + * + * This function is not allowed to fail; if it would fail to generate strong + * entropy, it must terminate the process instead. + **/ +void +crypto_rand_unmocked(char *to, size_t n) +{ + int r; + if (n == 0) + return; + + tor_assert(n < INT_MAX); + tor_assert(to); + r = RAND_bytes((unsigned char*)to, (int)n); + /* We consider a PRNG failure non-survivable. Let's assert so that we get a + * stack trace about where it happened. + */ + tor_assert(r >= 0); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + unsigned int val; + unsigned int cutoff; + tor_assert(max <= ((unsigned int)INT_MAX)+1); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT_MAX - (UINT_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + uint64_t val; + uint64_t cutoff; + tor_assert(max < UINT64_MAX); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT64_MAX - (UINT64_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * Generate and return a new random hostname starting with <b>prefix</b>, + * ending with <b>suffix</b>, and containing no fewer than + * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 + * characters. Does not check for failure. + * + * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE. + **/ +char * +crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, + const char *suffix) +{ + char *result, *rand_bytes; + int randlen, rand_bytes_len; + size_t resultlen, prefixlen; + + if (max_rand_len > MAX_DNS_LABEL_SIZE) + max_rand_len = MAX_DNS_LABEL_SIZE; + if (min_rand_len > max_rand_len) + min_rand_len = max_rand_len; + + randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1); + + prefixlen = strlen(prefix); + resultlen = prefixlen + strlen(suffix) + randlen + 16; + + rand_bytes_len = ((randlen*5)+7)/8; + if (rand_bytes_len % 5) + rand_bytes_len += 5 - (rand_bytes_len%5); + rand_bytes = tor_malloc(rand_bytes_len); + crypto_rand(rand_bytes, rand_bytes_len); + + result = tor_malloc(resultlen); + memcpy(result, prefix, prefixlen); + base32_encode(result+prefixlen, resultlen-prefixlen, + rand_bytes, rand_bytes_len); + tor_free(rand_bytes); + strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen)); + + return result; +} + +/** + * Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b> + * is empty. + **/ +void * +smartlist_choose(const smartlist_t *sl) +{ + int len = smartlist_len(sl); + if (len) + return smartlist_get(sl,crypto_rand_int(len)); + return NULL; /* no elements to choose from */ +} + +/** + * Scramble the elements of <b>sl</b> into a random order. + **/ +void +smartlist_shuffle(smartlist_t *sl) +{ + int i; + /* From the end of the list to the front, choose at random from the + positions we haven't looked at yet, and swap that position into the + current position. Remember to give "no swap" the same probability as + any other swap. */ + for (i = smartlist_len(sl)-1; i > 0; --i) { + int j = crypto_rand_int(i+1); + smartlist_swap(sl, i, j); + } +} + +/** Make sure that openssl is using its default PRNG. Return 1 if we had to + * adjust it; 0 otherwise. */ +int +crypto_force_rand_ssleay(void) +{ + RAND_METHOD *default_method; + default_method = RAND_OpenSSL(); + if (RAND_get_rand_method() != default_method) { + log_notice(LD_CRYPTO, "It appears that one of our engines has provided " + "a replacement the OpenSSL RNG. Resetting it to the default " + "implementation."); + RAND_set_rand_method(default_method); + return 1; + } + return 0; +} + +#endif /* !defined(CRYPTO_RAND_PRIVATE) */ + diff --git a/src/common/crypto_rand.h b/src/common/crypto_rand.h new file mode 100644 index 0000000000..bb02e51001 --- /dev/null +++ b/src/common/crypto_rand.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand.h + * + * \brief Common functions for using (pseudo-)random number generators. + **/ + +#ifndef TOR_CRYPTO_RAND_H +#define TOR_CRYPTO_RAND_H + +#include "torint.h" +#include "util.h" + +/* random numbers */ +int crypto_seed_rng(void) ATTR_WUR; +MOCK_DECL(void,crypto_rand,(char *to, size_t n)); +void crypto_rand_unmocked(char *to, size_t n); +void crypto_strongest_rand(uint8_t *out, size_t out_len); +int crypto_rand_int(unsigned int max); +int crypto_rand_int_range(unsigned int min, unsigned int max); +uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); +time_t crypto_rand_time_range(time_t min, time_t max); +uint64_t crypto_rand_uint64(uint64_t max); +double crypto_rand_double(void); +struct tor_weak_rng_t; +void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); + +char *crypto_random_hostname(int min_rand_len, int max_rand_len, + const char *prefix, const char *suffix); + +struct smartlist_t; +void *smartlist_choose(const struct smartlist_t *sl); +void smartlist_shuffle(struct smartlist_t *sl); +int crypto_force_rand_ssleay(void); + +#ifdef CRYPTO_RAND_PRIVATE + +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); + +#ifdef TOR_UNIT_TESTS +extern int break_strongest_rng_syscall; +extern int break_strongest_rng_fallback; +#endif +#endif /* defined(CRYPTO_RAND_PRIVATE) */ + +#endif /* !defined(TOR_CRYPTO_RAND_H) */ + diff --git a/src/common/crypto_rsa.c b/src/common/crypto_rsa.c index 259656810b..f66cdef3c5 100644 --- a/src/common/crypto_rsa.c +++ b/src/common/crypto_rsa.c @@ -9,12 +9,14 @@ * \brief Block of functions related with RSA utilities and operations. **/ -#include "crypto_rsa.h" #include "crypto.h" -#include "compat_openssl.h" #include "crypto_curve25519.h" -#include "crypto_ed25519.h" +#include "crypto_digest.h" #include "crypto_format.h" +#include "compat_openssl.h" +#include "crypto_rand.h" +#include "crypto_rsa.h" +#include "crypto_util.h" DISABLE_GCC_WARNING(redundant-decls) @@ -627,6 +629,148 @@ crypto_pk_copy_full(crypto_pk_t *env) return crypto_new_pk_from_rsa_(new_key); } +/** Perform a hybrid (public/secret) encryption on <b>fromlen</b> + * bytes of data from <b>from</b>, with padding type 'padding', + * storing the results on <b>to</b>. + * + * Returns the number of bytes written on success, -1 on failure. + * + * The encrypted data consists of: + * - The source data, padded and encrypted with the public key, if the + * padded source data is no longer than the public key, and <b>force</b> + * is false, OR + * - The beginning of the source data prefixed with a 16-byte symmetric key, + * padded and encrypted with the public key; followed by the rest of + * the source data encrypted in AES-CTR mode with the symmetric key. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ +int +crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, + char *to, size_t tolen, + const char *from, + size_t fromlen, + int padding, int force) +{ + int overhead, outlen, r; + size_t pkeylen, symlen; + crypto_cipher_t *cipher = NULL; + char *buf = NULL; + + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(fromlen < SIZE_T_CEILING); + + overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding)); + pkeylen = crypto_pk_keysize(env); + + if (!force && fromlen+overhead <= pkeylen) { + /* It all fits in a single encrypt. */ + return crypto_pk_public_encrypt(env,to, + tolen, + from,fromlen,padding); + } + tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN); + tor_assert(tolen >= pkeylen); + + char key[CIPHER_KEY_LEN]; + crypto_rand(key, sizeof(key)); /* generate a new key. */ + cipher = crypto_cipher_new(key); + + buf = tor_malloc(pkeylen+1); + memcpy(buf, key, CIPHER_KEY_LEN); + memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN); + + /* Length of symmetrically encrypted data. */ + symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN); + + outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding); + if (outlen!=(int)pkeylen) { + goto err; + } + r = crypto_cipher_encrypt(cipher, to+outlen, + from+pkeylen-overhead-CIPHER_KEY_LEN, symlen); + + if (r<0) goto err; + memwipe(buf, 0, pkeylen); + memwipe(key, 0, sizeof(key)); + tor_free(buf); + crypto_cipher_free(cipher); + tor_assert(outlen+symlen < INT_MAX); + return (int)(outlen + symlen); + err: + + memwipe(buf, 0, pkeylen); + memwipe(key, 0, sizeof(key)); + tor_free(buf); + crypto_cipher_free(cipher); + return -1; +} + +/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of + * bytes written on success, -1 on failure. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ +int +crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, + char *to, + size_t tolen, + const char *from, + size_t fromlen, + int padding, int warnOnFailure) +{ + int outlen, r; + size_t pkeylen; + crypto_cipher_t *cipher = NULL; + char *buf = NULL; + + tor_assert(fromlen < SIZE_T_CEILING); + pkeylen = crypto_pk_keysize(env); + + if (fromlen <= pkeylen) { + return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding, + warnOnFailure); + } + + buf = tor_malloc(pkeylen); + outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding, + warnOnFailure); + if (outlen<0) { + log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO, + "Error decrypting public-key data"); + goto err; + } + if (outlen < CIPHER_KEY_LEN) { + log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO, + "No room for a symmetric key"); + goto err; + } + cipher = crypto_cipher_new(buf); + if (!cipher) { + goto err; + } + memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN); + outlen -= CIPHER_KEY_LEN; + tor_assert(tolen - outlen >= fromlen - pkeylen); + r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen); + if (r<0) + goto err; + memwipe(buf,0,pkeylen); + tor_free(buf); + crypto_cipher_free(cipher); + tor_assert(outlen + fromlen < INT_MAX); + return (int)(outlen + (fromlen-pkeylen)); + err: + memwipe(buf,0,pkeylen); + tor_free(buf); + crypto_cipher_free(cipher); + return -1; +} + /** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key * in <b>env</b>, using the padding method <b>padding</b>. On success, * write the result to <b>to</b>, and return the number of bytes @@ -849,6 +993,122 @@ crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) return 0; } +/** Check a siglen-byte long signature at <b>sig</b> against + * <b>datalen</b> bytes of data at <b>data</b>, using the public key + * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for + * SHA1(data). Else return -1. + */ +MOCK_IMPL(int, +crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, + size_t datalen, const char *sig, + size_t siglen)) +{ + char digest[DIGEST_LEN]; + char *buf; + size_t buflen; + int r; + + tor_assert(env); + tor_assert(data); + tor_assert(sig); + tor_assert(datalen < SIZE_T_CEILING); + tor_assert(siglen < SIZE_T_CEILING); + + if (crypto_digest(digest,data,datalen)<0) { + log_warn(LD_BUG, "couldn't compute digest"); + return -1; + } + buflen = crypto_pk_keysize(env); + buf = tor_malloc(buflen); + r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen); + if (r != DIGEST_LEN) { + log_warn(LD_CRYPTO, "Invalid signature"); + tor_free(buf); + return -1; + } + if (tor_memneq(buf, digest, DIGEST_LEN)) { + log_warn(LD_CRYPTO, "Signature mismatched with digest."); + tor_free(buf); + return -1; + } + tor_free(buf); + + return 0; +} + +/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at + * <b>from</b>; sign the data with the private key in <b>env</b>, and + * store it in <b>to</b>. Return the number of bytes written on + * success, and -1 on failure. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +int +crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen) +{ + int r; + char digest[DIGEST_LEN]; + if (crypto_digest(digest,from,fromlen)<0) + return -1; + r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN); + memwipe(digest, 0, sizeof(digest)); + return r; +} + +/** Given a private or public key <b>pk</b>, put a SHA1 hash of the + * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space). + * Return 0 on success, -1 on failure. + */ +int +crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out) +{ + char *buf; + size_t buflen; + int len; + int rv = -1; + + buflen = crypto_pk_keysize(pk)*2; + buf = tor_malloc(buflen); + len = crypto_pk_asn1_encode(pk, buf, buflen); + if (len < 0) + goto done; + + if (crypto_digest(digest_out, buf, len) < 0) + goto done; + + rv = 0; + done: + tor_free(buf); + return rv; +} + +/** Compute all digests of the DER encoding of <b>pk</b>, and store them + * in <b>digests_out</b>. Return 0 on success, -1 on failure. */ +int +crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out) +{ + char *buf; + size_t buflen; + int len; + int rv = -1; + + buflen = crypto_pk_keysize(pk)*2; + buf = tor_malloc(buflen); + len = crypto_pk_asn1_encode(pk, buf, buflen); + if (len < 0) + goto done; + + if (crypto_common_digests(digests_out, (char*)buf, len) < 0) + goto done; + + rv = 0; + done: + tor_free(buf); + return rv; +} + /** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the * Base64 encoding of the DER representation of the private key as a NUL * terminated string, and return it via <b>priv_out</b>. Return 0 on diff --git a/src/common/crypto_rsa.h b/src/common/crypto_rsa.h index 5b9025c629..e952089318 100644 --- a/src/common/crypto_rsa.h +++ b/src/common/crypto_rsa.h @@ -15,13 +15,13 @@ #include "orconfig.h" +#include "crypto_digest.h" #include <stdio.h> #include "torint.h" #include "testsupport.h" #include "compat.h" #include "util.h" #include "torlog.h" -#include "crypto_curve25519.h" /** Length of our public keys. */ #define PK_BYTES (1024/8) @@ -35,7 +35,7 @@ /** A public key, or a public/private key-pair. */ typedef struct crypto_pk_t crypto_pk_t; -/* RSA enviroment setup */ +/* RSA environment setup */ MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void)); void crypto_pk_free_(crypto_pk_t *env); #define crypto_pk_free(pk) FREE_AND_NULL(crypto_pk_t, crypto_pk_free_, (pk)) @@ -69,6 +69,14 @@ crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig); crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig); int crypto_pk_key_is_private(const crypto_pk_t *key); int crypto_pk_public_exponent_ok(crypto_pk_t *env); +int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int force); +int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int warnOnFailure); int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, int padding); int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen, @@ -84,6 +92,13 @@ crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len); int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space); int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out); +MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, + const char *data, size_t datalen, const char *sig, size_t siglen)); +int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen); +int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out); +int crypto_pk_get_common_digests(crypto_pk_t *pk, + common_digests_t *digests_out); int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out); crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len); diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index b2fcca54c4..8543760ec5 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -12,10 +12,13 @@ #define CRYPTO_S2K_PRIVATE -#include "crypto.h" -#include "util.h" #include "compat.h" +#include "crypto.h" +#include "crypto_digest.h" +#include "crypto_rand.h" #include "crypto_s2k.h" +#include "crypto_util.h" +#include "util.h" #include <openssl/evp.h> diff --git a/src/common/crypto_util.c b/src/common/crypto_util.c new file mode 100644 index 0000000000..b0d5b6b2f7 --- /dev/null +++ b/src/common/crypto_util.c @@ -0,0 +1,107 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_util.c + * + * \brief Common cryptographic utilities. + **/ + +#ifndef CRYPTO_UTIL_PRIVATE +#define CRYPTO_UTIL_PRIVATE + +#include "crypto_util.h" + +#include <string.h> + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#include <wincrypt.h> +#endif /* defined(_WIN32) */ + +#include "util.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/crypto.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/** + * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to + * the value <b>byte</b>. + * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens. + * + * This function is preferable to memset, since many compilers will happily + * optimize out memset() when they can convince themselves that the data being + * cleared will never be read. + * + * Right now, our convention is to use this function when we are wiping data + * that's about to become inaccessible, such as stack buffers that are about + * to go out of scope or structures that are about to get freed. (In + * practice, it appears that the compilers we're currently using will optimize + * out the memset()s for stack-allocated buffers, but not those for + * about-to-be-freed structures. That could change, though, so we're being + * wary.) If there are live reads for the data, then you can just use + * memset(). + */ +void +memwipe(void *mem, uint8_t byte, size_t sz) +{ + if (sz == 0) { + return; + } + /* If sz is nonzero, then mem must not be NULL. */ + tor_assert(mem != NULL); + + /* Data this large is likely to be an underflow. */ + tor_assert(sz < SIZE_T_CEILING); + + /* Because whole-program-optimization exists, we may not be able to just + * have this function call "memset". A smart compiler could inline it, then + * eliminate dead memsets, and declare itself to be clever. */ + +#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) + /* Here's what you do on windows. */ + SecureZeroMemory(mem,sz); +#elif defined(HAVE_RTLSECUREZEROMEMORY) + RtlSecureZeroMemory(mem,sz); +#elif defined(HAVE_EXPLICIT_BZERO) + /* The BSDs provide this. */ + explicit_bzero(mem, sz); +#elif defined(HAVE_MEMSET_S) + /* This is in the C99 standard. */ + memset_s(mem, sz, 0, sz); +#else + /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk + * based on the pointer value, then uses that junk to update a global + * variable. It's an elaborate ruse to trick the compiler into not + * optimizing out the "wipe this memory" code. Read it if you like zany + * programming tricks! In later versions of Tor, we should look for better + * not-optimized-out memory wiping stuff... + * + * ...or maybe not. In practice, there are pure-asm implementations of + * OPENSSL_cleanse() on most platforms, which ought to do the job. + **/ + + OPENSSL_cleanse(mem, sz); +#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */ + + /* Just in case some caller of memwipe() is relying on getting a buffer + * filled with a particular value, fill the buffer. + * + * If this function gets inlined, this memset might get eliminated, but + * that's okay: We only care about this particular memset in the case where + * the caller should have been using memset(), and the memset() wouldn't get + * eliminated. In other words, this is here so that we won't break anything + * if somebody accidentally calls memwipe() instead of memset(). + **/ + memset(mem, byte, sz); +} + +#endif /* !defined(CRYPTO_UTIL_PRIVATE) */ + diff --git a/src/common/crypto_util.h b/src/common/crypto_util.h new file mode 100644 index 0000000000..922942b371 --- /dev/null +++ b/src/common/crypto_util.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_util.h + * + * \brief Common functions for cryptographic routines. + **/ + +#ifndef TOR_CRYPTO_UTIL_H +#define TOR_CRYPTO_UTIL_H + +#include "torint.h" + +/** OpenSSL-based utility functions. */ +void memwipe(void *mem, uint8_t byte, size_t sz); + +#ifdef CRYPTO_UTIL_PRIVATE +#ifdef TOR_UNIT_TESTS +#endif /* defined(TOR_UNIT_TESTS) */ +#endif /* defined(CRYPTO_UTIL_PRIVATE) */ + +#endif /* !defined(TOR_CRYPTO_UTIL_H) */ + diff --git a/src/common/include.am b/src/common/include.am index 6945285108..cfaf993674 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -74,7 +74,11 @@ LIBOR_CTIME_A_SRC = \ src/common/di_ops.c src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC) +if UNITTESTS_ENABLED src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC) +else +src_common_libor_ctime_testing_a_SOURCES = +endif src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) @@ -97,6 +101,7 @@ LIBOR_A_SRC = \ src/common/util_process.c \ src/common/sandbox.c \ src/common/storagedir.c \ + src/common/token_bucket.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ @@ -114,11 +119,14 @@ LIBOR_CRYPTO_A_SRC = \ src/common/compress_zlib.c \ src/common/compress_zstd.c \ src/common/crypto.c \ - src/common/crypto_rsa.c \ + src/common/crypto_digest.c \ + src/common/crypto_format.c \ src/common/crypto_openssl_mgt.c \ src/common/crypto_pwbox.c \ + src/common/crypto_rand.c \ + src/common/crypto_rsa.c \ src/common/crypto_s2k.c \ - src/common/crypto_format.c \ + src/common/crypto_util.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -133,9 +141,15 @@ src_common_libor_a_SOURCES = $(LIBOR_A_SRC) src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC) +if UNITTESTS_ENABLED src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC) src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC) +else +src_common_libor_testing_a_SOURCES = +src_common_libor_crypto_testing_a_SOURCES = +src_common_libor_event_testing_a_SOURCES = +endif src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) @@ -165,13 +179,16 @@ COMMONHEADERS = \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ + src/common/crypto_digest.h \ src/common/crypto_curve25519.h \ src/common/crypto_ed25519.h \ src/common/crypto_format.h \ src/common/crypto_openssl_mgt.h \ - src/common/crypto_rsa.h \ src/common/crypto_pwbox.h \ + src/common/crypto_rand.h \ + src/common/crypto_rsa.h \ src/common/crypto_s2k.h \ + src/common/crypto_util.h \ src/common/di_ops.h \ src/common/handles.h \ src/common/memarea.h \ @@ -182,6 +199,7 @@ COMMONHEADERS = \ src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ + src/common/token_bucket.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/log.c b/src/common/log.c index 9f4a8b2bc2..ebd50f62d3 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -52,6 +52,13 @@ #define raw_assert(x) assert(x) // assert OK +/** Defining compile-time constants for Tor log levels (used by the Rust + * log wrapper at src/rust/tor_log) */ +const int LOG_WARN_ = LOG_WARN; +const int LOG_NOTICE_ = LOG_NOTICE; +const log_domain_mask_t LD_GENERAL_ = LD_GENERAL; +const log_domain_mask_t LD_NET_ = LD_NET; + /** Information for a single logfile; only used in log.c */ typedef struct logfile_t { struct logfile_t *next; /**< Next logfile_t in the linked list. */ @@ -163,6 +170,9 @@ typedef struct pending_log_message_t { /** Log messages waiting to be replayed onto callback-based logs */ static smartlist_t *pending_cb_messages = NULL; +/** Callback to invoke when pending_cb_messages becomes nonempty. */ +static pending_callback_callback pending_cb_cb = NULL; + /** Log messages waiting to be replayed once the logging system is initialized. */ static smartlist_t *pending_startup_messages = NULL; @@ -225,6 +235,30 @@ log_set_application_name(const char *name) appname = name ? tor_strdup(name) : NULL; } +/** Return true if some of the running logs might be interested in a log + * message of the given severity in the given domains. If this function + * returns true, the log message might be ignored anyway, but if it returns + * false, it is definitely_ safe not to log the message. */ +int +log_message_is_interesting(int severity, log_domain_mask_t domain) +{ + (void) domain; + return (severity <= log_global_min_severity_); +} + +/** + * As tor_log, but takes an optional function name, and does not treat its + * <b>string</b> as a printf format. + * + * For use by Rust integration. + */ +void +tor_log_string(int severity, log_domain_mask_t domain, + const char *function, const char *string) +{ + log_fn_(severity, domain, function, "%s", string); +} + /** Log time granularity in milliseconds. */ static int log_time_granularity = 1; @@ -507,6 +541,9 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, smartlist_add(pending_cb_messages, pending_log_message_new(severity,domain,NULL,msg_after_prefix)); *callbacks_deferred = 1; + if (smartlist_len(pending_cb_messages) == 1 && pending_cb_cb) { + pending_cb_cb(); + } } } else { lf->callback(severity, domain, msg_after_prefix); @@ -794,6 +831,7 @@ logs_free_all(void) logfiles = NULL; messages = pending_cb_messages; pending_cb_messages = NULL; + pending_cb_cb = NULL; messages2 = pending_startup_messages; pending_startup_messages = NULL; UNLOCK_LOGS(); @@ -957,6 +995,24 @@ add_temp_log(int min_severity) } /** + * Register "cb" as the callback to call when there are new pending log + * callbacks to be flushed with flush_pending_log_callbacks(). + * + * Note that this callback, if present, can be invoked from any thread. + * + * This callback must not log. + * + * It is intentional that this function contains the name "callback" twice: it + * sets a "callback" to be called on the condition that there is a "pending + * callback". + **/ +void +logs_set_pending_callback_callback(pending_callback_callback cb) +{ + pending_cb_cb = cb; +} + +/** * Add a log handler to send messages in <b>severity</b> * to the function <b>cb</b>. */ diff --git a/src/common/procmon.c b/src/common/procmon.c index abcbbeaa21..73c14cd584 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -10,8 +10,6 @@ #include "util.h" -#include <event2/event.h> - #ifdef HAVE_SIGNAL_H #include <signal.h> #endif @@ -44,7 +42,7 @@ typedef int pid_t; /* Currently we need to poll in some way on all systems. */ #ifdef PROCMON_POLLS -static void tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, +static void tor_process_monitor_poll_cb(periodic_timer_t *ev, void *procmon_); #endif @@ -136,7 +134,7 @@ struct tor_process_monitor_t { /** A Libevent event structure, to either poll for the process's * existence or receive a notification when the process ends. */ - struct event *e; + periodic_timer_t *e; /** A callback to be called when the process ends. */ tor_procmon_callback_t cb; @@ -159,9 +157,6 @@ tor_validate_process_specifier(const char *process_spec, return parse_process_specifier(process_spec, &ppspec, msg); } -/* XXXX we should use periodic_timer_new() for this stuff */ -#define PERIODIC_TIMER_FLAGS EV_PERSIST - /* DOCDOC poll_interval_tv */ static const struct timeval poll_interval_tv = {15, 0}; @@ -225,13 +220,9 @@ tor_process_monitor_new(struct event_base *base, procmon->cb_arg = cb_arg; #ifdef PROCMON_POLLS - procmon->e = tor_event_new(base, -1 /* no FD */, PERIODIC_TIMER_FLAGS, - tor_process_monitor_poll_cb, procmon); - /* Note: If you port this file to plain Libevent 2, check that - * procmon->e is non-NULL. We don't need to here because - * tor_evtimer_new never returns NULL. */ - - evtimer_add(procmon->e, &poll_interval_tv); + procmon->e = periodic_timer_new(base, + &poll_interval_tv, + tor_process_monitor_poll_cb, procmon); #else /* !(defined(PROCMON_POLLS)) */ #error OOPS? #endif /* defined(PROCMON_POLLS) */ @@ -246,14 +237,12 @@ tor_process_monitor_new(struct event_base *base, /** Libevent callback to poll for the existence of the process * monitored by <b>procmon_</b>. */ static void -tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, - void *procmon_) +tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) { + (void)event; tor_process_monitor_t *procmon = (tor_process_monitor_t *)(procmon_); int its_dead_jim; - (void)unused1; (void)unused2; - tor_assert(procmon != NULL); #ifdef _WIN32 @@ -336,7 +325,7 @@ tor_process_monitor_free_(tor_process_monitor_t *procmon) #endif if (procmon->e != NULL) - tor_event_free(procmon->e); + periodic_timer_free(procmon->e); tor_free(procmon); } diff --git a/src/common/timers.c b/src/common/timers.c index 552080b11e..6f6236ed3b 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -37,8 +37,6 @@ #include "torlog.h" #include "util.h" -#include <event2/event.h> - struct timeout_cb { timer_cb_fn_t cb; void *arg; @@ -66,10 +64,15 @@ struct timeout_cb { * above TIMEOUT_MAX can also be super-inefficient. Choosing 5 here sets * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */ #define WHEEL_NUM 5 +#if SIZEOF_VOID_P == 4 +/* On 32-bit platforms, we want to override wheel_bit, so that timeout.c will + * use 32-bit math. */ +#define WHEEL_BIT 5 +#endif #include "src/ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; -static struct event *global_timer_event = NULL; +static struct mainloop_event_t *global_timer_event = NULL; static monotime_t start_of_time; @@ -147,7 +150,7 @@ libevent_timer_reschedule(void) if (delay > MIN_CHECK_TICKS) delay = MIN_CHECK_TICKS; timeout_to_tv(delay, &d); - event_add(global_timer_event, &d); + mainloop_event_schedule(global_timer_event, &d); } /** Run the callback of every timer that has expired, based on the current @@ -170,10 +173,9 @@ timers_run_pending(void) * have fired, activate their callbacks, and reschedule the libevent timer. */ static void -libevent_timer_callback(evutil_socket_t fd, short what, void *arg) +libevent_timer_callback(mainloop_event_t *ev, void *arg) { - (void)fd; - (void)what; + (void)ev; (void)arg; timers_run_pending(); @@ -203,9 +205,8 @@ timers_initialize(void) monotime_init(); monotime_get(&start_of_time); - struct event *timer_event; - timer_event = tor_event_new(tor_libevent_get_base(), - -1, 0, libevent_timer_callback, NULL); + mainloop_event_t *timer_event; + timer_event = mainloop_event_new(libevent_timer_callback, NULL); tor_assert(timer_event); global_timer_event = timer_event; @@ -219,7 +220,7 @@ void timers_shutdown(void) { if (global_timer_event) { - tor_event_free(global_timer_event); + mainloop_event_free(global_timer_event); global_timer_event = NULL; } if (global_timeouts) { diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c new file mode 100644 index 0000000000..f2396ec58a --- /dev/null +++ b/src/common/token_bucket.c @@ -0,0 +1,255 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file token_bucket.c + * \brief Functions to use and manipulate token buckets, used for + * rate-limiting on connections and globally. + * + * Tor uses these token buckets to keep track of bandwidth usage, and + * sometimes other things too. + * + * There are two layers of abstraction here: "raw" token buckets, in which all + * the pieces are decoupled, and "read-write" token buckets, which combine all + * the moving parts into one. + * + * Token buckets may become negative. + **/ + +#define TOKEN_BUCKET_PRIVATE + +#include "token_bucket.h" +#include "util_bug.h" + +/** + * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg. + * + * Note that the <b>rate</b> value is in arbitrary units, but those units will + * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and + * so on. + */ +void +token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst) +{ + tor_assert_nonfatal(rate > 0); + tor_assert_nonfatal(burst > 0); + if (burst > TOKEN_BUCKET_MAX_BURST) + burst = TOKEN_BUCKET_MAX_BURST; + + cfg->rate = rate; + cfg->burst = burst; +} + +/** + * Initialize a raw token bucket and its associated timestamp to the "full" + * state, according to <b>cfg</b>. + */ +void +token_bucket_raw_reset(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = cfg->burst; +} + +/** + * Adust a preexisting token bucket to respect the new configuration + * <b>cfg</b>, by decreasing its current level if needed. */ +void +token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = MIN(bucket->bucket, cfg->burst); +} + +/** + * Given an amount of <b>elapsed</b> time units, and a bucket configuration + * <b>cfg</b>, refill the level of <b>bucket</b> accordingly. Note that the + * units of time in <b>elapsed</b> must correspond to those used to set the + * rate in <b>cfg</b>, or the result will be illogical. + */ +int +token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed) +{ + const int was_empty = (bucket->bucket <= 0); + /* The casts here prevent an underflow. + * + * Note that even if the bucket value is negative, subtracting it from + * "burst" will still produce a correct result. If this result is + * ridiculously high, then the "elapsed > gap / rate" check below + * should catch it. */ + const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket); + + if (elapsed > gap / cfg->rate) { + bucket->bucket = cfg->burst; + } else { + bucket->bucket += cfg->rate * elapsed; + } + + return was_empty && bucket->bucket > 0; +} + +/** + * Decrement a provided bucket by <b>n</b> units. Note that <b>n</b> + * must be nonnegative. + */ +int +token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n) +{ + if (BUG(n < 0)) + return 0; + const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket; + bucket->bucket -= n; + return becomes_empty; +} + +/** Convert a rate in bytes per second to a rate in bytes per step */ +STATIC uint32_t +rate_per_sec_to_rate_per_step(uint32_t rate) +{ + /* + The precise calculation we'd want to do is + + (rate / 1000) * to_approximate_msec(TICKS_PER_STEP). But to minimize + rounding error, we do it this way instead, and divide last. + */ + uint64_t units = (uint64_t) rate * TICKS_PER_STEP; + uint32_t val = (uint32_t) + (monotime_coarse_stamp_units_to_approx_msec(units) / 1000); + return val ? val : 1; +} + +/** + * Initialize a token bucket in *<b>bucket</b>, set up to allow <b>rate</b> + * bytes per second, with a maximum burst of <b>burst</b> bytes. The bucket + * is created such that <b>now_ts</b> is the current timestamp. The bucket + * starts out full. + */ +void +token_bucket_rw_init(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst, + uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_rw_t)); + token_bucket_rw_adjust(bucket, rate, burst); + token_bucket_rw_reset(bucket, now_ts); +} + +/** + * Change the configured rate (in bytes per second) and burst (in bytes) + * for the token bucket in *<b>bucket</b>. + */ +void +token_bucket_rw_adjust(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, + rate_per_sec_to_rate_per_step(rate), + burst); + token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg); +} + +/** + * Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. + */ +void +token_bucket_rw_reset(token_bucket_rw_t *bucket, + uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** + * Refill <b>bucket</b> as appropriate, given that the current timestamp + * is <b>now_ts</b>. + * + * Return a bitmask containing TB_READ iff read bucket was empty and became + * nonempty, and TB_WRITE iff the write bucket was empty and became nonempty. + */ +int +token_bucket_rw_refill(token_bucket_rw_t *bucket, + uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return 0; + } + const uint32_t elapsed_steps = elapsed_ticks / TICKS_PER_STEP; + + if (!elapsed_steps) { + /* Note that if less than one whole step elapsed, we don't advance the + * time in last_refilled_at. That's intentional: we want to make sure + * that we add some bytes to it eventually. */ + return 0; + } + + int flags = 0; + if (token_bucket_raw_refill_steps(&bucket->read_bucket, + &bucket->cfg, elapsed_steps)) + flags |= TB_READ; + if (token_bucket_raw_refill_steps(&bucket->write_bucket, + &bucket->cfg, elapsed_steps)) + flags |= TB_WRITE; + + bucket->last_refilled_at_timestamp = now_ts; + return flags; +} + +/** + * Decrement the read token bucket in <b>bucket</b> by <b>n</b> bytes. + * + * Return true if the bucket was nonempty and became empty; return false + * otherwise. + */ +int +token_bucket_rw_dec_read(token_bucket_rw_t *bucket, + ssize_t n) +{ + return token_bucket_raw_dec(&bucket->read_bucket, n); +} + +/** + * Decrement the write token bucket in <b>bucket</b> by <b>n</b> bytes. + * + * Return true if the bucket was nonempty and became empty; return false + * otherwise. + */ +int +token_bucket_rw_dec_write(token_bucket_rw_t *bucket, + ssize_t n) +{ + return token_bucket_raw_dec(&bucket->write_bucket, n); +} + +/** + * As token_bucket_rw_dec_read and token_bucket_rw_dec_write, in a single + * operation. Return a bitmask of TB_READ and TB_WRITE to indicate + * which buckets became empty. + */ +int +token_bucket_rw_dec(token_bucket_rw_t *bucket, + ssize_t n_read, ssize_t n_written) +{ + int flags = 0; + if (token_bucket_rw_dec_read(bucket, n_read)) + flags |= TB_READ; + if (token_bucket_rw_dec_write(bucket, n_written)) + flags |= TB_WRITE; + return flags; +} + diff --git a/src/common/token_bucket.h b/src/common/token_bucket.h new file mode 100644 index 0000000000..0e7832e838 --- /dev/null +++ b/src/common/token_bucket.h @@ -0,0 +1,118 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file token_bucket_rw.h + * \brief Headers for token_bucket_rw.c + **/ + +#ifndef TOR_TOKEN_BUCKET_H +#define TOR_TOKEN_BUCKET_H + +#include "torint.h" +#include "testsupport.h" + +/** Largest allowable burst value for a token buffer. */ +#define TOKEN_BUCKET_MAX_BURST INT32_MAX + +/** A generic token buffer configuration: determines the number of tokens + * added to the bucket in each time unit (the "rate"), and the maximum number + * of tokens in the bucket (the "burst") */ +typedef struct token_bucket_cfg_t { + uint32_t rate; + int32_t burst; +} token_bucket_cfg_t; + +/** A raw token bucket, decoupled from its configuration and timestamp. */ +typedef struct token_bucket_raw_t { + int32_t bucket; +} token_bucket_raw_t; + +void token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst); + +void token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +void token_bucket_raw_reset(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +int token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n); + +int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed_steps); + +static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket); +/** Return the current number of bytes set in a token bucket. */ +static inline size_t +token_bucket_raw_get(const token_bucket_raw_t *bucket) +{ + return bucket->bucket >= 0 ? bucket->bucket : 0; +} + +/** A convenience type containing all the pieces needed for a coupled + * read-bucket and write-bucket that have the same rate limit, and which use + * "timestamp units" (see compat_time.h) for their time. */ +typedef struct token_bucket_rw_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t read_bucket; + token_bucket_raw_t write_bucket; + uint32_t last_refilled_at_timestamp; +} token_bucket_rw_t; + +void token_bucket_rw_init(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst, + uint32_t now_ts); + +void token_bucket_rw_adjust(token_bucket_rw_t *bucket, + uint32_t rate, uint32_t burst); + +void token_bucket_rw_reset(token_bucket_rw_t *bucket, + uint32_t now_ts); + +#define TB_READ 1 +#define TB_WRITE 2 + +int token_bucket_rw_refill(token_bucket_rw_t *bucket, + uint32_t now_ts); + +int token_bucket_rw_dec_read(token_bucket_rw_t *bucket, + ssize_t n); +int token_bucket_rw_dec_write(token_bucket_rw_t *bucket, + ssize_t n); + +int token_bucket_rw_dec(token_bucket_rw_t *bucket, + ssize_t n_read, ssize_t n_written); + +static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket); +static inline size_t +token_bucket_rw_get_read(const token_bucket_rw_t *bucket) +{ + return token_bucket_raw_get(&bucket->read_bucket); +} + +static inline size_t token_bucket_rw_get_write( + const token_bucket_rw_t *bucket); +static inline size_t +token_bucket_rw_get_write(const token_bucket_rw_t *bucket) +{ + return token_bucket_raw_get(&bucket->write_bucket); +} + +#ifdef TOKEN_BUCKET_PRIVATE + +/* To avoid making the rates too small, we consider units of "steps", + * where a "step" is defined as this many timestamp ticks. Keep this + * a power of two if you can. */ +#define TICKS_PER_STEP 16 + +STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate); + +#endif + +#endif /* TOR_TOKEN_BUCKET_H */ + diff --git a/src/common/torint.h b/src/common/torint.h index 0b8061d24f..fc7818fe2c 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -40,6 +40,8 @@ #include <inttypes.h> #endif +#include <stdbool.h> + #if (SIZEOF_INT8_T != 0) #define HAVE_INT8_T #endif diff --git a/src/common/torlog.h b/src/common/torlog.h index cadfe3b879..de389883c0 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -154,6 +154,8 @@ int add_android_log(const log_severity_list_t *severity, const char *android_identity_tag); #endif // HAVE_ANDROID_LOG_H. int add_callback_log(const log_severity_list_t *severity, log_callback cb); +typedef void (*pending_callback_callback)(void); +void logs_set_pending_callback_callback(pending_callback_callback cb); void logs_set_domain_logging(int enabled); int get_min_log_level(void); void switch_logs_debug(void); @@ -191,6 +193,10 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, const char *format, ...) CHECK_PRINTF(5,6); +int log_message_is_interesting(int severity, log_domain_mask_t domain); +void tor_log_string(int severity, log_domain_mask_t domain, + const char *function, const char *string); + #if defined(__GNUC__) && __GNUC__ <= 3 /* These are the GCC varidaic macros, so that older versions of GCC don't @@ -248,6 +254,16 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, args, ##__VA_ARGS__) #endif /* defined(__GNUC__) && __GNUC__ <= 3 */ +/** This defines log levels that are linked in the Rust log module, rather + * than re-defining these in both Rust and C. + * + * C_RUST_COUPLED src/rust/tor_log LogSeverity, LogDomain + */ +extern const int LOG_WARN_; +extern const int LOG_NOTICE_; +extern const log_domain_mask_t LD_NET_; +extern const log_domain_mask_t LD_GENERAL_; + #ifdef LOG_PRIVATE MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, const char *funcname, const char *suffix, const char *format, diff --git a/src/common/tortls.c b/src/common/tortls.c index 5f1930d884..b55511569b 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -25,6 +25,9 @@ #include <ws2tcpip.h> #endif +#include "crypto.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "compat.h" /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in @@ -32,7 +35,6 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> -#include "crypto.h" #ifdef OPENSSL_NO_EC #error "We require OpenSSL with ECC support" @@ -56,10 +58,25 @@ ENABLE_GCC_WARNING(redundant-decls) #include "container.h" #include <string.h> +#ifdef OPENSSL_1_1_API +#define X509_get_notBefore_const(cert) \ + X509_get0_notBefore(cert) +#define X509_get_notAfter_const(cert) \ + X509_get0_notAfter(cert) +#ifndef X509_get_notBefore +#define X509_get_notBefore(cert) \ + X509_getm_notBefore(cert) +#endif +#ifndef X509_get_notAfter +#define X509_get_notAfter(cert) \ + X509_getm_notAfter(cert) +#endif +#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert)) +#endif /* Copied from or.h */ #define LEGAL_NICKNAME_CHARACTERS \ @@ -358,8 +375,12 @@ tor_tls_init(void) check_no_tls_errors(); if (!tls_library_is_initialized) { +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL); +#else SSL_library_init(); SSL_load_error_strings(); +#endif #if (SIZEOF_VOID_P >= 8 && \ OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1)) diff --git a/src/common/tortls.h b/src/common/tortls.h index 1dbf0b332f..7c867bfff2 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -11,7 +11,7 @@ * \brief Headers for tortls.c **/ -#include "crypto.h" +#include "crypto_rsa.h" #include "compat_openssl.h" #include "compat.h" #include "testsupport.h" diff --git a/src/common/util.c b/src/common/util.c index a68fd30d09..dece5877f1 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -16,7 +16,7 @@ #define UTIL_PRIVATE #include "util.h" #include "torlog.h" -#include "crypto.h" +#include "crypto_digest.h" #include "torint.h" #include "container.h" #include "address.h" @@ -572,6 +572,19 @@ add_laplace_noise(int64_t signal_, double random_, double delta_f, return signal_ + noise; } +/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather + * than overflow */ +uint32_t +tor_add_u32_nowrap(uint32_t a, uint32_t b) +{ + /* a+b > UINT32_MAX check, without overflow */ + if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) { + return UINT32_MAX; + } else { + return a+b; + } +} + /* Helper: return greatest common divisor of a,b */ static uint64_t gcd64(uint64_t a, uint64_t b) @@ -1821,6 +1834,15 @@ format_iso_time(char *buf, time_t t) strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm)); } +/** As format_local_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid + * embedding an internal space. */ +void +format_local_iso_time_nospace(char *buf, time_t t) +{ + format_local_iso_time(buf, t); + buf[10] = 'T'; +} + /** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid * embedding an internal space. */ void @@ -4779,8 +4801,8 @@ process_environment_t * process_environment_make(struct smartlist_t *env_vars) { process_environment_t *env = tor_malloc_zero(sizeof(process_environment_t)); - size_t n_env_vars = smartlist_len(env_vars); - size_t i; + int n_env_vars = smartlist_len(env_vars); + int i; size_t total_env_length; smartlist_t *env_vars_sorted; @@ -5111,30 +5133,6 @@ stream_status_to_string(enum stream_status stream_status) } } -/* DOCDOC */ -static void -log_portfw_spawn_error_message(const char *buf, - const char *executable, int *child_status) -{ - /* Parse error message */ - int retval, child_state, saved_errno; - retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Failed to start child process \"%s\" in state %d: %s", - executable, child_state, strerror(saved_errno)); - if (child_status) - *child_status = 1; - } else { - /* Failed to parse message from child process, log it as a - warning */ - log_warn(LD_GENERAL, - "Unexpected message from port forwarding helper \"%s\": %s", - executable, buf); - } -} - #ifdef _WIN32 /** Return a smartlist containing lines outputted from @@ -5180,51 +5178,6 @@ tor_get_lines_from_handle, (HANDLE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. - * Returns -1 if there is a error reading, and 0 otherwise. - * If the generated stream is flushed more often than on new lines, or - * a read exceeds 256 bytes, lines will be truncated. This should be fixed, - * along with the corresponding problem on *nix (see bug #2045). - */ -static int -log_from_handle(HANDLE *pipe, int severity) -{ - char buf[256]; - int pos; - smartlist_t *lines; - - pos = tor_read_all_handle(pipe, buf, sizeof(buf) - 1, NULL); - if (pos < 0) { - /* Error */ - log_warn(LD_GENERAL, "Failed to read data from subprocess"); - return -1; - } - - if (0 == pos) { - /* There's nothing to read (process is busy or has exited) */ - log_debug(LD_GENERAL, "Subprocess had nothing to say"); - return 0; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - buf[pos] = '\0'; - log_debug(LD_GENERAL, "Subprocess had %d bytes to say", pos); - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, buf, pos); - - /* Log each line */ - SMARTLIST_FOREACH(lines, char *, line, - { - log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", line); - }); - smartlist_free(lines); - - return 0; -} - #else /* !(defined(_WIN32)) */ /** Return a smartlist containing lines outputted from @@ -5254,42 +5207,6 @@ tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) return lines; } -/** Read from fd, and send lines to log at the specified log level. - * Returns 1 if stream is closed normally, -1 if there is a error reading, and - * 0 otherwise. Handles lines from tor-fw-helper and - * tor_spawn_background() specially. - */ -static int -log_from_pipe(int fd, int severity, const char *executable, - int *child_status) -{ - char buf[256]; - enum stream_status r; - - for (;;) { - r = get_string_from_pipe(fd, buf, sizeof(buf) - 1); - - if (r == IO_STREAM_CLOSED) { - return 1; - } else if (r == IO_STREAM_EAGAIN) { - return 0; - } else if (r == IO_STREAM_TERM) { - return -1; - } - - tor_assert(r == IO_STREAM_OKAY); - - /* Check if buf starts with SPAWN_ERROR_MESSAGE */ - if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) { - log_portfw_spawn_error_message(buf, executable, child_status); - } else { - log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf); - } - } - - /* We should never get here */ - return -1; -} #endif /* defined(_WIN32) */ /** Reads from <b>fd</b> and stores input in <b>buf_out</b> making @@ -5332,294 +5249,6 @@ get_string_from_pipe(int fd, char *buf_out, size_t count) return IO_STREAM_OKAY; } -/** Parse a <b>line</b> from tor-fw-helper and issue an appropriate - * log message to our user. */ -static void -handle_fw_helper_line(const char *executable, const char *line) -{ - smartlist_t *tokens = smartlist_new(); - char *message = NULL; - char *message_for_log = NULL; - const char *external_port = NULL; - const char *internal_port = NULL; - const char *result = NULL; - int port = 0; - int success = 0; - - if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) { - /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's - * possible that it got sent after we tried to read it in log_from_pipe. - * - * XXX Ideally, we should be using one of stdout/stderr for the real - * output, and one for the output of the startup code. We used to do that - * before cd05f35d2c. - */ - int child_status; - log_portfw_spawn_error_message(line, executable, &child_status); - goto done; - } - - smartlist_split_string(tokens, line, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - - if (smartlist_len(tokens) < 5) - goto err; - - if (strcmp(smartlist_get(tokens, 0), "tor-fw-helper") || - strcmp(smartlist_get(tokens, 1), "tcp-forward")) - goto err; - - external_port = smartlist_get(tokens, 2); - internal_port = smartlist_get(tokens, 3); - result = smartlist_get(tokens, 4); - - if (smartlist_len(tokens) > 5) { - /* If there are more than 5 tokens, they are part of [<message>]. - Let's use a second smartlist to form the whole message; - strncat loops suck. */ - int i; - int message_words_n = smartlist_len(tokens) - 5; - smartlist_t *message_sl = smartlist_new(); - for (i = 0; i < message_words_n; i++) - smartlist_add(message_sl, smartlist_get(tokens, 5+i)); - - tor_assert(smartlist_len(message_sl) > 0); - message = smartlist_join_strings(message_sl, " ", 0, NULL); - - /* wrap the message in log-friendly wrapping */ - tor_asprintf(&message_for_log, " ('%s')", message); - - smartlist_free(message_sl); - } - - port = atoi(external_port); - if (port < 1 || port > 65535) - goto err; - - port = atoi(internal_port); - if (port < 1 || port > 65535) - goto err; - - if (!strcmp(result, "SUCCESS")) - success = 1; - else if (!strcmp(result, "FAIL")) - success = 0; - else - goto err; - - if (!success) { - log_warn(LD_GENERAL, "Tor was unable to forward TCP port '%s' to '%s'%s. " - "Please make sure that your router supports port " - "forwarding protocols (like NAT-PMP). Note that if '%s' is " - "your ORPort, your relay will be unable to receive inbound " - "traffic.", external_port, internal_port, - message_for_log ? message_for_log : "", - internal_port); - } else { - log_info(LD_GENERAL, - "Tor successfully forwarded TCP port '%s' to '%s'%s.", - external_port, internal_port, - message_for_log ? message_for_log : ""); - } - - goto done; - - err: - log_warn(LD_GENERAL, "tor-fw-helper sent us a string we could not " - "parse (%s).", line); - - done: - SMARTLIST_FOREACH(tokens, char *, cp, tor_free(cp)); - smartlist_free(tokens); - tor_free(message); - tor_free(message_for_log); -} - -/** Read what tor-fw-helper has to say in its stdout and handle it - * appropriately */ -static int -handle_fw_helper_output(const char *executable, - process_handle_t *process_handle) -{ - smartlist_t *fw_helper_output = NULL; - enum stream_status stream_status = 0; - - fw_helper_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(process_handle), - &stream_status); - if (!fw_helper_output) { /* didn't get any output from tor-fw-helper */ - /* if EAGAIN we should retry in the future */ - return (stream_status == IO_STREAM_EAGAIN) ? 0 : -1; - } - - /* Handle the lines we got: */ - SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) { - handle_fw_helper_line(executable, line); - tor_free(line); - } SMARTLIST_FOREACH_END(line); - - smartlist_free(fw_helper_output); - - return 0; -} - -/** Spawn tor-fw-helper and ask it to forward the ports in - * <b>ports_to_forward</b>. <b>ports_to_forward</b> contains strings - * of the form "<external port>:<internal port>", which is the format - * that tor-fw-helper expects. */ -void -tor_check_port_forwarding(const char *filename, - smartlist_t *ports_to_forward, - time_t now) -{ -/* When fw-helper succeeds, how long do we wait until running it again */ -#define TIME_TO_EXEC_FWHELPER_SUCCESS 300 -/* When fw-helper failed to start, how long do we wait until running it again - */ -#define TIME_TO_EXEC_FWHELPER_FAIL 60 - - /* Static variables are initialized to zero, so child_handle.status=0 - * which corresponds to it not running on startup */ - static process_handle_t *child_handle=NULL; - - static time_t time_to_run_helper = 0; - int stderr_status, retval; - int stdout_status = 0; - - tor_assert(filename); - - /* Start the child, if it is not already running */ - if ((!child_handle || child_handle->status != PROCESS_STATUS_RUNNING) && - time_to_run_helper < now) { - /*tor-fw-helper cli looks like this: tor_fw_helper -p :5555 -p 4555:1111 */ - const char **argv; /* cli arguments */ - int args_n, status; - int argv_index = 0; /* index inside 'argv' */ - - tor_assert(smartlist_len(ports_to_forward) > 0); - - /* check for overflow during 'argv' allocation: - (len(ports_to_forward)*2 + 2)*sizeof(char*) > SIZE_MAX == - len(ports_to_forward) > (((SIZE_MAX/sizeof(char*)) - 2)/2) */ - if ((size_t) smartlist_len(ports_to_forward) > - (((SIZE_MAX/sizeof(char*)) - 2)/2)) { - log_warn(LD_GENERAL, - "Overflow during argv allocation. This shouldn't happen."); - return; - } - /* check for overflow during 'argv_index' increase: - ((len(ports_to_forward)*2 + 2) > INT_MAX) == - len(ports_to_forward) > (INT_MAX - 2)/2 */ - if (smartlist_len(ports_to_forward) > (INT_MAX - 2)/2) { - log_warn(LD_GENERAL, - "Overflow during argv_index increase. This shouldn't happen."); - return; - } - - /* Calculate number of cli arguments: one for the filename, two - for each smartlist element (one for "-p" and one for the - ports), and one for the final NULL. */ - args_n = 1 + 2*smartlist_len(ports_to_forward) + 1; - argv = tor_calloc(args_n, sizeof(char *)); - - argv[argv_index++] = filename; - SMARTLIST_FOREACH_BEGIN(ports_to_forward, const char *, port) { - argv[argv_index++] = "-p"; - argv[argv_index++] = port; - } SMARTLIST_FOREACH_END(port); - argv[argv_index] = NULL; - - /* Assume tor-fw-helper will succeed, start it later*/ - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS; - - if (child_handle) { - tor_process_handle_destroy(child_handle, 1); - child_handle = NULL; - } - -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - status = tor_spawn_background(NULL, argv, NULL, &child_handle); -#else - status = tor_spawn_background(filename, argv, NULL, &child_handle); -#endif /* defined(_WIN32) */ - - tor_free_((void*)argv); - argv=NULL; - - if (PROCESS_STATUS_ERROR == status) { - log_warn(LD_GENERAL, "Failed to start port forwarding helper %s", - filename); - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; - return; - } - - log_info(LD_GENERAL, - "Started port forwarding helper (%s) with pid '%d'", - filename, tor_process_get_pid(child_handle)); - } - - /* If child is running, read from its stdout and stderr) */ - if (child_handle && PROCESS_STATUS_RUNNING == child_handle->status) { - /* Read from stdout/stderr and log result */ - retval = 0; -#ifdef _WIN32 - stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO); -#else - stderr_status = log_from_pipe(child_handle->stderr_pipe, - LOG_INFO, filename, &retval); -#endif /* defined(_WIN32) */ - if (handle_fw_helper_output(filename, child_handle) < 0) { - log_warn(LD_GENERAL, "Failed to handle fw helper output."); - stdout_status = -1; - retval = -1; - } - - if (retval) { - /* There was a problem in the child process */ - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; - } - - /* Combine the two statuses in order of severity */ - if (-1 == stdout_status || -1 == stderr_status) - /* There was a failure */ - retval = -1; -#ifdef _WIN32 - else if (!child_handle || tor_get_exit_code(child_handle, 0, NULL) != - PROCESS_EXIT_RUNNING) { - /* process has exited or there was an error */ - /* TODO: Do something with the process return value */ - /* TODO: What if the process output something since - * between log_from_handle and tor_get_exit_code? */ - retval = 1; - } -#else /* !(defined(_WIN32)) */ - else if (1 == stdout_status || 1 == stderr_status) - /* stdout or stderr was closed, the process probably - * exited. It will be reaped by waitpid() in main.c */ - /* TODO: Do something with the process return value */ - retval = 1; -#endif /* defined(_WIN32) */ - else - /* Both are fine */ - retval = 0; - - /* If either pipe indicates a failure, act on it */ - if (0 != retval) { - if (1 == retval) { - log_info(LD_GENERAL, "Port forwarding helper terminated"); - child_handle->status = PROCESS_STATUS_NOTRUNNING; - } else { - log_warn(LD_GENERAL, "Failed to read from port forwarding helper"); - child_handle->status = PROCESS_STATUS_ERROR; - } - - /* TODO: The child might not actually be finished (maybe it failed or - closed stdout/stderr), so maybe we shouldn't start another? */ - } - } -} - /** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */ void tor_init_weak_random(tor_weak_rng_t *rng, unsigned seed) diff --git a/src/common/util.h b/src/common/util.h index 9380789128..7172b7da08 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -73,9 +73,9 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, } \ STMT_END #else /* !(defined(USE_DMALLOC)) */ -/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc. - * Unlike the free() function, tor_free() will still work on NULL pointers, - * and it sets the pointer value to NULL after freeing it. +/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, + * etc. Unlike the free() function, the tor_free() macro sets the + * pointer value to NULL after freeing it. * * This is a macro. If you need a function pointer to release memory from * tor_malloc(), use tor_free_(). @@ -88,17 +88,13 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, #ifdef __GNUC__ #define tor_free(p) STMT_BEGIN \ typeof(&(p)) tor_free__tmpvar = &(p); \ - if (PREDICT_LIKELY((*tor_free__tmpvar)!=NULL)) { \ - raw_free(*tor_free__tmpvar); \ - *tor_free__tmpvar=NULL; \ - } \ + raw_free(*tor_free__tmpvar); \ + *tor_free__tmpvar=NULL; \ STMT_END #else #define tor_free(p) STMT_BEGIN \ - if (PREDICT_LIKELY((p)!=NULL)) { \ - raw_free(p); \ - (p)=NULL; \ - } \ + raw_free(p); \ + (p)=NULL; \ STMT_END #endif #endif /* defined(USE_DMALLOC) */ @@ -180,6 +176,8 @@ int n_bits_set_u8(uint8_t v); int64_t clamp_double_to_int64(double number); void simplify_fraction64(uint64_t *numer, uint64_t *denom); +uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b); + /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> * and positive <b>b</b>. Works on integer types only. Not defined if a+(b-1) * can overflow. */ @@ -273,6 +271,7 @@ int parse_rfc1123_time(const char *buf, time_t *t); #define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7) void format_local_iso_time(char *buf, time_t t); void format_iso_time(char *buf, time_t t); +void format_local_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace_usec(char *buf, const struct timeval *tv); int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace); @@ -418,11 +417,6 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); int write_pidfile(const char *filename); -/* Port forwarding */ -void tor_check_port_forwarding(const char *filename, - struct smartlist_t *ports_to_forward, - time_t now); - void tor_disable_spawning_background_processes(void); typedef struct process_handle_t process_handle_t; @@ -461,9 +455,7 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, void (*free_old)(void*), int free_p); -/* Values of process_handle_t.status. PROCESS_STATUS_NOTRUNNING must be - * 0 because tor_check_port_forwarding depends on this being the initial - * statue of the static instance of process_handle_t */ +/* Values of process_handle_t.status. */ #define PROCESS_STATUS_NOTRUNNING 0 #define PROCESS_STATUS_RUNNING 1 #define PROCESS_STATUS_ERROR -1 diff --git a/src/common/workqueue.c b/src/common/workqueue.c index ec96959b7d..563a98af96 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -1,3 +1,4 @@ + /* copyright (c) 2013-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -24,13 +25,16 @@ #include "orconfig.h" #include "compat.h" +#include "compat_libevent.h" #include "compat_threads.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "workqueue.h" #include "tor_queue.h" #include "torlog.h" +#include <event2/event.h> + #define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH #define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW #define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1) @@ -63,6 +67,9 @@ struct threadpool_s { void (*free_update_arg_fn)(void *); /** Array of n_threads update arguments. */ void **update_args; + /** Event to notice when another thread has sent a reply. */ + struct event *reply_event; + void (*reply_cb)(threadpool_t *); /** Number of elements in threads. */ int n_threads; @@ -597,15 +604,41 @@ replyqueue_new(uint32_t alertsocks_flags) return rq; } -/** - * Return the "read socket" for a given reply queue. The main thread should - * listen for read events on this socket, and call replyqueue_process() every - * time it triggers. +/** Internal: Run from the libevent mainloop when there is work to handle in + * the reply queue handler. */ +static void +reply_event_cb(evutil_socket_t sock, short events, void *arg) +{ + threadpool_t *tp = arg; + (void) sock; + (void) events; + replyqueue_process(tp->reply_queue); + if (tp->reply_cb) + tp->reply_cb(tp); +} + +/** Register the threadpool <b>tp</b>'s reply queue with the libevent + * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after + * each time there is work to process from the reply queue. Return 0 on + * success, -1 on failure. */ -tor_socket_t -replyqueue_get_socket(replyqueue_t *rq) +int +threadpool_register_reply_event(threadpool_t *tp, + void (*cb)(threadpool_t *tp)) { - return rq->alert.read_fd; + struct event_base *base = tor_libevent_get_base(); + + if (tp->reply_event) { + tor_event_free(tp->reply_event); + } + tp->reply_event = tor_event_new(base, + tp->reply_queue->alert.read_fd, + EV_READ|EV_PERSIST, + reply_event_cb, + tp); + tor_assert(tp->reply_event); + tp->reply_cb = cb; + return event_add(tp->reply_event, NULL); } /** diff --git a/src/common/workqueue.h b/src/common/workqueue.h index eb885e680d..e1fe612e2b 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -56,8 +56,11 @@ threadpool_t *threadpool_new(int n_threads, replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); -tor_socket_t replyqueue_get_socket(replyqueue_t *rq); void replyqueue_process(replyqueue_t *queue); +struct event_base; +int threadpool_register_reply_event(threadpool_t *tp, + void (*cb)(threadpool_t *tp)); + #endif /* !defined(TOR_WORKQUEUE_H) */ diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 90f91e5cb9..86429f1176 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -100,7 +100,7 @@ ## A handle for your relay, so people don't have to refer to it by key. ## Nicknames must be between 1 and 19 characters inclusive, and must ## contain only the alphanumeric characters (a-z, A-Z, 0-9). No unicode, -## no emoji. +## no emoji. If not set, "Unnamed" will be used. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 4e183478eb..72cca0be31 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -106,6 +106,7 @@ ## A handle for your relay, so people don't have to refer to it by key. ## Nicknames must be between 1 and 19 characters inclusive, and must ## contain only the characters [a-zA-Z0-9]. +## If not set, "Unnamed" will be used. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/ext/ed25519/donna/ed25519-hash-custom.h b/src/ext/ed25519/donna/ed25519-hash-custom.h index 609451abd5..cdeab3e45b 100644 --- a/src/ext/ed25519/donna/ed25519-hash-custom.h +++ b/src/ext/ed25519/donna/ed25519-hash-custom.h @@ -9,7 +9,7 @@ void ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen); */ -#include "crypto.h" +#include "crypto_digest.h" typedef struct ed25519_hash_context { crypto_digest_t *ctx; diff --git a/src/ext/ed25519/donna/ed25519-randombytes-custom.h b/src/ext/ed25519/donna/ed25519-randombytes-custom.h index 3fb0959fc4..27eade4f95 100644 --- a/src/ext/ed25519/donna/ed25519-randombytes-custom.h +++ b/src/ext/ed25519/donna/ed25519-randombytes-custom.h @@ -8,7 +8,7 @@ */ /* Tor: Instead of calling OpenSSL's CSPRNG directly, call the wrapper. */ -#include "crypto.h" +#include "crypto_rand.h" static void ED25519_FN(ed25519_randombytes_unsafe) (void *p, size_t len) diff --git a/src/ext/ed25519/donna/ed25519_tor.c b/src/ext/ed25519/donna/ed25519_tor.c index 84fc3850a2..43de9faaea 100644 --- a/src/ext/ed25519/donna/ed25519_tor.c +++ b/src/ext/ed25519/donna/ed25519_tor.c @@ -40,6 +40,8 @@ #include "ed25519-randombytes.h" #include "ed25519-hash.h" +#include "crypto_util.h" + typedef unsigned char ed25519_signature[64]; typedef unsigned char ed25519_public_key[32]; typedef unsigned char ed25519_secret_key[32]; diff --git a/src/ext/ed25519/ref10/blinding.c b/src/ext/ed25519/ref10/blinding.c index a3b32fa80c..88e84cac20 100644 --- a/src/ext/ed25519/ref10/blinding.c +++ b/src/ext/ed25519/ref10/blinding.c @@ -7,7 +7,7 @@ #include "ed25519_ref10.h" #include <string.h> -#include "crypto.h" +#include "crypto_util.h" static void ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param) diff --git a/src/ext/ed25519/ref10/crypto_hash_sha512.h b/src/ext/ed25519/ref10/crypto_hash_sha512.h index 5dad935c79..7faddb1597 100644 --- a/src/ext/ed25519/ref10/crypto_hash_sha512.h +++ b/src/ext/ed25519/ref10/crypto_hash_sha512.h @@ -1,5 +1,5 @@ /* Added for Tor. */ -#include "crypto.h" +#include "crypto_digest.h" /* Set 'out' to the 512-bit SHA512 hash of the 'len'-byte string in 'inp' */ #define crypto_hash_sha512(out, inp, len) \ diff --git a/src/ext/ed25519/ref10/keypair.c b/src/ext/ed25519/ref10/keypair.c index 68a88f9adc..c437f0a4f2 100644 --- a/src/ext/ed25519/ref10/keypair.c +++ b/src/ext/ed25519/ref10/keypair.c @@ -6,6 +6,9 @@ #include "crypto_hash_sha512.h" #include "ge.h" +#include "crypto_rand.h" +#include "crypto_util.h" + int crypto_sign_seckey(unsigned char *sk) { diff --git a/src/ext/ed25519/ref10/randombytes.h b/src/ext/ed25519/ref10/randombytes.h index 8bf31631f0..a21dde8540 100644 --- a/src/ext/ed25519/ref10/randombytes.h +++ b/src/ext/ed25519/ref10/randombytes.h @@ -1,4 +1,4 @@ /* Added for Tor. */ -#include "crypto.h" +#include "crypto_rand.h" #define randombytes(b, n) \ (crypto_strongest_rand((b), (n)), 0) diff --git a/src/ext/keccak-tiny/keccak-tiny-unrolled.c b/src/ext/keccak-tiny/keccak-tiny-unrolled.c index d8d7fe335a..07e8c95bcf 100644 --- a/src/ext/keccak-tiny/keccak-tiny-unrolled.c +++ b/src/ext/keccak-tiny/keccak-tiny-unrolled.c @@ -9,7 +9,7 @@ #include "keccak-tiny.h" #include <string.h> -#include "crypto.h" +#include "crypto_util.h" #include "byteorder.h" /******** Endianness conversion helpers ********/ diff --git a/src/ext/rust b/src/ext/rust -Subproject fbc0c25785696a25b9cbc09ed645cc8d404ee0f +Subproject aa37fb84fb829902e83ca11a7244bbc6b86b809 diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index 713ec219ce..d4b514d2c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -150,7 +150,7 @@ #else #define ctz(n) ctz32(n) #define clz(n) clz32(n) -#define fls(n) ((int)(32 - clz32(n))) +#define fls(n) ((int)(32 - clz32((uint32_t)n))) #endif #if WHEEL_BIT == 6 @@ -432,7 +432,7 @@ TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) { * or can be replaced with a simpler operation. */ oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT)); - pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot); + pending = rotl(((WHEEL_C(1) << _elapsed) - 1), oslot); nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT)); pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed); diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 96ce275578..7f861e4d24 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -21,9 +21,10 @@ #include "config.h" #include "connection_edge.h" #include "control.h" +#include "crypto_rand.h" #include "dns.h" -#include "routerset.h" #include "nodelist.h" +#include "routerset.h" /** A client-side struct to remember requests to rewrite addresses * to new addresses. These structs are stored in the hash table @@ -959,9 +960,11 @@ addressmap_get_virtual_address(int type) char tmp[TOR_ADDR_BUF_LEN]; tor_addr_to_str(tmp, &addr, sizeof(tmp), 0); if (strmap_get(addressmap, tmp)) { + // LCOV_EXCL_START log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.", buf, tmp); continue; + // LCOV_EXCL_STOP } return tor_strdup(buf); @@ -970,8 +973,10 @@ addressmap_get_virtual_address(int type) log_warn(LD_CONFIG, "Ran out of virtual addresses!"); return NULL; } else { + // LCOV_EXCL_START log_warn(LD_BUG, "Called with unsupported address type (%d)", type); return NULL; + // LCOV_EXCL_STOP } } diff --git a/src/or/bridges.c b/src/or/bridges.c index 29d00f37ba..699e030e6c 100644 --- a/src/or/bridges.c +++ b/src/or/bridges.c @@ -11,6 +11,8 @@ * Bridges are fixed entry nodes, used for censorship circumvention. **/ +#define TOR_BRIDGES_PRIVATE + #include "or.h" #include "bridges.h" #include "circuitbuild.h" @@ -93,7 +95,7 @@ sweep_bridge_list(void) } /** Initialize the bridge list to empty, creating it if needed. */ -static void +STATIC void clear_bridge_list(void) { if (!bridge_list) @@ -156,7 +158,7 @@ bridge_get_addr_port(const bridge_info_t *bridge) * bridge with no known digest whose address matches any of the * tor_addr_port_t's in <b>orports</b>, return that bridge. Else return * NULL. */ -static bridge_info_t * +STATIC bridge_info_t * get_configured_bridge_by_orports_digest(const char *digest, const smartlist_t *orports) { @@ -350,7 +352,7 @@ bridge_has_digest(const bridge_info_t *bridge, const char *digest) * existing bridge with the same address and port, and warn the user as * appropriate. */ -static void +STATIC void bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port, const char *digest, const char *transport_name) { @@ -471,7 +473,7 @@ bridge_add_from_config(bridge_line_t *bridge_line) } /** If <b>digest</b> is one of our known bridges, return it. */ -bridge_info_t * +STATIC bridge_info_t * find_bridge_by_digest(const char *digest) { if (! bridge_list) diff --git a/src/or/bridges.h b/src/or/bridges.h index 54a6250259..3108eb555d 100644 --- a/src/or/bridges.h +++ b/src/or/bridges.h @@ -20,7 +20,6 @@ typedef struct bridge_info_t bridge_info_t; void mark_bridge_list(void); void sweep_bridge_list(void); const smartlist_t *bridge_list_get(void); -bridge_info_t *find_bridge_by_digest(const char *digest); const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge); const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge); bridge_info_t *get_configured_bridge_by_addr_port_digest( @@ -65,5 +64,17 @@ MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id, void bridges_free_all(void); +#ifdef TOR_BRIDGES_PRIVATE +STATIC void clear_bridge_list(void); +STATIC bridge_info_t *find_bridge_by_digest(const char *digest); +STATIC bridge_info_t *get_configured_bridge_by_orports_digest( + const char *digest, + const smartlist_t *orports); +STATIC void bridge_resolve_conflicts(const tor_addr_t *addr, + uint16_t port, + const char *digest, + const char *transport_name); +#endif /* defined(TOR_BRIDGES_PRIVATE) */ + #endif /* !defined(TOR_BRIDGES_H) */ diff --git a/src/or/channel.c b/src/or/channel.c index a4740dd752..c30e508018 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "entrynodes.h" #include "geoip.h" +#include "main.h" #include "nodelist.h" #include "relay.h" #include "rephist.h" @@ -404,6 +405,7 @@ channel_register(channel_t *chan) /* Put it in the finished list, creating it if necessary */ if (!finished_channels) finished_channels = smartlist_new(); smartlist_add(finished_channels, chan); + mainloop_schedule_postloop_cleanup(); } else { /* Put it in the active list, creating it if necessary */ if (!active_channels) active_channels = smartlist_new(); @@ -1548,6 +1550,7 @@ channel_change_state_(channel_t *chan, channel_state_t to_state) if (active_channels) smartlist_remove(active_channels, chan); if (!finished_channels) finished_channels = smartlist_new(); smartlist_add(finished_channels, chan); + mainloop_schedule_postloop_cleanup(); } /* Need to put on active list? */ else if (!was_active && is_active) { @@ -1666,6 +1669,7 @@ channel_listener_change_state(channel_listener_t *chan_l, if (active_listeners) smartlist_remove(active_listeners, chan_l); if (!finished_listeners) finished_listeners = smartlist_new(); smartlist_add(finished_listeners, chan_l); + mainloop_schedule_postloop_cleanup(); } /* Need to put on active list? */ else if (!was_active && is_active) { @@ -2109,21 +2113,6 @@ channel_listener_dumpstats(int severity) } /** - * Set the cmux policy on all active channels. - */ -void -channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol) -{ - if (!active_channels) return; - - SMARTLIST_FOREACH_BEGIN(active_channels, channel_t *, curr) { - if (curr->cmux) { - circuitmux_set_policy(curr->cmux, pol); - } - } SMARTLIST_FOREACH_END(curr); -} - -/** * Clean up channels. * * This gets called periodically from run_scheduled_events() in main.c; @@ -2402,7 +2391,7 @@ channel_get_for_extend(const char *rsa_id_digest, { channel_t *chan, *best = NULL; int n_inprogress_goodaddr = 0, n_old = 0; - int n_noncanonical = 0, n_possible = 0; + int n_noncanonical = 0; tor_assert(msg_out); tor_assert(launch_out); @@ -2465,8 +2454,6 @@ channel_get_for_extend(const char *rsa_id_digest, continue; } - ++n_possible; - if (!best) { best = chan; /* If we have no 'best' so far, this one is good enough. */ continue; diff --git a/src/or/channel.h b/src/or/channel.h index 0af5aed414..6cf8cd7f72 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -422,9 +422,6 @@ void channel_free_all(void); void channel_dumpstats(int severity); void channel_listener_dumpstats(int severity); -/* Set the cmux policy on all active channels */ -void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol); - #ifdef TOR_CHANNEL_INTERNAL_ #ifdef CHANNEL_PRIVATE_ diff --git a/src/or/channelpadding.c b/src/or/channelpadding.c index ba432e716b..3dbdd019ab 100644 --- a/src/or/channelpadding.c +++ b/src/or/channelpadding.c @@ -16,11 +16,11 @@ #include "networkstatus.h" #include "connection.h" #include "connection_or.h" +#include "crypto_rand.h" #include "main.h" #include "rephist.h" #include "router.h" #include "compat_time.h" -#include <event2/event.h> #include "rendservice.h" STATIC int32_t channelpadding_get_netflow_inactive_timeout_ms( diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 9000703b01..54d94f6109 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -160,9 +160,8 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->write_var_cell = channel_tls_write_var_cell_method; chan->cmux = circuitmux_alloc(); - if (cell_ewma_enabled()) { - circuitmux_set_policy(chan->cmux, &ewma_policy); - } + /* We only have one policy for now so always set it to EWMA. */ + circuitmux_set_policy(chan->cmux, &ewma_policy); } /** diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index c1c1ca31be..3cdd16261a 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -30,6 +30,7 @@ #include "circuitstats.h" #include "connection_edge.h" #include "config.h" +#include "crypto_rand.h" #include "entrynodes.h" #include "networkstatus.h" #include "relay.h" @@ -892,6 +893,7 @@ pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) /* Check nonce */ if (ipv4_host == ocirc->pathbias_probe_nonce) { pathbias_mark_use_success(ocirc); + circuit_read_valid_data(ocirc, rh.length); circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); log_info(LD_CIRC, "Got valid path bias probe back for circ %d, stream %d.", @@ -913,6 +915,68 @@ pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) } /** + * Check if a cell is counts as valid data for a circuit, + * and if so, count it as valid. + */ +void +pathbias_count_valid_cells(circuit_t *circ, const cell_t *cell) +{ + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + relay_header_t rh; + + relay_header_unpack(&rh, cell->payload); + + /* Check to see if this is a cell from a previous connection, + * or is a request to close the circuit. */ + switch (rh.command) { + case RELAY_COMMAND_TRUNCATED: + /* Truncated cells can arrive on path bias circs. When they do, + * just process them. This closes the circ, but it was junk anyway. + * No reason to wait for the probe. */ + circuit_read_valid_data(ocirc, rh.length); + circuit_truncated(TO_ORIGIN_CIRCUIT(circ), + get_uint8(cell->payload + RELAY_HEADER_SIZE)); + + break; + + case RELAY_COMMAND_END: + if (connection_half_edge_is_valid_end(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } + break; + + case RELAY_COMMAND_DATA: + if (connection_half_edge_is_valid_data(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } + break; + + case RELAY_COMMAND_SENDME: + if (connection_half_edge_is_valid_sendme(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } + break; + + case RELAY_COMMAND_CONNECTED: + if (connection_half_edge_is_valid_connected(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } + break; + + case RELAY_COMMAND_RESOLVED: + if (connection_half_edge_is_valid_resolved(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } + break; + } +} + +/** * Check if a circuit was used and/or closed successfully. * * If we attempted to use the circuit to carry a stream but failed diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h index c9e572d2ae..689b2a6207 100644 --- a/src/or/circpathbias.h +++ b/src/or/circpathbias.h @@ -20,6 +20,7 @@ void pathbias_count_build_success(origin_circuit_t *circ); int pathbias_count_build_attempt(origin_circuit_t *circ); int pathbias_check_close(origin_circuit_t *circ, int reason); int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); +void pathbias_count_valid_cells(circuit_t *circ, const cell_t *cell); void pathbias_count_use_attempt(origin_circuit_t *circ); void pathbias_mark_use_success(origin_circuit_t *circ); void pathbias_mark_use_rollback(origin_circuit_t *circ); diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 75307c367f..8f17f27868 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -43,7 +43,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" -#include "crypto.h" +#include "crypto_rand.h" #include "directory.h" #include "entrynodes.h" #include "hs_ntor.h" @@ -56,6 +56,7 @@ #include "onion_fast.h" #include "policies.h" #include "relay.h" +#include "relay_crypto.h" #include "rendcommon.h" #include "rephist.h" #include "router.h" @@ -71,10 +72,7 @@ static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, static int circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, int relayed); -static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit, - int is_hs_v3_rp_circuit); static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); -static int onion_extend_cpath(origin_circuit_t *circ); STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static int circuit_send_first_onion_skin(origin_circuit_t *circ); static int circuit_build_no_more_hops(origin_circuit_t *circ); @@ -1055,7 +1053,7 @@ circuit_build_no_more_hops(origin_circuit_t *circ) clear_broken_connection_map(1); if (server_mode(options) && !check_whether_orport_reachable(options)) { inform_testing_reachability(); - consider_testing_reachability(1, 1); + router_do_reachability_checks(1, 1); } } @@ -1132,19 +1130,29 @@ circuit_send_intermediate_onion_skin(origin_circuit_t *circ, return 0; } -/** Our clock just jumped by <b>seconds_elapsed</b>. Assume - * something has also gone wrong with our network: notify the user, - * and abandon all not-yet-used circuits. */ +/** Our clock just jumped by <b>seconds_elapsed</b>. If <b>was_idle</b> is + * true, then the monotonic time matches; otherwise it doesn't. Assume + * something has also gone wrong with our network: notify the user, and + * abandon all not-yet-used circuits. */ void -circuit_note_clock_jumped(int seconds_elapsed) +circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " - "assuming established circuits no longer work.", - seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, - seconds_elapsed >=0 ? "forward" : "backward"); - control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", - seconds_elapsed); + if (was_idle) { + tor_log(severity, LD_GENERAL, "Tor has been idle for "I64_FORMAT + " seconds; assuming established circuits no longer work.", + I64_PRINTF_ARG(seconds_elapsed)); + } else { + tor_log(severity, LD_GENERAL, + "Your system clock just jumped "I64_FORMAT" seconds %s; " + "assuming established circuits no longer work.", + I64_PRINTF_ARG( + seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed), + seconds_elapsed >=0 ? "forward" : "backward"); + } + control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME="I64_FORMAT + " IDLE=%d", + I64_PRINTF_ARG(seconds_elapsed), was_idle?1:0); /* so we log when it works again */ note_that_we_maybe_cant_complete_circuits(); control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", @@ -1337,69 +1345,10 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, int reverse, int is_hs_v3) { - crypto_digest_t *tmp_digest; - crypto_cipher_t *tmp_crypto; - size_t digest_len = 0; - size_t cipher_key_len = 0; tor_assert(cpath); - tor_assert(key_data); - tor_assert(!(cpath->f_crypto || cpath->b_crypto || - cpath->f_digest || cpath->b_digest)); - - /* Basic key size validation */ - if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) { - return -1; - } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) { - return -1; - } - - /* If we are using this cpath for next gen onion services use SHA3-256, - otherwise use good ol' SHA1 */ - if (is_hs_v3) { - digest_len = DIGEST256_LEN; - cipher_key_len = CIPHER256_KEY_LEN; - cpath->f_digest = crypto_digest256_new(DIGEST_SHA3_256); - cpath->b_digest = crypto_digest256_new(DIGEST_SHA3_256); - } else { - digest_len = DIGEST_LEN; - cipher_key_len = CIPHER_KEY_LEN; - cpath->f_digest = crypto_digest_new(); - cpath->b_digest = crypto_digest_new(); - } - - tor_assert(digest_len != 0); - tor_assert(cipher_key_len != 0); - const int cipher_key_bits = (int) cipher_key_len * 8; - - crypto_digest_add_bytes(cpath->f_digest, key_data, digest_len); - crypto_digest_add_bytes(cpath->b_digest, key_data+digest_len, digest_len); - - cpath->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len), - cipher_key_bits); - if (!cpath->f_crypto) { - log_warn(LD_BUG,"Forward cipher initialization failed."); - return -1; - } - - cpath->b_crypto = crypto_cipher_new_with_bits( - key_data+(2*digest_len)+cipher_key_len, - cipher_key_bits); - if (!cpath->b_crypto) { - log_warn(LD_BUG,"Backward cipher initialization failed."); - return -1; - } - - if (reverse) { - tmp_digest = cpath->f_digest; - cpath->f_digest = cpath->b_digest; - cpath->b_digest = tmp_digest; - tmp_crypto = cpath->f_crypto; - cpath->f_crypto = cpath->b_crypto; - cpath->b_crypto = tmp_crypto; - } - - return 0; + return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse, + is_hs_v3); } /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>. @@ -1470,13 +1419,12 @@ circuit_finish_handshake(origin_circuit_t *circ, * just give up: force circ to close, and return 0. */ int -circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, int reason) +circuit_truncated(origin_circuit_t *circ, int reason) { // crypt_path_t *victim; // connection_t *stream; tor_assert(circ); - tor_assert(layer); /* XXX Since we don't send truncates currently, getting a truncated * means that a connection broke or an extend failed. For now, @@ -1522,7 +1470,6 @@ onionskin_answer(or_circuit_t *circ, const uint8_t *rend_circ_nonce) { cell_t cell; - crypt_path_t *tmp_cpath; tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN); @@ -1533,25 +1480,15 @@ onionskin_answer(or_circuit_t *circ, } cell.circ_id = circ->p_circ_id; - tmp_cpath = tor_malloc_zero(sizeof(crypt_path_t)); - tmp_cpath->magic = CRYPT_PATH_MAGIC; - circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.", (unsigned int)get_uint32(keys), (unsigned int)get_uint32(keys+20)); - if (circuit_init_cpath_crypto(tmp_cpath, keys, keys_len, 0, 0)<0) { + if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); - tor_free(tmp_cpath); return -1; } - circ->n_digest = tmp_cpath->f_digest; - circ->n_crypto = tmp_cpath->f_crypto; - circ->p_digest = tmp_cpath->b_digest; - circ->p_crypto = tmp_cpath->b_crypto; - tmp_cpath->magic = 0; - tor_free(tmp_cpath); memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN); @@ -1612,7 +1549,7 @@ onionskin_answer(or_circuit_t *circ, * rend_service_launch_establish_intro()) * * - We are a router testing its own reachabiity - * (CIRCUIT_PURPOSE_TESTING, via consider_testing_reachability()) + * (CIRCUIT_PURPOSE_TESTING, via router_do_reachability_checks()) * * onion_pick_cpath_exit() bypasses us (by not calling * new_route_len()) in the one-hop tunnel case, so we don't need to @@ -2353,7 +2290,7 @@ warn_if_last_router_excluded(origin_circuit_t *circ, * be used as an HS v3 rendezvous point. * * Return 0 if ok, -1 if circuit should be closed. */ -static int +STATIC int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, int is_hs_v3_rp_circuit) { @@ -2524,12 +2461,71 @@ cpath_get_n_hops(crypt_path_t **head_ptr) #endif /* defined(TOR_UNIT_TESTS) */ /** + * Build the exclude list for vanguard circuits. + * + * For vanguard circuits we exclude all the already chosen nodes (including the + * exit) from being middle hops to prevent the creation of A - B - A subpaths. + * We also allow the 4th hop to be the same as the guard node so as to not leak + * guard information to RP/IP/HSDirs. + * + * For vanguard circuits, we don't apply any subnet or family restrictions. + * This is to avoid impossible-to-build circuit paths, or just situations where + * our earlier guards prevent us from using most of our later ones. + * + * The alternative is building the circuit in reverse. Reverse calls to + * onion_extend_cpath() (ie: select outer hops first) would then have the + * property that you don't gain information about inner hops by observing + * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487 + * for this. + * + * (Note further that we still exclude the exit to prevent A - B - A + * at the end of the path. */ +static smartlist_t * +build_vanguard_middle_exclude_list(uint8_t purpose, + cpath_build_state_t *state, + crypt_path_t *head, + int cur_len) +{ + smartlist_t *excluded; + const node_t *r; + crypt_path_t *cpath; + int i; + + (void) purpose; + + excluded = smartlist_new(); + + /* Add the exit to the exclude list (note that the exit/last hop is always + * chosen first in circuit_establish_circuit()). */ + if ((r = build_state_get_exit_node(state))) { + smartlist_add(excluded, (node_t*)r); + } + + /* If we are picking the 4th hop, allow that node to be the guard too. + * This prevents us from avoiding the Guard for those hops, which + * gives the adversary information about our guard if they control + * the RP, IP, or HSDIR. We don't do this check based on purpose + * because we also want to allow HS_VANGUARDS pre-build circuits + * to use the guard for that last hop. + */ + if (cur_len == DEFAULT_ROUTE_LEN+1) { + /* Skip the first hop for the exclude list below */ + head = head->next; + cur_len--; + } + + for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) { + if ((r = node_get_by_id(cpath->extend_info->identity_digest))) { + smartlist_add(excluded, (node_t*)r); + } + } + + return excluded; +} + +/** * Build a list of nodes to exclude from the choice of this middle * hop, based on already chosen nodes. - * - * XXX: At present, this function does not exclude any nodes from - * the vanguard circuits. See - * https://trac.torproject.org/projects/tor/ticket/24487 */ static smartlist_t * build_middle_exclude_list(uint8_t purpose, @@ -2542,32 +2538,21 @@ build_middle_exclude_list(uint8_t purpose, crypt_path_t *cpath; int i; + /** Vanguard circuits have their own path selection rules */ + if (circuit_should_use_vanguards(purpose)) { + return build_vanguard_middle_exclude_list(purpose, state, head, cur_len); + } + excluded = smartlist_new(); - /* Add the exit to the exclude list (note that the exit/last hop is always - * chosen first in circuit_establish_circuit()). */ + /* For non-vanguard circuits, add the exit and its family to the exclude list + * (note that the exit/last hop is always chosen first in + * circuit_establish_circuit()). */ if ((r = build_state_get_exit_node(state))) { nodelist_add_node_and_family(excluded, r); } - /* XXX: We don't apply any other previously selected node restrictions for - * vanguards, and allow nodes to be reused for those hop positions in the - * same circuit. This is because after many rotations, you get to learn - * inner guard nodes through the nodes that are not selected for outer - * hops. - * - * The alternative is building the circuit in reverse. Reverse calls to - * onion_extend_cpath() (ie: select outer hops first) would then have the - * property that you don't gain information about inner hops by observing - * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487 - * for this. - * - * (Note further that we can and do still exclude the exit in the block - * above, because it is chosen first in circuit_establish_circuit()..) */ - if (circuit_should_use_vanguards(purpose)) { - return excluded; - } - + /* also exclude all other already chosen nodes and their family */ for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) { if ((r = node_get_by_id(cpath->extend_info->identity_digest))) { nodelist_add_node_and_family(excluded, r); @@ -2667,7 +2652,9 @@ choose_good_middle_server(uint8_t purpose, /** If a hidden service circuit wants a specific middle node, pin it. */ if (middle_node_must_be_vanguard(options, purpose, cur_len)) { log_debug(LD_GENERAL, "Picking a sticky node (cur_len = %d)", cur_len); - return pick_vanguard_middle_node(options, flags, cur_len, excluded); + choice = pick_vanguard_middle_node(options, flags, cur_len, excluded); + smartlist_free(excluded); + return choice; } choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); @@ -2707,7 +2694,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state, /* This request is for an entry server to use for a regular circuit, * and we use entry guard nodes. Just return one of the guard nodes. */ tor_assert(guard_state_out); - return guards_choose_guard(state, guard_state_out); + return guards_choose_guard(state, purpose, guard_state_out); } excluded = smartlist_new(); @@ -2750,7 +2737,7 @@ onion_next_hop_in_cpath(crypt_path_t *cpath) * Return 1 if the path is complete, 0 if we successfully added a hop, * and -1 on error. */ -static int +STATIC int onion_extend_cpath(origin_circuit_t *circ) { uint8_t purpose = circ->base_.purpose; @@ -2872,14 +2859,13 @@ extend_info_from_node(const node_t *node, int for_direct_connect) return NULL; } - /* Choose a preferred address first, but fall back to an allowed address. - * choose_address returns 1 on success, but get_prim_orport returns 0. */ + /* Choose a preferred address first, but fall back to an allowed address. */ if (for_direct_connect) - valid_addr = fascist_firewall_choose_address_node(node, - FIREWALL_OR_CONNECTION, - 0, &ap); - else - valid_addr = !node_get_prim_orport(node, &ap); + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &ap); + else { + node_get_prim_orport(node, &ap); + } + valid_addr = tor_addr_port_is_valid_ap(&ap, 0); if (valid_addr) log_debug(LD_CIRC, "using %s for %s", diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index bea31ad0dd..e3d30c1104 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -29,7 +29,7 @@ void circuit_n_chan_done(channel_t *chan, int status, int inform_testing_reachability(void); int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ); int circuit_send_next_onion_skin(origin_circuit_t *circ); -void circuit_note_clock_jumped(int seconds_elapsed); +void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle); int circuit_extend(cell_t *cell, circuit_t *circ); int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, @@ -37,8 +37,7 @@ int circuit_init_cpath_crypto(crypt_path_t *cpath, struct created_cell_t; int circuit_finish_handshake(origin_circuit_t *circ, const struct created_cell_t *created_cell); -int circuit_truncated(origin_circuit_t *circ, crypt_path_t *layer, - int reason); +int circuit_truncated(origin_circuit_t *circ, int reason); int onionskin_answer(or_circuit_t *circ, const struct created_cell_t *created_cell, const char *keys, size_t keys_len, @@ -83,6 +82,13 @@ STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); + +STATIC int onion_extend_cpath(origin_circuit_t *circ); + +STATIC int +onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, + int is_hs_v3_rp_circuit); + #if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags, const or_options_t *options); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 7bdef0b878..ad9b902ac6 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -65,6 +65,8 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "entrynodes.h" #include "main.h" #include "hs_circuit.h" @@ -76,6 +78,7 @@ #include "onion_fast.h" #include "policies.h" #include "relay.h" +#include "relay_crypto.h" #include "rendclient.h" #include "rendcommon.h" #include "rephist.h" @@ -406,9 +409,6 @@ circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id, circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan); if (chan) { - tor_assert(bool_eq(or_circ->p_chan_cells.n, - or_circ->next_active_on_p_chan)); - chan->timestamp_last_had_circuits = approx_time(); } @@ -431,8 +431,6 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id, circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan); if (chan) { - tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan)); - chan->timestamp_last_had_circuits = approx_time(); } @@ -1043,6 +1041,14 @@ circuit_free_(circuit_t *circ) circuit_remove_from_origin_circuit_list(ocirc); + if (ocirc->half_streams) { + SMARTLIST_FOREACH_BEGIN(ocirc->half_streams, half_edge_t *, + half_conn) { + half_edge_free(half_conn); + } SMARTLIST_FOREACH_END(half_conn); + smartlist_free(ocirc->half_streams); + } + if (ocirc->build_state) { extend_info_free(ocirc->build_state->chosen_exit); circuit_free_cpath_node(ocirc->build_state->pending_final_cpath); @@ -1087,10 +1093,7 @@ circuit_free_(circuit_t *circ) should_free = (ocirc->workqueue_entry == NULL); - crypto_cipher_free(ocirc->p_crypto); - crypto_digest_free(ocirc->p_digest); - crypto_cipher_free(ocirc->n_crypto); - crypto_digest_free(ocirc->n_digest); + relay_crypto_clear(ô->crypto); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -1230,10 +1233,7 @@ circuit_free_cpath_node(crypt_path_t *victim) if (!victim) return; - crypto_cipher_free(victim->f_crypto); - crypto_cipher_free(victim->b_crypto); - crypto_digest_free(victim->f_digest); - crypto_digest_free(victim->b_digest); + relay_crypto_clear(&victim->crypto); onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); @@ -2077,6 +2077,7 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line, circuits_pending_close = smartlist_new(); smartlist_add(circuits_pending_close, circ); + mainloop_schedule_postloop_cleanup(); log_info(LD_GENERAL, "Circuit %u (id: %" PRIu32 ") marked for close at " "%s:%d (orig reason: %d, new reason: %d)", @@ -2323,6 +2324,20 @@ n_cells_in_circ_queues(const circuit_t *c) return n; } +/** Return the number of bytes allocated for <b>c</c>'s half-open streams. */ +static size_t +circuit_alloc_in_half_streams(const circuit_t *c) +{ + if (! CIRCUIT_IS_ORIGIN(c)) { + return 0; + } + const origin_circuit_t *ocirc = CONST_TO_ORIGIN_CIRCUIT(c); + if (ocirc->half_streams) + return smartlist_len(ocirc->half_streams) * sizeof(half_edge_t); + else + return 0; +} + /** * Return the age of the oldest cell queued on <b>c</b>, in timestamp units. * Return 0 if there are no cells queued on c. Requires that <b>now</b> be @@ -2557,6 +2572,7 @@ circuits_handle_oom(size_t current_allocation) /* Now, kill the circuit. */ n = n_cells_in_circ_queues(circ); + const size_t half_stream_alloc = circuit_alloc_in_half_streams(circ); if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } @@ -2566,6 +2582,7 @@ circuits_handle_oom(size_t current_allocation) ++n_circuits_killed; mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += half_stream_alloc; mem_recovered += freed; if (mem_recovered >= mem_to_recover) @@ -2596,8 +2613,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp) switch (cp->state) { case CPATH_STATE_OPEN: - tor_assert(cp->f_crypto); - tor_assert(cp->b_crypto); + relay_crypto_assert_ok(&cp->crypto); /* fall through */ case CPATH_STATE_CLOSED: /*XXXX Assert that there's no handshake_state either. */ @@ -2687,10 +2703,7 @@ assert_circuit_ok,(const circuit_t *c)) c->state == CIRCUIT_STATE_GUARD_WAIT) { tor_assert(!c->n_chan_create_cell); if (or_circ) { - tor_assert(or_circ->n_crypto); - tor_assert(or_circ->p_crypto); - tor_assert(or_circ->n_digest); - tor_assert(or_circ->p_digest); + relay_crypto_assert_ok(&or_circ->crypto); } } if (c->state == CIRCUIT_STATE_CHAN_WAIT && !c->marked_for_close) { @@ -2714,4 +2727,3 @@ assert_circuit_ok,(const circuit_t *c)) tor_assert(!or_circ || !or_circ->rend_splice); } } - diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index fe3d8f1332..f9f5faa057 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -114,13 +114,6 @@ struct circuitmux_s { */ chanid_circid_muxinfo_map_t *chanid_circid_map; - /* - * Double-linked ring of circuits with queued cells waiting for room to - * free up on this connection's outbuf. Every time we pull cells from - * a circuit, we advance this pointer to the next circuit in the ring. - */ - struct circuit_t *active_circuits_head, *active_circuits_tail; - /** List of queued destroy cells */ destroy_cell_queue_t destroy_cell_queue; /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit @@ -177,17 +170,6 @@ struct chanid_circid_muxinfo_t { }; /* - * Internal-use #defines - */ - -#ifdef CMUX_PARANOIA -#define circuitmux_assert_okay_paranoid(cmux) \ - circuitmux_assert_okay(cmux) -#else -#define circuitmux_assert_okay_paranoid(cmux) -#endif /* defined(CMUX_PARANOIA) */ - -/* * Static function declarations */ @@ -199,21 +181,9 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a); static chanid_circid_muxinfo_t * circuitmux_find_map_entry(circuitmux_t *cmux, circuit_t *circ); static void -circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); +circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ); static void -circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -static inline void -circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -static inline circuit_t ** -circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ); -static inline circuit_t ** -circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ); -static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux); -static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux); -static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux); +circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ); /* Static global variables */ @@ -223,119 +193,6 @@ static int64_t global_destroy_ctr = 0; /* Function definitions */ /** - * Linked list helpers - */ - -/** - * Move an active circuit to the tail of the cmux's active circuits list; - * used by circuitmux_notify_xmit_cells(). - */ - -static inline void -circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) -{ - circuit_t **next_p = NULL, **prev_p = NULL; - circuit_t **next_prev = NULL, **prev_next = NULL; - circuit_t **tail_next = NULL; - or_circuit_t *or_circ = NULL; - - tor_assert(cmux); - tor_assert(circ); - - circuitmux_assert_okay_paranoid(cmux); - - /* Figure out our next_p and prev_p for this cmux/direction */ - if (direction) { - if (direction == CELL_DIRECTION_OUT) { - tor_assert(circ->n_mux == cmux); - next_p = &(circ->next_active_on_n_chan); - prev_p = &(circ->prev_active_on_n_chan); - } else { - or_circ = TO_OR_CIRCUIT(circ); - tor_assert(or_circ->p_mux == cmux); - next_p = &(or_circ->next_active_on_p_chan); - prev_p = &(or_circ->prev_active_on_p_chan); - } - } else { - if (circ->n_mux == cmux) { - next_p = &(circ->next_active_on_n_chan); - prev_p = &(circ->prev_active_on_n_chan); - } else { - or_circ = TO_OR_CIRCUIT(circ); - tor_assert(or_circ->p_mux == cmux); - next_p = &(or_circ->next_active_on_p_chan); - prev_p = &(or_circ->prev_active_on_p_chan); - } - } - tor_assert(next_p); - tor_assert(prev_p); - - /* Check if this really is an active circuit */ - if ((*next_p == NULL && *prev_p == NULL) && - !(circ == cmux->active_circuits_head || - circ == cmux->active_circuits_tail)) { - /* Not active, no-op */ - return; - } - - /* Check if this is already the tail */ - if (circ == cmux->active_circuits_tail) return; - - /* Okay, we have to move it; figure out next_prev and prev_next */ - if (*next_p) next_prev = circuitmux_prev_active_circ_p(cmux, *next_p); - if (*prev_p) prev_next = circuitmux_next_active_circ_p(cmux, *prev_p); - /* Adjust the previous node's next pointer, if any */ - if (prev_next) *prev_next = *next_p; - /* Otherwise, we were the head */ - else cmux->active_circuits_head = *next_p; - /* Adjust the next node's previous pointer, if any */ - if (next_prev) *next_prev = *prev_p; - /* We're out of the list; now re-attach at the tail */ - /* Adjust our next and prev pointers */ - *next_p = NULL; - *prev_p = cmux->active_circuits_tail; - /* Set the next pointer of the tail, or the head if none */ - if (cmux->active_circuits_tail) { - tail_next = circuitmux_next_active_circ_p(cmux, - cmux->active_circuits_tail); - *tail_next = circ; - } else { - cmux->active_circuits_head = circ; - } - /* Set the tail to this circuit */ - cmux->active_circuits_tail = circ; - - circuitmux_assert_okay_paranoid(cmux); -} - -static inline circuit_t ** -circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ) -{ - tor_assert(cmux); - tor_assert(circ); - - if (circ->n_mux == cmux) return &(circ->next_active_on_n_chan); - else { - tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux); - return &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan); - } -} - -static inline circuit_t ** -circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ) -{ - tor_assert(cmux); - tor_assert(circ); - - if (circ->n_mux == cmux) return &(circ->prev_active_on_n_chan); - else { - tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux); - return &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan); - } -} - -/** * Helper for chanid_circid_cell_count_map_t hash table: compare the channel * ID and circuit ID for a and b, and return less than, equal to, or greater * than zero appropriately. @@ -406,11 +263,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) circuit_t *circ = NULL; tor_assert(cmux); - /* - * Don't circuitmux_assert_okay_paranoid() here; this gets called when - * channels are being freed and have already been unregistered, so - * the channel ID lookups it does will fail. - */ i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); while (i) { @@ -435,7 +287,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) */ if (to_remove->muxinfo.cell_count > 0) { - circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_OUT); + circuitmux_make_circuit_inactive(cmux, circ); } /* Clear n_mux */ @@ -450,7 +302,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) */ if (to_remove->muxinfo.cell_count > 0) { - circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_IN); + circuitmux_make_circuit_inactive(cmux, circ); } /* @@ -606,9 +458,7 @@ circuitmux_clear_policy(circuitmux_t *cmux) tor_assert(cmux); /* Internally, this is just setting policy to NULL */ - if (cmux->policy) { - circuitmux_set_policy(cmux, NULL); - } + circuitmux_set_policy(cmux, NULL); } /** @@ -944,7 +794,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, tor_assert(circ); tor_assert(direction == CELL_DIRECTION_IN || direction == CELL_DIRECTION_OUT); - circuitmux_assert_okay_paranoid(cmux); /* * Figure out which channel we're using, and get the circuit's current @@ -1002,10 +851,10 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, */ if (hashent->muxinfo.cell_count > 0 && cell_count == 0) { --(cmux->n_active_circuits); - circuitmux_make_circuit_inactive(cmux, circ, direction); + circuitmux_make_circuit_inactive(cmux, circ); } else if (hashent->muxinfo.cell_count == 0 && cell_count > 0) { ++(cmux->n_active_circuits); - circuitmux_make_circuit_active(cmux, circ, direction); + circuitmux_make_circuit_active(cmux, circ); } cmux->n_cells -= hashent->muxinfo.cell_count; cmux->n_cells += cell_count; @@ -1033,7 +882,7 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, hashent->muxinfo.cell_count = cell_count; hashent->muxinfo.direction = direction; /* Allocate policy specific circuit data if we need it */ - if (cmux->policy && cmux->policy->alloc_circ_data) { + if (cmux->policy->alloc_circ_data) { /* Assert that we have the means to free policy-specific data */ tor_assert(cmux->policy->free_circ_data); /* Allocate it */ @@ -1053,25 +902,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux; else TO_OR_CIRCUIT(circ)->p_mux = cmux; - /* Make sure the next/prev pointers are NULL */ - if (direction == CELL_DIRECTION_OUT) { - circ->next_active_on_n_chan = NULL; - circ->prev_active_on_n_chan = NULL; - } else { - TO_OR_CIRCUIT(circ)->next_active_on_p_chan = NULL; - TO_OR_CIRCUIT(circ)->prev_active_on_p_chan = NULL; - } - /* Update counters */ ++(cmux->n_circuits); if (cell_count > 0) { ++(cmux->n_active_circuits); - circuitmux_make_circuit_active(cmux, circ, direction); + circuitmux_make_circuit_active(cmux, circ); } cmux->n_cells += cell_count; } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1095,7 +933,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) tor_assert(cmux); tor_assert(cmux->chanid_circid_map); tor_assert(circ); - circuitmux_assert_okay_paranoid(cmux); /* See if we have it for n_chan/n_circ_id */ if (circ->n_chan) { @@ -1133,7 +970,7 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) if (hashent->muxinfo.cell_count > 0) { --(cmux->n_active_circuits); /* This does policy notifies, so comes before freeing policy data */ - circuitmux_make_circuit_inactive(cmux, circ, last_searched_direction); + circuitmux_make_circuit_inactive(cmux, circ); } cmux->n_cells -= hashent->muxinfo.cell_count; @@ -1162,8 +999,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) /* Free the hash entry */ tor_free(hashent); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1172,94 +1007,22 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) */ static void -circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ) { - circuit_t **next_active = NULL, **prev_active = NULL, **next_prev = NULL; - circuitmux_t *circuit_cmux = NULL; - chanid_circid_muxinfo_t *hashent = NULL; - channel_t *chan = NULL; - circid_t circ_id; - int already_active; - tor_assert(cmux); + tor_assert(cmux->policy); tor_assert(circ); - tor_assert(direction == CELL_DIRECTION_OUT || - direction == CELL_DIRECTION_IN); - /* - * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count - * already got changed and we have to update the list for it to be consistent - * again. - */ - - /* Get the right set of active list links for this direction */ - if (direction == CELL_DIRECTION_OUT) { - next_active = &(circ->next_active_on_n_chan); - prev_active = &(circ->prev_active_on_n_chan); - circuit_cmux = circ->n_mux; - chan = circ->n_chan; - circ_id = circ->n_circ_id; - } else { - next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan); - prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan); - circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux; - chan = TO_OR_CIRCUIT(circ)->p_chan; - circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - /* Assert that it is attached to this mux and a channel */ - tor_assert(cmux == circuit_cmux); - tor_assert(chan != NULL); - - /* - * Check if the circuit really was inactive; if it's active, at least one - * of the next_active and prev_active pointers will not be NULL, or this - * circuit will be either the head or tail of the list for this cmux. - */ - already_active = (*prev_active != NULL || *next_active != NULL || - cmux->active_circuits_head == circ || - cmux->active_circuits_tail == circ); - - /* If we're already active, log a warning and finish */ - if (already_active) { - log_warn(LD_CIRC, - "Circuit %u on channel " U64_FORMAT " was already active", - (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier)); - return; - } - - /* - * This is going at the head of the list; if the old head is not NULL, - * then its prev pointer should point to this. - */ - *next_active = cmux->active_circuits_head; /* Next is old head */ - *prev_active = NULL; /* Prev is NULL (this will be the head) */ - if (cmux->active_circuits_head) { - /* The list had an old head; update its prev pointer */ - next_prev = - circuitmux_prev_active_circ_p(cmux, cmux->active_circuits_head); - tor_assert(next_prev); - *next_prev = circ; - } else { - /* The list was empty; this becomes the tail as well */ - cmux->active_circuits_tail = circ; - } - /* This becomes the new head of the list */ - cmux->active_circuits_head = circ; /* Policy-specific notification */ - if (cmux->policy && - cmux->policy->notify_circ_active) { + if (cmux->policy->notify_circ_active) { /* Okay, we need to check the circuit for policy data now */ - hashent = circuitmux_find_map_entry(cmux, circ); + chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ); /* We should have found something */ tor_assert(hashent); /* Notify */ cmux->policy->notify_circ_active(cmux, cmux->policy_data, circ, hashent->muxinfo.policy_data); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1268,112 +1031,22 @@ circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, */ static void -circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ) { - circuit_t **next_active = NULL, **prev_active = NULL; - circuit_t **next_prev = NULL, **prev_next = NULL; - circuitmux_t *circuit_cmux = NULL; - chanid_circid_muxinfo_t *hashent = NULL; - channel_t *chan = NULL; - circid_t circ_id; - int already_inactive; - tor_assert(cmux); + tor_assert(cmux->policy); tor_assert(circ); - tor_assert(direction == CELL_DIRECTION_OUT || - direction == CELL_DIRECTION_IN); - /* - * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count - * already got changed and we have to update the list for it to be consistent - * again. - */ - - /* Get the right set of active list links for this direction */ - if (direction == CELL_DIRECTION_OUT) { - next_active = &(circ->next_active_on_n_chan); - prev_active = &(circ->prev_active_on_n_chan); - circuit_cmux = circ->n_mux; - chan = circ->n_chan; - circ_id = circ->n_circ_id; - } else { - next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan); - prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan); - circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux; - chan = TO_OR_CIRCUIT(circ)->p_chan; - circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - /* Assert that it is attached to this mux and a channel */ - tor_assert(cmux == circuit_cmux); - tor_assert(chan != NULL); - - /* - * Check if the circuit really was active; if it's inactive, the - * next_active and prev_active pointers will be NULL and this circuit - * will not be the head or tail of the list for this cmux. - */ - already_inactive = (*prev_active == NULL && *next_active == NULL && - cmux->active_circuits_head != circ && - cmux->active_circuits_tail != circ); - - /* If we're already inactive, log a warning and finish */ - if (already_inactive) { - log_warn(LD_CIRC, - "Circuit %d on channel " U64_FORMAT " was already inactive", - (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier)); - return; - } - - /* Remove from the list; first get next_prev and prev_next */ - if (*next_active) { - /* - * If there's a next circuit, its previous circuit becomes this - * circuit's previous circuit. - */ - next_prev = circuitmux_prev_active_circ_p(cmux, *next_active); - } else { - /* Else, the tail becomes this circuit's previous circuit */ - next_prev = &(cmux->active_circuits_tail); - } - - /* Got next_prev, now prev_next */ - if (*prev_active) { - /* - * If there's a previous circuit, its next circuit becomes this circuit's - * next circuit. - */ - prev_next = circuitmux_next_active_circ_p(cmux, *prev_active); - } else { - /* Else, the head becomes this circuit's next circuit */ - prev_next = &(cmux->active_circuits_head); - } - - /* Assert that we got sensible values for the next/prev pointers */ - tor_assert(next_prev != NULL); - tor_assert(prev_next != NULL); - - /* Update the next/prev pointers - this removes circ from the list */ - *next_prev = *prev_active; - *prev_next = *next_active; - - /* Now null out prev_active/next_active */ - *prev_active = NULL; - *next_active = NULL; /* Policy-specific notification */ - if (cmux->policy && - cmux->policy->notify_circ_inactive) { + if (cmux->policy->notify_circ_inactive) { /* Okay, we need to check the circuit for policy data now */ - hashent = circuitmux_find_map_entry(cmux, circ); + chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ); /* We should have found something */ tor_assert(hashent); /* Notify */ cmux->policy->notify_circ_inactive(cmux, cmux->policy_data, circ, hashent->muxinfo.policy_data); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1400,8 +1073,6 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, tor_assert(cmux); tor_assert(circ); - circuitmux_assert_okay_paranoid(cmux); - /* Search for this circuit's entry */ hashent = circuitmux_find_map_entry(cmux, circ); /* Assert that we found one */ @@ -1412,7 +1083,7 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, cmux->n_cells += n_cells; /* Do we need to notify a cmux policy? */ - if (cmux->policy && cmux->policy->notify_set_n_cells) { + if (cmux->policy->notify_set_n_cells) { /* Call notify_set_n_cells */ cmux->policy->notify_set_n_cells(cmux, cmux->policy_data, @@ -1428,21 +1099,15 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, if (hashent->muxinfo.cell_count > 0 && n_cells == 0) { --(cmux->n_active_circuits); hashent->muxinfo.cell_count = n_cells; - circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction); + circuitmux_make_circuit_inactive(cmux, circ); /* Is the old cell count == 0 and the new cell count > 0 ? */ } else if (hashent->muxinfo.cell_count == 0 && n_cells > 0) { ++(cmux->n_active_circuits); hashent->muxinfo.cell_count = n_cells; - circuitmux_make_circuit_active(cmux, circ, hashent->muxinfo.direction); + circuitmux_make_circuit_active(cmux, circ); } else { - /* - * Update the entry cell count like this so we can put a - * circuitmux_assert_okay_paranoid inside make_circuit_(in)active() too. - */ hashent->muxinfo.cell_count = n_cells; } - - circuitmux_assert_okay_paranoid(cmux); } /* @@ -1468,6 +1133,9 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux, circuit_t *circ = NULL; tor_assert(cmux); + tor_assert(cmux->policy); + /* This callback is mandatory. */ + tor_assert(cmux->policy->pick_active_circuit); tor_assert(destroy_queue_out); *destroy_queue_out = NULL; @@ -1486,14 +1154,7 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux, /* We also must have a cell available for this to be the case */ tor_assert(cmux->n_cells > 0); /* Do we have a policy-provided circuit selector? */ - if (cmux->policy && cmux->policy->pick_active_circuit) { - circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data); - } - /* Fall back on the head of the active circuits list */ - if (!circ) { - tor_assert(cmux->active_circuits_head); - circ = cmux->active_circuits_head; - } + circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data); cmux->last_cell_was_destroy = 0; } else { tor_assert(cmux->n_cells == 0); @@ -1517,7 +1178,6 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, tor_assert(cmux); tor_assert(circ); - circuitmux_assert_okay_paranoid(cmux); if (n_cells == 0) return; @@ -1544,17 +1204,11 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, /* Adjust the mux cell counter */ cmux->n_cells -= n_cells; - /* If we aren't making it inactive later, move it to the tail of the list */ - if (!becomes_inactive) { - circuitmux_move_active_circ_to_tail(cmux, circ, - hashent->muxinfo.direction); - } - /* * We call notify_xmit_cells() before making the circuit inactive if needed, * so the policy can always count on this coming in on an active circuit. */ - if (cmux->policy && cmux->policy->notify_xmit_cells) { + if (cmux->policy->notify_xmit_cells) { cmux->policy->notify_xmit_cells(cmux, cmux->policy_data, circ, hashent->muxinfo.policy_data, n_cells); @@ -1566,10 +1220,8 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, */ if (becomes_inactive) { --(cmux->n_active_circuits); - circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction); + circuitmux_make_circuit_inactive(cmux, circ); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1592,282 +1244,6 @@ circuitmux_notify_xmit_destroy(circuitmux_t *cmux) I64_PRINTF_ARG(global_destroy_ctr)); } -/* - * Circuitmux consistency checking assertions - */ - -/** - * Check that circuitmux data structures are consistent and fail with an - * assert if not. - */ - -void -circuitmux_assert_okay(circuitmux_t *cmux) -{ - tor_assert(cmux); - - /* - * Pass 1: iterate the hash table; for each entry: - * a) Check that the circuit has this cmux for n_mux or p_mux - * b) If the cell_count is > 0, set the mark bit; otherwise clear it - * c) Also check activeness (cell_count > 0 should be active) - * d) Count the number of circuits, active circuits and queued cells - * and at the end check that they match the counters in the cmux. - * - * Pass 2: iterate the active circuits list; for each entry, - * make sure the circuit is attached to this mux and appears - * in the hash table. Make sure the mark bit is 1, and clear - * it in the hash table entry. Consistency-check the linked - * list pointers. - * - * Pass 3: iterate the hash table again; assert if any active circuits - * (mark bit set to 1) are discovered that weren't cleared in pass 2 - * (don't appear in the linked list). - */ - - circuitmux_assert_okay_pass_one(cmux); - circuitmux_assert_okay_pass_two(cmux); - circuitmux_assert_okay_pass_three(cmux); -} - -/** - * Do the first pass of circuitmux_assert_okay(); see the comment in that - * function. - */ - -static void -circuitmux_assert_okay_pass_one(circuitmux_t *cmux) -{ - chanid_circid_muxinfo_t **i = NULL; - uint64_t chan_id; - channel_t *chan; - circid_t circ_id; - circuit_t *circ; - or_circuit_t *or_circ; - circuit_t **next_p, **prev_p; - unsigned int n_circuits, n_active_circuits, n_cells; - - tor_assert(cmux); - tor_assert(cmux->chanid_circid_map); - - /* Reset the counters */ - n_circuits = n_active_circuits = n_cells = 0; - /* Start iterating the hash table */ - i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); - while (i) { - /* Assert that the hash table entry isn't null */ - tor_assert(*i); - - /* Get the channel and circuit id */ - chan_id = (*i)->chan_id; - circ_id = (*i)->circ_id; - - /* Find the channel and circuit, assert that they exist */ - chan = channel_find_by_global_id(chan_id); - tor_assert(chan); - circ = circuit_get_by_circid_channel_even_if_marked(circ_id, chan); - tor_assert(circ); - - /* Assert that we know which direction this is going */ - tor_assert((*i)->muxinfo.direction == CELL_DIRECTION_OUT || - (*i)->muxinfo.direction == CELL_DIRECTION_IN); - - if ((*i)->muxinfo.direction == CELL_DIRECTION_OUT) { - /* We should be n_mux on this circuit */ - tor_assert(cmux == circ->n_mux); - tor_assert(chan == circ->n_chan); - /* Get next and prev for next test */ - next_p = &(circ->next_active_on_n_chan); - prev_p = &(circ->prev_active_on_n_chan); - } else { - /* This should be an or_circuit_t and we should be p_mux */ - or_circ = TO_OR_CIRCUIT(circ); - tor_assert(cmux == or_circ->p_mux); - tor_assert(chan == or_circ->p_chan); - /* Get next and prev for next test */ - next_p = &(or_circ->next_active_on_p_chan); - prev_p = &(or_circ->prev_active_on_p_chan); - } - - /* - * Should this circuit be active? I.e., does the mux know about > 0 - * cells on it? - */ - const int circ_is_active = ((*i)->muxinfo.cell_count > 0); - - /* It should be in the linked list iff it's active */ - if (circ_is_active) { - /* Either we have a next link or we are the tail */ - tor_assert(*next_p || (circ == cmux->active_circuits_tail)); - /* Either we have a prev link or we are the head */ - tor_assert(*prev_p || (circ == cmux->active_circuits_head)); - /* Increment the active circuits counter */ - ++n_active_circuits; - } else { - /* Shouldn't be in list, so no next or prev link */ - tor_assert(!(*next_p)); - tor_assert(!(*prev_p)); - /* And can't be head or tail */ - tor_assert(circ != cmux->active_circuits_head); - tor_assert(circ != cmux->active_circuits_tail); - } - - /* Increment the circuits counter */ - ++n_circuits; - /* Adjust the cell counter */ - n_cells += (*i)->muxinfo.cell_count; - - /* Set the mark bit to circ_is_active */ - (*i)->muxinfo.mark = circ_is_active; - - /* Advance to the next entry */ - i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i); - } - - /* Now check the counters */ - tor_assert(n_cells == cmux->n_cells); - tor_assert(n_circuits == cmux->n_circuits); - tor_assert(n_active_circuits == cmux->n_active_circuits); -} - -/** - * Do the second pass of circuitmux_assert_okay(); see the comment in that - * function. - */ - -static void -circuitmux_assert_okay_pass_two(circuitmux_t *cmux) -{ - circuit_t *curr_circ, *prev_circ = NULL, *next_circ; - or_circuit_t *curr_or_circ; - uint64_t curr_chan_id; - circid_t curr_circ_id; - circuit_t **next_p, **prev_p; - channel_t *chan; - unsigned int n_active_circuits = 0; - chanid_circid_muxinfo_t search, *hashent = NULL; - - tor_assert(cmux); - tor_assert(cmux->chanid_circid_map); - - /* - * Walk the linked list of active circuits in cmux; keep track of the - * previous circuit seen for consistency checking purposes. Count them - * to make sure the number in the linked list matches - * cmux->n_active_circuits. - */ - curr_circ = cmux->active_circuits_head; - while (curr_circ) { - /* Reset some things */ - chan = NULL; - curr_or_circ = NULL; - next_circ = NULL; - next_p = prev_p = NULL; - cell_direction_t direction; - - /* Figure out if this is n_mux or p_mux */ - if (cmux == curr_circ->n_mux) { - /* Get next_p and prev_p */ - next_p = &(curr_circ->next_active_on_n_chan); - prev_p = &(curr_circ->prev_active_on_n_chan); - /* Get the channel */ - chan = curr_circ->n_chan; - /* Get the circuit id */ - curr_circ_id = curr_circ->n_circ_id; - /* Remember the direction */ - direction = CELL_DIRECTION_OUT; - } else { - /* We must be p_mux and this must be an or_circuit_t */ - curr_or_circ = TO_OR_CIRCUIT(curr_circ); - tor_assert(cmux == curr_or_circ->p_mux); - /* Get next_p and prev_p */ - next_p = &(curr_or_circ->next_active_on_p_chan); - prev_p = &(curr_or_circ->prev_active_on_p_chan); - /* Get the channel */ - chan = curr_or_circ->p_chan; - /* Get the circuit id */ - curr_circ_id = curr_or_circ->p_circ_id; - /* Remember the direction */ - direction = CELL_DIRECTION_IN; - } - - /* Assert that we got a channel and get the channel ID */ - tor_assert(chan); - curr_chan_id = chan->global_identifier; - - /* Assert that prev_p points to last circuit we saw */ - tor_assert(*prev_p == prev_circ); - /* If that's NULL, assert that we are the head */ - if (!(*prev_p)) tor_assert(curr_circ == cmux->active_circuits_head); - - /* Get the next circuit */ - next_circ = *next_p; - /* If it's NULL, assert that we are the tail */ - if (!(*next_p)) tor_assert(curr_circ == cmux->active_circuits_tail); - - /* Now find the hash table entry for this circuit */ - search.chan_id = curr_chan_id; - search.circ_id = curr_circ_id; - hashent = HT_FIND(chanid_circid_muxinfo_map, cmux->chanid_circid_map, - &search); - - /* Assert that we have one */ - tor_assert(hashent); - - /* Assert that the direction matches */ - tor_assert(direction == hashent->muxinfo.direction); - - /* Assert that the hash entry got marked in pass one */ - tor_assert(hashent->muxinfo.mark); - - /* Clear the mark */ - hashent->muxinfo.mark = 0; - - /* Increment the counter */ - ++n_active_circuits; - - /* Advance to the next active circuit and update prev_circ */ - prev_circ = curr_circ; - curr_circ = next_circ; - } - - /* Assert that the counter matches the cmux */ - tor_assert(n_active_circuits == cmux->n_active_circuits); -} - -/** - * Do the third pass of circuitmux_assert_okay(); see the comment in that - * function. - */ - -static void -circuitmux_assert_okay_pass_three(circuitmux_t *cmux) -{ - chanid_circid_muxinfo_t **i = NULL; - - tor_assert(cmux); - tor_assert(cmux->chanid_circid_map); - - /* Start iterating the hash table */ - i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); - - /* Advance through each entry */ - while (i) { - /* Assert that it isn't null */ - tor_assert(*i); - - /* - * Assert that this entry is not marked - i.e., that either we didn't - * think it should be active in pass one or we saw it in the active - * circuits linked list. - */ - tor_assert(!((*i)->muxinfo.mark)); - - /* Advance to the next entry */ - i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i); - } -} - /*DOCDOC */ void circuitmux_append_destroy_cell(channel_t *chan, diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c index fde2d22a89..e5d5a14581 100644 --- a/src/or/circuitmux_ewma.c +++ b/src/or/circuitmux_ewma.c @@ -28,7 +28,7 @@ * **/ -#define TOR_CIRCUITMUX_EWMA_C_ +#define CIRCUITMUX_EWMA_PRIVATE #include "orconfig.h" @@ -37,6 +37,7 @@ #include "or.h" #include "circuitmux.h" #include "circuitmux_ewma.h" +#include "crypto_rand.h" #include "networkstatus.h" /*** EWMA parameter #defines ***/ @@ -169,8 +170,6 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol) static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma); static int compare_cell_ewma_counts(const void *p1, const void *p2); -static unsigned cell_ewma_tick_from_timeval(const struct timeval *now, - double *remainder_out); static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma); static inline double get_scale_factor(unsigned from_tick, unsigned to_tick); static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol); @@ -223,8 +222,6 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, * has value ewma_scale_factor ** N.) */ static double ewma_scale_factor = 0.1; -/* DOCDOC ewma_enabled */ -static int ewma_enabled = 0; /*** EWMA circuitmux_policy_t method table ***/ @@ -241,8 +238,26 @@ circuitmux_policy_t ewma_policy = { /*.cmp_cmux =*/ ewma_cmp_cmux }; +/** Have we initialized the ewma tick-counting logic? */ +static int ewma_ticks_initialized = 0; +/** At what monotime_coarse_t did the current tick begin? */ +static monotime_coarse_t start_of_current_tick; +/** What is the number of the current tick? */ +static unsigned current_tick_num; + /*** EWMA method implementations using the below EWMA helper functions ***/ +/** Compute and return the current cell_ewma tick. */ +static inline unsigned int +cell_ewma_get_tick(void) +{ + monotime_coarse_t now; + monotime_coarse_get(&now); + int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, + &now); + return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN); +} + /** * Allocate an ewma_policy_data_t and upcast it to a circuitmux_policy_data_t; * this is called when setting the policy on a circuitmux_t to ewma_policy. @@ -416,8 +431,6 @@ ewma_notify_xmit_cells(circuitmux_t *cmux, ewma_policy_circ_data_t *cdata = NULL; unsigned int tick; double fractional_tick, ewma_increment; - /* The current (hi-res) time */ - struct timeval now_hires; cell_ewma_t *cell_ewma, *tmp; tor_assert(cmux); @@ -430,8 +443,7 @@ ewma_notify_xmit_cells(circuitmux_t *cmux, cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data); /* Rescale the EWMAs if needed */ - tor_gettimeofday_cached(&now_hires); - tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + tick = cell_ewma_get_current_tick_and_fraction(&fractional_tick); if (tick != pol->active_circuit_pqueue_last_recalibrated) { scale_active_circuits(pol, tick); @@ -592,79 +604,122 @@ cell_ewma_to_circuit(cell_ewma_t *ewma) rescale. */ -/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs - * and the fraction of the tick that has elapsed between the start of the tick - * and <b>now</b>. Return the former and store the latter in - * *<b>remainder_out</b>. +/** + * Initialize the system that tells which ewma tick we are in. + */ +STATIC void +cell_ewma_initialize_ticks(void) +{ + if (ewma_ticks_initialized) + return; + monotime_coarse_get(&start_of_current_tick); + crypto_rand((char*)¤t_tick_num, sizeof(current_tick_num)); + ewma_ticks_initialized = 1; +} + +/** Compute the current cell_ewma tick and the fraction of the tick that has + * elapsed between the start of the tick and the current time. Return the + * former and store the latter in *<b>remainder_out</b>. * * These tick values are not meant to be shared between Tor instances, or used * for other purposes. */ - -static unsigned -cell_ewma_tick_from_timeval(const struct timeval *now, - double *remainder_out) +STATIC unsigned +cell_ewma_get_current_tick_and_fraction(double *remainder_out) { - unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); - /* rem */ - double rem = (now->tv_sec % EWMA_TICK_LEN) + - ((double)(now->tv_usec)) / 1.0e6; - *remainder_out = rem / EWMA_TICK_LEN; - return res; + if (BUG(!ewma_ticks_initialized)) { + cell_ewma_initialize_ticks(); // LCOV_EXCL_LINE + } + monotime_coarse_t now; + monotime_coarse_get(&now); + int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, + &now); + if (msec_diff > (1000*EWMA_TICK_LEN)) { + unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN); + monotime_coarse_add_msec(&start_of_current_tick, + &start_of_current_tick, + ticks_difference * 1000 * EWMA_TICK_LEN); + current_tick_num += ticks_difference; + msec_diff %= 1000*EWMA_TICK_LEN; + } + *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN); + return current_tick_num; } -/** Tell the caller whether ewma_enabled is set */ -int -cell_ewma_enabled(void) +/* Default value for the CircuitPriorityHalflifeMsec consensus parameter in + * msec. */ +#define CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT 30000 +/* Minimum and maximum value for the CircuitPriorityHalflifeMsec consensus + * parameter. */ +#define CMUX_PRIORITY_HALFLIFE_MSEC_MIN 1 +#define CMUX_PRIORITY_HALFLIFE_MSEC_MAX INT32_MAX + +/* Return the value of the circuit priority halflife from the options if + * available or else from the consensus (in that order). If none can be found, + * a default value is returned. + * + * The source_msg points to a string describing from where the value was + * picked so it can be used for logging. */ +static double +get_circuit_priority_halflife(const or_options_t *options, + const networkstatus_t *consensus, + const char **source_msg) { - return ewma_enabled; -} + int32_t halflife_ms; + double halflife; + /* Compute the default value now. We might need it. */ + double halflife_default = + ((double) CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT) / 1000.0; -/** Compute and return the current cell_ewma tick. */ -unsigned int -cell_ewma_get_tick(void) -{ - return ((unsigned)approx_time() / EWMA_TICK_LEN); + /* Try to get it from configuration file first. */ + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + *source_msg = "CircuitPriorityHalflife in configuration"; + goto end; + } + + /* Try to get the msec value from the consensus. */ + halflife_ms = networkstatus_get_param(consensus, + "CircuitPriorityHalflifeMsec", + CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT, + CMUX_PRIORITY_HALFLIFE_MSEC_MIN, + CMUX_PRIORITY_HALFLIFE_MSEC_MAX); + halflife = ((double) halflife_ms) / 1000.0; + *source_msg = "CircuitPriorityHalflifeMsec in consensus"; + + end: + /* We should never go below the EPSILON else we would consider it disabled + * and we can't have that. */ + if (halflife < EPSILON) { + log_warn(LD_CONFIG, "CircuitPriorityHalflife is too small (%f). " + "Adjusting to the smallest value allowed: %f.", + halflife, halflife_default); + halflife = halflife_default; + } + return halflife; } /** Adjust the global cell scale factor based on <b>options</b> */ void -cell_ewma_set_scale_factor(const or_options_t *options, - const networkstatus_t *consensus) +cmux_ewma_set_options(const or_options_t *options, + const networkstatus_t *consensus) { - int32_t halflife_ms; double halflife; const char *source; - if (options && options->CircuitPriorityHalflife >= -EPSILON) { - halflife = options->CircuitPriorityHalflife; - source = "CircuitPriorityHalflife in configuration"; - } else if (consensus && (halflife_ms = networkstatus_get_param( - consensus, "CircuitPriorityHalflifeMsec", - -1, -1, INT32_MAX)) >= 0) { - halflife = ((double)halflife_ms)/1000.0; - source = "CircuitPriorityHalflifeMsec in consensus"; - } else { - halflife = EWMA_DEFAULT_HALFLIFE; - source = "Default value"; - } - if (halflife <= EPSILON) { - /* The cell EWMA algorithm is disabled. */ - ewma_scale_factor = 0.1; - ewma_enabled = 0; - log_info(LD_OR, - "Disabled cell_ewma algorithm because of value in %s", - source); - } else { - /* convert halflife into halflife-per-tick. */ - halflife /= EWMA_TICK_LEN; - /* compute per-tick scale factor. */ - ewma_scale_factor = exp( LOG_ONEHALF / halflife ); - ewma_enabled = 1; - log_info(LD_OR, - "Enabled cell_ewma algorithm because of value in %s; " - "scale factor is %f per %d seconds", - source, ewma_scale_factor, EWMA_TICK_LEN); - } + cell_ewma_initialize_ticks(); + + /* Both options and consensus can be NULL. This assures us to either get a + * valid configured value or the default one. */ + halflife = get_circuit_priority_halflife(options, consensus, &source); + + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %f per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); } /** Return the multiplier necessary to convert the value of a cell sent in @@ -763,3 +818,12 @@ pop_first_cell_ewma(ewma_policy_data_t *pol) offsetof(cell_ewma_t, heap_index)); } +/** + * Drop all resources held by circuitmux_ewma.c, and deinitialize the + * module. */ +void +circuitmux_ewma_free_all(void) +{ + ewma_ticks_initialized = 0; +} + diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index 8f4e57865e..f0c4c36095 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -12,13 +12,19 @@ #include "or.h" #include "circuitmux.h" +/* The public EWMA policy callbacks object. */ extern circuitmux_policy_t ewma_policy; /* Externally visible EWMA functions */ -int cell_ewma_enabled(void); -unsigned int cell_ewma_get_tick(void); -void cell_ewma_set_scale_factor(const or_options_t *options, - const networkstatus_t *consensus); +void cmux_ewma_set_options(const or_options_t *options, + const networkstatus_t *consensus); + +void circuitmux_ewma_free_all(void); + +#ifdef CIRCUITMUX_EWMA_PRIVATE +STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out); +STATIC void cell_ewma_initialize_ticks(void); +#endif #endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */ diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index f06c2e5e38..94f75c590f 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -31,10 +31,12 @@ #include "config.h" #include "confparse.h" #include "control.h" +#include "crypto_rand.h" #include "main.h" #include "networkstatus.h" #include "rendclient.h" #include "rendservice.h" +#include "router.h" #include "statefile.h" #include "circuitlist.h" #include "circuituse.h" @@ -125,7 +127,7 @@ circuit_build_times_disabled_(const or_options_t *options, ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled", 0, 0, 1); int config_disabled = !options->LearnCircuitBuildTimeout; - int dirauth_disabled = options->AuthoritativeDir; + int dirauth_disabled = authdir_mode(options); int state_disabled = did_last_state_file_write_fail() ? 1 : 0; /* LearnCircuitBuildTimeout and Tor2web/Single Onion Services are * incompatible in two ways: diff --git a/src/or/circuituse.c b/src/or/circuituse.c index dc62e4d09c..8e007ce920 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -55,7 +55,6 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" -#include "config.h" static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); @@ -1632,7 +1631,7 @@ circuit_testing_opened(origin_circuit_t *circ) router_perform_bandwidth_test(NUM_PARALLEL_TESTING_CIRCS, time(NULL)); have_performed_bandwidth_test = 1; } else - consider_testing_reachability(1, 0); + router_do_reachability_checks(1, 0); } /** A testing circuit has failed to build. Take whatever stats we want. */ @@ -2607,7 +2606,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", (unsigned)circ->base_.n_circ_id); /* reset it, so we can measure circ timeouts */ - ENTRY_TO_CONN(apconn)->timestamp_lastread = time(NULL); + ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL); ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams; ENTRY_TO_EDGE_CONN(apconn)->on_circuit = TO_CIRCUIT(circ); /* assert_connection_ok(conn, time(NULL)); */ @@ -3107,3 +3106,41 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ) circ->unusable_for_new_conns = 1; } +/** + * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to + * the valid delivered written fields and the overhead field, + * respectively. + */ +void +circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len) +{ + if (!circ) return; + + tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE); + + circ->n_delivered_written_circ_bw = + tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len); + circ->n_overhead_written_circ_bw = + tor_add_u32_nowrap(circ->n_overhead_written_circ_bw, + RELAY_PAYLOAD_SIZE-relay_body_len); +} + +/** + * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to + * the valid delivered read field and the overhead field, + * respectively. + */ +void +circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len) +{ + if (!circ) return; + + tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE); + + circ->n_delivered_read_circ_bw = + tor_add_u32_nowrap(circ->n_delivered_read_circ_bw, relay_body_len); + circ->n_overhead_read_circ_bw = + tor_add_u32_nowrap(circ->n_overhead_read_circ_bw, + RELAY_PAYLOAD_SIZE-relay_body_len); +} + diff --git a/src/or/circuituse.h b/src/or/circuituse.h index 71c818b978..6458bd6908 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -65,6 +65,8 @@ void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ); int circuit_purpose_is_hidden_service(uint8_t); int circuit_should_use_vanguards(uint8_t); +void circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len); +void circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len); #ifdef TOR_UNIT_TESTS /* Used only by circuituse.c and test_circuituse.c */ diff --git a/src/or/command.c b/src/or/command.c index 7280be1396..39950f41bf 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -46,6 +46,7 @@ #include "config.h" #include "control.h" #include "cpuworker.h" +#include "crypto_util.h" #include "dos.h" #include "hibernate.h" #include "nodelist.h" @@ -339,7 +340,9 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } - if (connection_or_digest_is_known_relay(chan->identity_digest)) { + if (!channel_is_client(chan)) { + /* remember create types we've seen, but don't remember them from + * clients, to be extra conservative about client statistics. */ rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); } @@ -493,6 +496,17 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) /* if we're a relay and treating connections with recent local * traffic better, then this is one of them. */ channel_timestamp_client(chan); + + /* Count all circuit bytes here for control port accuracy. We want + * to count even invalid/dropped relay cells, hence counting + * before the recognized check and the connection_edge_process_relay + * cell checks. + */ + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + + /* Count the payload bytes only. We don't care about cell headers */ + ocirc->n_read_circ_bw = tor_add_u32_nowrap(ocirc->n_read_circ_bw, + CELL_PAYLOAD_SIZE); } if (!CIRCUIT_IS_ORIGIN(circ) && diff --git a/src/or/config.c b/src/or/config.c index e50e6efa20..53c19e3900 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,3 +1,4 @@ + /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. @@ -78,8 +79,9 @@ #include "control.h" #include "confparse.h" #include "cpuworker.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "dirserv.h" -#include "dirvote.h" #include "dns.h" #include "dos.h" #include "entrynodes.h" @@ -104,12 +106,16 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" +#include "voting_schedule.h" #ifdef _WIN32 #include <shlobj.h> #endif #include "procmon.h" +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" + #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -126,6 +132,11 @@ static const char unix_socket_prefix[] = "unix:"; * configuration. */ static const char unix_q_socket_prefix[] = "unix:\""; +/** macro to help with the bulk rename of *DownloadSchedule to + * *DowloadInitialDelay . */ +#define DOWNLOAD_SCHEDULE(name) \ + { #name "DownloadSchedule", #name "DownloadInitialDelay", 0, 1 } + /** A list of abbreviations and aliases to map command-line options, obsolete * option names, or alternative option names, to their current values. */ static config_abbrev_t option_abbrevs_[] = { @@ -175,6 +186,16 @@ static config_abbrev_t option_abbrevs_[] = { { "_HSLayer2Nodes", "HSLayer2Nodes", 0, 1 }, { "_HSLayer3Nodes", "HSLayer3Nodes", 0, 1 }, + DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthority), + DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthorityOnly), + DOWNLOAD_SCHEDULE(ClientBootstrapConsensusFallback), + DOWNLOAD_SCHEDULE(TestingBridge), + DOWNLOAD_SCHEDULE(TestingBridgeBootstrap), + DOWNLOAD_SCHEDULE(TestingClient), + DOWNLOAD_SCHEDULE(TestingClientConsensus), + DOWNLOAD_SCHEDULE(TestingServer), + DOWNLOAD_SCHEDULE(TestingServerConsensus), + { NULL, NULL, 0, 0}, }; @@ -267,7 +288,7 @@ static config_var_t option_vars_[] = { OBSOLETE("CircuitIdleTimeout"), V(CircuitsAvailableTimeout, INTERVAL, "0"), V(CircuitStreamTimeout, INTERVAL, "0"), - V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ + V(CircuitPriorityHalflife, DOUBLE, "-1.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), @@ -337,7 +358,7 @@ static config_var_t option_vars_[] = { V(DownloadExtraInfo, BOOL, "0"), V(TestingEnableConnBwEvent, BOOL, "0"), V(TestingEnableCellStatsEvent, BOOL, "0"), - V(TestingEnableTbEmptyEvent, BOOL, "0"), + OBSOLETE("TestingEnableTbEmptyEvent"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), @@ -457,6 +478,7 @@ static config_var_t option_vars_[] = { V(NumCPUs, UINT, "0"), V(NumDirectoryGuards, UINT, "0"), V(NumEntryGuards, UINT, "0"), + V(NumPrimaryGuards, UINT, "0"), V(OfflineMasterKey, BOOL, "0"), OBSOLETE("ORListenAddress"), VPORT(ORPort), @@ -495,8 +517,8 @@ static config_var_t option_vars_[] = { V(TestingSigningKeySlop, INTERVAL, "1 day"), V(OptimisticData, AUTOBOOL, "auto"), - V(PortForwarding, BOOL, "0"), - V(PortForwardingHelper, FILENAME, "tor-fw-helper"), + OBSOLETE("PortForwarding"), + OBSOLETE("PortForwardingHelper"), OBSOLETE("PreferTunneledDirConns"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), @@ -599,16 +621,10 @@ static config_var_t option_vars_[] = { VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), VAR("__OwningControllerFD",INT,OwningControllerFD, "-1"), V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"), - V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, " - "300, 900, 2147483647"), - V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, " - "2147483647"), - V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " - "300, 600, 1800, 1800, 1800, 1800, " - "1800, 3600, 7200"), - V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " - "300, 600, 1800, 3600, 3600, 3600, " - "10800, 21600, 43200"), + V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), /* With the ClientBootstrapConsensus*Download* below: * Clients with only authorities will try: * - at least 3 authorities over 10 seconds, then exponentially backoff, @@ -624,13 +640,11 @@ static config_var_t option_vars_[] = { * * When clients have authorities and fallbacks available, they use these * schedules: (we stagger the times to avoid thundering herds) */ - V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL, - "6, 11, 3600, 10800, 25200, 54000, 111600, 262800" /* 3 days + 1 hour */), - V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL, - "0, 1, 4, 11, 3600, 10800, 25200, 54000, 111600, 262800"), + V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "6"), + V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"), /* When clients only have authorities available, they use this schedule: */ - V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL, - "0, 3, 7, 3600, 10800, 25200, 54000, 111600, 262800"), + V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL, + "0"), /* We don't want to overwhelm slow networks (or mirrors whose replies are * blocked), but we also don't want to fail if only some mirrors are * blackholed. Clients will try 3 directories simultaneously. @@ -638,14 +652,12 @@ static config_var_t option_vars_[] = { V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"), /* When a client has any running bridges, check each bridge occasionally, * whether or not that bridge is actually up. */ - V(TestingBridgeDownloadSchedule, CSV_INTERVAL, - "10800, 25200, 54000, 111600, 262800"), + V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"), /* When a client is just starting, or has no running bridges, check each * bridge a few times quickly, and then try again later. These schedules * are much longer than the other schedules, because we try each and every * configured bridge with this schedule. */ - V(TestingBridgeBootstrapDownloadSchedule, CSV_INTERVAL, - "0, 30, 90, 600, 3600, 10800, 25200, 54000, 111600, 262800"), + V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"), V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"), V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"), OBSOLETE("TestingConsensusMaxDownloadTries"), @@ -672,12 +684,10 @@ static const config_var_t testing_tor_network_defaults[] = { V(EnforceDistinctSubnets, BOOL, "0"), V(AssumeReachable, BOOL, "1"), V(AuthDirMaxServersPerAddr, UINT, "0"), - V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL, - "0, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), - V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL, - "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), - V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL, - "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), + V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"), + V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"), + V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL, + "0"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), V(ClientRejectInternalAddresses, BOOL, "0"), V(CountPrivateBandwidth, BOOL, "1"), @@ -692,22 +702,16 @@ static const config_var_t testing_tor_network_defaults[] = { V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), - V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, " - "20, 30, 60"), - V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, " - "30, 60"), - V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " - "15, 20, 30, 60"), - V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " - "15, 20, 30, 60"), - V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "10, 30, 60"), - V(TestingBridgeBootstrapDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " - "15, 20, 30, 60"), + V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"), + V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"), V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"), V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), V(TestingEnableConnBwEvent, BOOL, "1"), V(TestingEnableCellStatsEvent, BOOL, "1"), - V(TestingEnableTbEmptyEvent, BOOL, "1"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), V(RendPostPeriod, INTERVAL, "2 minutes"), @@ -747,6 +751,8 @@ static int options_transition_affects_workers( const or_options_t *old_options, const or_options_t *new_options); static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); +static int options_transition_affects_dirauth_timing( + const or_options_t *old_options, const or_options_t *new_options); static int normalize_nickname_list(config_line_t **normalized_out, const config_line_t *lst, const char *name, char **msg); @@ -905,8 +911,13 @@ set_options(or_options_t *new_val, char **msg) smartlist_free(elements); } - if (old_options != global_options) + if (old_options != global_options) { or_options_free(old_options); + /* If we are here it means we've successfully applied the new options and + * that the global options have been changed to the new values. We'll + * check if we need to remove or add periodic events. */ + periodic_events_on_new_options(global_options); + } return 0; } @@ -1439,9 +1450,9 @@ options_act_reversible(const or_options_t *old_options, char **msg) consider_hibernation(time(NULL)); /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) We don't want to rebind if we're hibernating. If - * networking is disabled, this will close all but the control listeners, - * but disable those. */ + * ports under 1024.) We don't want to rebind if we're hibernating or + * shutting down. If networking is disabled, this will close all but the + * control listeners, but disable those. */ if (!we_are_hibernating()) { if (retry_all_listeners(replaced_listeners, new_listeners, options->DisableNetwork) < 0) { @@ -1562,6 +1573,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) tor_malloc_zero(sizeof(log_severity_list_t)); close_temp_logs(); add_callback_log(severity, control_event_logmsg); + logs_set_pending_callback_callback(control_event_logmsg_pending); control_adjust_event_log_severity(); tor_free(severity); tor_log_update_sigsafe_err_fds(); @@ -1669,8 +1681,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) int options_need_geoip_info(const or_options_t *options, const char **reason_out) { - int bridge_usage = - options->BridgeRelay && options->BridgeRecordUsageByCountry; + int bridge_usage = should_record_bridge_info(options); int routerset_usage = routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || @@ -1760,6 +1771,32 @@ options_transition_affects_guards(const or_options_t *old_options, return 0; } +/** + * Return true if changing the configuration from <b>old</b> to <b>new</b> + * affects the timing of the voting subsystem + */ +static int +options_transition_affects_dirauth_timing(const or_options_t *old_options, + const or_options_t *new_options) +{ + tor_assert(old_options); + tor_assert(new_options); + + if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options)) + return 1; + if (! authdir_mode_v3(new_options)) + return 0; + YES_IF_CHANGED_INT(V3AuthVotingInterval); + YES_IF_CHANGED_INT(V3AuthVoteDelay); + YES_IF_CHANGED_INT(V3AuthDistDelay); + YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval); + YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay); + YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay); + YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset); + + return 0; +} + /** Fetch the active option list, and take actions based on it. All of the * things we do should survive being done repeatedly. If present, * <b>old_options</b> contains the previous value of the options. @@ -1778,7 +1815,6 @@ options_act(const or_options_t *old_options) char *msg=NULL; const int transition_affects_workers = old_options && options_transition_affects_workers(old_options, options); - int old_ewma_enabled; const int transition_affects_guards = old_options && options_transition_affects_guards(old_options, options); @@ -1983,6 +2019,9 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } + /* See whether we need to enable/disable our once-a-second timer. */ + reschedule_per_second_timer(); + /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || @@ -2052,16 +2091,8 @@ options_act(const or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); - old_ewma_enabled = cell_ewma_enabled(); /* Change the cell EWMA settings */ - cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); - /* If we just enabled ewma, set the cmux policy on all active channels */ - if (cell_ewma_enabled() && !old_ewma_enabled) { - channel_set_cmux_policy_everywhere(&ewma_policy); - } else if (!cell_ewma_enabled() && old_ewma_enabled) { - /* Turn it off everywhere */ - channel_set_cmux_policy_everywhere(NULL); - } + cmux_ewma_set_options(options, networkstatus_get_latest_consensus()); /* Update the BridgePassword's hashed version as needed. We store this as a * digest so that we can do side-channel-proof comparisons on it. @@ -2224,6 +2255,12 @@ options_act(const or_options_t *old_options) options->PerConnBWBurst != old_options->PerConnBWBurst) connection_or_update_token_buckets(get_connection_array(), options); + if (options->BandwidthRate != old_options->BandwidthRate || + options->BandwidthBurst != old_options->BandwidthBurst || + options->RelayBandwidthRate != old_options->RelayBandwidthRate || + options->RelayBandwidthBurst != old_options->RelayBandwidthBurst) + connection_bucket_adjust(options); + if (options->MainloopStats != old_options->MainloopStats) { reset_main_loop_counters(); } @@ -2273,6 +2310,11 @@ options_act(const or_options_t *old_options) } if ((!old_options || !old_options->EntryStatistics) && options->EntryStatistics && !should_record_bridge_info(options)) { + /* If we get here, we've started recording bridge info when we didn't + * do so before. Note that "should_record_bridge_info()" will + * always be false at this point, because of the earlier block + * that cleared EntryStatistics when public_server_mode() was false. + * We're leaving it in as defensive programming. */ if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) { geoip_entry_stats_init(now); print_notice = 1; @@ -2342,8 +2384,10 @@ options_act(const or_options_t *old_options) /* We may need to reschedule some directory stuff if our status changed. */ if (old_options) { - if (authdir_mode_v3(options) && !authdir_mode_v3(old_options)) - dirvote_recalculate_timing(options, time(NULL)); + if (options_transition_affects_dirauth_timing(old_options, options)) { + voting_schedule_recalculate_timing(options, time(NULL)); + reschedule_dirvote(options); + } if (!bool_eq(directory_fetches_dir_info_early(options), directory_fetches_dir_info_early(old_options)) || !bool_eq(directory_fetches_dir_info_later(options), @@ -3491,12 +3535,14 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Versioning authoritative dir servers must set " "Recommended*Versions."); +#ifdef HAVE_MODULE_DIRAUTH char *t; /* Call these functions to produce warnings only. */ t = format_recommended_version_list(options->RecommendedClientVersions, 1); tor_free(t); t = format_recommended_version_list(options->RecommendedServerVersions, 1); tor_free(t); +#endif if (options->UseEntryGuards) { log_info(LD_CONFIG, "Authoritative directory servers can't set " @@ -3796,6 +3842,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "http://freehaven.net/anonbib/#hs-attack06 for details."); } + if (options->NumPrimaryGuards && options->NumEntryGuards && + options->NumEntryGuards > options->NumPrimaryGuards) { + REJECT("NumEntryGuards must not be greater than NumPrimaryGuards."); + } + if (options->EntryNodes && routerset_is_list(options->EntryNodes) && (routerset_len(options->EntryNodes) == 1) && @@ -3918,15 +3969,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); - if (options->PortForwarding && options->Sandbox) { - REJECT("PortForwarding is not compatible with Sandbox; at most one can " - "be set"); - } - if (options->PortForwarding && options->NoExec) { - COMPLAIN("Both PortForwarding and NoExec are set; PortForwarding will " - "be ignored."); - } - if (ensure_bandwidth_cap(&options->BandwidthRate, "BandwidthRate", msg) < 0) return -1; @@ -4403,12 +4445,12 @@ options_validate(or_options_t *old_options, or_options_t *options, CHECK_DEFAULT(TestingV3AuthVotingStartOffset); CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability); CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime); - CHECK_DEFAULT(TestingServerDownloadSchedule); - CHECK_DEFAULT(TestingClientDownloadSchedule); - CHECK_DEFAULT(TestingServerConsensusDownloadSchedule); - CHECK_DEFAULT(TestingClientConsensusDownloadSchedule); - CHECK_DEFAULT(TestingBridgeDownloadSchedule); - CHECK_DEFAULT(TestingBridgeBootstrapDownloadSchedule); + CHECK_DEFAULT(TestingServerDownloadInitialDelay); + CHECK_DEFAULT(TestingClientDownloadInitialDelay); + CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay); + CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay); + CHECK_DEFAULT(TestingBridgeDownloadInitialDelay); + CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay); CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest); CHECK_DEFAULT(TestingDirConnectionMaxStall); CHECK_DEFAULT(TestingAuthKeyLifetime); @@ -4506,12 +4548,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "Tor networks!"); } - if (options->TestingEnableTbEmptyEvent && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingEnableTbEmptyEvent may only be changed in testing " - "Tor networks!"); - } - if (options->TestingTorNetwork) { log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node " "almost unusable in the public Tor network, and is " @@ -4655,15 +4691,14 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, if (options->DirCache) { if (total_mem < DIRCACHE_MIN_MEM_BYTES) { if (options->BridgeRelay) { - *msg = tor_strdup("Running a Bridge with less than " - STRINGIFY(DIRCACHE_MIN_MEM_MB) " MB of memory is not " - "recommended."); + tor_asprintf(msg, "Running a Bridge with less than %d MB of memory " + "is not recommended.", DIRCACHE_MIN_MEM_MB); } else { - *msg = tor_strdup("Being a directory cache (default) with less than " - STRINGIFY(DIRCACHE_MIN_MEM_MB) " MB of memory is not " - "recommended and may consume most of the available " - "resources, consider disabling this functionality by " - "setting the DirCache option to 0."); + tor_asprintf(msg, "Being a directory cache (default) with less than " + "%d MB of memory is not recommended and may consume " + "most of the available resources. Consider disabling " + "this functionality by setting the DirCache option " + "to 0.", DIRCACHE_MIN_MEM_MB); } } } else { @@ -8138,7 +8173,10 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_ISOTIME: type = "Time"; break; case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; case CONFIG_TYPE_CSV: type = "CommaList"; break; - case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break; + /* This type accepts more inputs than TimeInterval, but it ignores + * everything after the first entry, so we may as well pretend + * it's a TimeInterval. */ + case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break; case CONFIG_TYPE_LINELIST: type = "LineList"; break; case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break; case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; @@ -8425,3 +8463,17 @@ init_cookie_authentication(const char *fname, const char *header, tor_free(cookie_file_str); return retval; } + +/** + * Return true if any option is set in <b>options</b> to make us behave + * as a client. + */ +int +options_any_client_port_set(const or_options_t *options) +{ + return (options->SocksPort_set || + options->TransPort_set || + options->NATDPort_set || + options->DNSPort_set || + options->HTTPTunnelPort_set); +} diff --git a/src/or/config.h b/src/or/config.h index 1d3c27217e..4b41274434 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -214,6 +214,9 @@ smartlist_t *get_options_from_transport_options_line(const char *line, const char *transport); smartlist_t *get_options_for_server_transport(const char *transport); +/* Port helper functions. */ +int options_any_client_port_set(const or_options_t *options); + #ifdef CONFIG_PRIVATE #define CL_PORT_NO_STREAM_OPTIONS (1u<<0) diff --git a/src/or/confparse.c b/src/or/confparse.c index 64ed9ee6bb..6bab790945 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -162,8 +162,6 @@ config_assign_value(const config_format_t *fmt, void *options, int i, ok; const config_var_t *var; void *lvalue; - int *csv_int; - smartlist_t *csv_str; CONFIG_CHECK(fmt, options); @@ -195,6 +193,30 @@ config_assign_value(const config_format_t *fmt, void *options, *(int *)lvalue = i; break; + case CONFIG_TYPE_CSV_INTERVAL: { + /* We used to have entire smartlists here. But now that all of our + * download schedules use exponential backoff, only the first part + * matters. */ + const char *comma = strchr(c->value, ','); + const char *val = c->value; + char *tmp = NULL; + if (comma) { + tmp = tor_strndup(c->value, comma - c->value); + val = tmp; + } + + i = config_parse_interval(val, &ok); + if (!ok) { + tor_asprintf(msg, + "Interval '%s %s' is malformed or out of bounds.", + c->key, c->value); + return -1; + } + *(int *)lvalue = i; + tor_free(tmp); + break; + } + case CONFIG_TYPE_INTERVAL: { i = config_parse_interval(c->value, &ok); if (!ok) { @@ -298,36 +320,6 @@ config_assign_value(const config_format_t *fmt, void *options, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); break; - case CONFIG_TYPE_CSV_INTERVAL: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp)); - smartlist_clear(*(smartlist_t**)lvalue); - } else { - *(smartlist_t**)lvalue = smartlist_new(); - } - csv_str = smartlist_new(); - smartlist_split_string(csv_str, c->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(csv_str, char *, str) - { - i = config_parse_interval(str, &ok); - if (!ok) { - tor_asprintf(msg, - "Interval in '%s %s' is malformed or out of bounds.", - c->key, c->value); - SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); - smartlist_free(csv_str); - return -1; - } - csv_int = tor_malloc_zero(sizeof(int)); - *csv_int = i; - smartlist_add(*(smartlist_t**)lvalue, csv_int); - } - SMARTLIST_FOREACH_END(str); - SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); - smartlist_free(csv_str); - break; - case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: { @@ -528,7 +520,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, const config_var_t *var; const void *value; config_line_t *result; - smartlist_t *csv_str; tor_assert(options && key); CONFIG_CHECK(fmt, options); @@ -571,6 +562,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, break; } /* fall through */ + case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_UINT: @@ -611,20 +603,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, else result->value = tor_strdup(""); break; - case CONFIG_TYPE_CSV_INTERVAL: - if (*(smartlist_t**)value) { - csv_str = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i) - { - smartlist_add_asprintf(csv_str, "%d", *i); - } - SMARTLIST_FOREACH_END(i); - result->value = smartlist_join_strings(csv_str, ",", 0, NULL); - SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); - smartlist_free(csv_str); - } else - result->value = tor_strdup(""); - break; case CONFIG_TYPE_OBSOLETE: log_fn(LOG_INFO, LD_CONFIG, "You asked me for the value of an obsolete config option '%s'.", @@ -789,6 +767,7 @@ config_clear(const config_format_t *fmt, void *options, case CONFIG_TYPE_ISOTIME: *(time_t*)lvalue = 0; break; + case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_UINT: @@ -816,13 +795,6 @@ config_clear(const config_format_t *fmt, void *options, *(smartlist_t **)lvalue = NULL; } break; - case CONFIG_TYPE_CSV_INTERVAL: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp)); - smartlist_free(*(smartlist_t **)lvalue); - *(smartlist_t **)lvalue = NULL; - } - break; case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: config_free_lines(*(config_line_t **)lvalue); diff --git a/src/or/confparse.h b/src/or/confparse.h index f1f2030343..4b4bf0adb4 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -28,7 +28,9 @@ typedef enum config_type_t { * optional whitespace. */ CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and * optional whitespace, representing intervals in - * seconds, with optional units */ + * seconds, with optional units. We allow + * multiple values here for legacy reasons, but + * ignore every value after the first. */ CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, * mixed with other keywords. */ @@ -62,7 +64,7 @@ typedef union { int *AUTOBOOL; time_t *ISOTIME; smartlist_t **CSV; - smartlist_t **CSV_INTERVAL; + int *CSV_INTERVAL; config_line_t **LINELIST; config_line_t **LINELIST_S; config_line_t **LINELIST_V; diff --git a/src/or/connection.c b/src/or/connection.c index 2a6b10763e..0a2a635096 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2001 Matej Pfajfar. +/* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2017, The Tor Project, Inc. */ @@ -76,6 +76,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" #include "dns.h" @@ -85,6 +86,7 @@ #include "ext_orport.h" #include "geoip.h" #include "main.h" +#include "hibernate.h" #include "hs_common.h" #include "hs_ident.h" #include "nodelist.h" @@ -101,7 +103,6 @@ #include "transports.h" #include "routerparse.h" #include "sandbox.h" -#include "transports.h" #ifdef HAVE_PWD_H #include <pwd.h> @@ -120,8 +121,6 @@ static connection_t *connection_listener_new( static void connection_init(time_t now, connection_t *conn, int type, int socket_family); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_bucket_should_increase(int bucket, - or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); @@ -140,6 +139,8 @@ static const char *proxy_type_to_string(int proxy_type); static int get_proxy_type(void); const tor_addr_t *conn_get_outbound_address(sa_family_t family, const or_options_t *options, unsigned int conn_type); +static void reenable_blocked_connection_init(const or_options_t *options); +static void reenable_blocked_connection_schedule(void); /** The last addresses that our network interface seemed to have been * binding to. We use this as one way to detect when our IP changes. @@ -460,8 +461,8 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family) } conn->timestamp_created = now; - conn->timestamp_lastread = now; - conn->timestamp_lastwritten = now; + conn->timestamp_last_read_allowed = now; + conn->timestamp_last_write_allowed = now; } /** Create a link between <b>conn_a</b> and <b>conn_b</b>. */ @@ -775,8 +776,8 @@ connection_close_immediate(connection_t *conn) connection_unregister_events(conn); /* Prevent the event from getting unblocked. */ - conn->read_blocked_on_bw = - conn->write_blocked_on_bw = 0; + conn->read_blocked_on_bw = 0; + conn->write_blocked_on_bw = 0; if (SOCKET_OK(conn->s)) tor_close_socket(conn->s); @@ -859,7 +860,7 @@ connection_mark_for_close_internal_, (connection_t *conn, /* in case we're going to be held-open-til-flushed, reset * the number of seconds since last successful write, so * we get our whole 15 seconds */ - conn->timestamp_lastwritten = time(NULL); + conn->timestamp_last_write_allowed = time(NULL); } /** Find each connection that has hold_open_until_flushed set to @@ -881,7 +882,7 @@ connection_expire_held_open(void) */ if (conn->hold_open_until_flushed) { tor_assert(conn->marked_for_close); - if (now - conn->timestamp_lastwritten >= 15) { + if (now - conn->timestamp_last_write_allowed >= 15) { int severity; if (conn->type == CONN_TYPE_EXIT || (conn->type == CONN_TYPE_DIR && @@ -1764,13 +1765,13 @@ connection_connect_sockaddr,(connection_t *conn, tor_assert(sa); tor_assert(socket_error); - if (get_options()->DisableNetwork) { - /* We should never even try to connect anyplace if DisableNetwork is set. - * Warn if we do, and refuse to make the connection. + if (net_is_completely_disabled()) { + /* We should never even try to connect anyplace if the network is + * completely shut off. * - * We only check DisableNetwork here, not we_are_hibernating(), since - * we'll still try to fulfill client requests sometimes in the latter case - * (if it is soft hibernation) */ + * (We don't check net_is_disabled() here, since we still sometimes + * want to open connections when we're in soft hibernation.) + */ static ratelim_t disablenet_violated = RATELIM_INIT(30*60); *socket_error = SOCK_ERRNO(ENETUNREACH); log_fn_ratelim(&disablenet_violated, LOG_WARN, LD_BUG, @@ -2820,10 +2821,10 @@ connection_is_rate_limited(connection_t *conn) return 1; } -/** Did either global write bucket run dry last second? If so, - * we are likely to run dry again this second, so be stingy with the - * tokens we just put in. */ -static int write_buckets_empty_last_second = 0; +/** When was either global write bucket last empty? If this was recent, then + * we're probably low on bandwidth, and we should be stingy with our bandwidth + * usage. */ +static time_t write_buckets_last_empty_at = -100; /** How many seconds of no active local circuits will make the * connection revert to the "relayed" bandwidth class? */ @@ -2851,25 +2852,25 @@ connection_counts_as_relayed_traffic(connection_t *conn, time_t now) * write many of them or just a few; and <b>conn_bucket</b> (if * non-negative) provides an upper limit for our answer. */ static ssize_t -connection_bucket_round_robin(int base, int priority, - ssize_t global_bucket, ssize_t conn_bucket) +connection_bucket_get_share(int base, int priority, + ssize_t global_bucket_val, ssize_t conn_bucket) { ssize_t at_most; ssize_t num_bytes_high = (priority ? 32 : 16) * base; ssize_t num_bytes_low = (priority ? 4 : 2) * base; - /* Do a rudimentary round-robin so one circuit can't hog a connection. + /* Do a rudimentary limiting so one circuit can't hog a connection. * Pick at most 32 cells, at least 4 cells if possible, and if we're in * the middle pick 1/8 of the available bandwidth. */ - at_most = global_bucket / 8; + at_most = global_bucket_val / 8; at_most -= (at_most % base); /* round down */ if (at_most > num_bytes_high) /* 16 KB, or 8 KB for low-priority */ at_most = num_bytes_high; else if (at_most < num_bytes_low) /* 2 KB, or 1 KB for low-priority */ at_most = num_bytes_low; - if (at_most > global_bucket) - at_most = global_bucket; + if (at_most > global_bucket_val) + at_most = global_bucket_val; if (conn_bucket >= 0 && at_most > conn_bucket) at_most = conn_bucket; @@ -2885,13 +2886,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now) { int base = RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; - int conn_bucket = -1; - int global_bucket = global_read_bucket; + ssize_t conn_bucket = -1; + size_t global_bucket_val = token_bucket_rw_get_read(&global_bucket); if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_OPEN) - conn_bucket = or_conn->read_bucket; + conn_bucket = token_bucket_rw_get_read(&or_conn->bucket); base = get_cell_network_size(or_conn->wide_circ_ids); } @@ -2900,12 +2901,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now) return conn_bucket>=0 ? conn_bucket : 1<<14; } - if (connection_counts_as_relayed_traffic(conn, now) && - global_relayed_read_bucket <= global_read_bucket) - global_bucket = global_relayed_read_bucket; + if (connection_counts_as_relayed_traffic(conn, now)) { + size_t relayed = token_bucket_rw_get_read(&global_relayed_bucket); + global_bucket_val = MIN(global_bucket_val, relayed); + } - return connection_bucket_round_robin(base, priority, - global_bucket, conn_bucket); + return connection_bucket_get_share(base, priority, + global_bucket_val, conn_bucket); } /** How many bytes at most can we write onto this connection? */ @@ -2914,8 +2916,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now) { int base = RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; - int conn_bucket = (int)conn->outbuf_flushlen; - int global_bucket = global_write_bucket; + size_t conn_bucket = conn->outbuf_flushlen; + size_t global_bucket_val = token_bucket_rw_get_write(&global_bucket); if (!connection_is_rate_limited(conn)) { /* be willing to write to local conns even if our buckets are empty */ @@ -2923,22 +2925,21 @@ connection_bucket_write_limit(connection_t *conn, time_t now) } if (connection_speaks_cells(conn)) { - /* use the per-conn write limit if it's lower, but if it's less - * than zero just use zero */ + /* use the per-conn write limit if it's lower */ or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_OPEN) - if (or_conn->write_bucket < conn_bucket) - conn_bucket = or_conn->write_bucket >= 0 ? - or_conn->write_bucket : 0; + conn_bucket = MIN(conn_bucket, + token_bucket_rw_get_write(&or_conn->bucket)); base = get_cell_network_size(or_conn->wide_circ_ids); } - if (connection_counts_as_relayed_traffic(conn, now) && - global_relayed_write_bucket <= global_write_bucket) - global_bucket = global_relayed_write_bucket; + if (connection_counts_as_relayed_traffic(conn, now)) { + size_t relayed = token_bucket_rw_get_write(&global_relayed_bucket); + global_bucket_val = MIN(global_bucket_val, relayed); + } - return connection_bucket_round_robin(base, priority, - global_bucket, conn_bucket); + return connection_bucket_get_share(base, priority, + global_bucket_val, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -2963,27 +2964,31 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int global_write_bucket_low(connection_t *conn, size_t attempt, int priority) { - int smaller_bucket = global_write_bucket < global_relayed_write_bucket ? - global_write_bucket : global_relayed_write_bucket; + size_t smaller_bucket = + MIN(token_bucket_rw_get_write(&global_bucket), + token_bucket_rw_get_write(&global_relayed_bucket)); if (authdir_mode(get_options()) && priority>1) return 0; /* there's always room to answer v2 if we're an auth dir */ if (!connection_is_rate_limited(conn)) return 0; /* local conns don't get limited */ - if (smaller_bucket < (int)attempt) + if (smaller_bucket < attempt) return 1; /* not enough space no matter the priority */ - if (write_buckets_empty_last_second) - return 1; /* we're already hitting our limits, no more please */ + { + const time_t diff = approx_time() - write_buckets_last_empty_at; + if (diff <= 1) + return 1; /* we're already hitting our limits, no more please */ + } if (priority == 1) { /* old-style v1 query */ /* Could we handle *two* of these requests within the next two seconds? */ const or_options_t *options = get_options(); - int64_t can_write = (int64_t)smaller_bucket + size_t can_write = (size_t) (smaller_bucket + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate : - options->BandwidthRate); - if (can_write < 2*(int64_t)attempt) + options->BandwidthRate)); + if (can_write < 2*attempt) return 1; } else { /* v2 query */ /* no further constraints yet */ @@ -2991,6 +2996,10 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } +/** When did we last tell the accounting subsystem about transmitted + * bandwidth? */ +static time_t last_recorded_accounting_at = 0; + /** Helper: adjusts our bandwidth history and informs the controller as * appropriate, given that we have just read <b>num_read</b> bytes and written * <b>num_written</b> bytes on <b>conn</b>. */ @@ -3021,59 +3030,22 @@ record_num_bytes_transferred_impl(connection_t *conn, } if (conn->type == CONN_TYPE_EXIT) rep_hist_note_exit_bytes(conn->port, num_written, num_read); -} -/** Helper: convert given <b>tvnow</b> time value to milliseconds since - * midnight. */ -static uint32_t -msec_since_midnight(const struct timeval *tvnow) -{ - return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) + - ((uint32_t)tvnow->tv_usec / (uint32_t)1000L)); -} + /* Remember these bytes towards statistics. */ + stats_increment_bytes_read_and_written(num_read, num_written); -/** Helper: return the time in milliseconds since <b>last_empty_time</b> - * when a bucket ran empty that previously had <b>tokens_before</b> tokens - * now has <b>tokens_after</b> tokens after refilling at timestamp - * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since - * last refilling that bucket. Return 0 if the bucket has not been empty - * since the last refill or has not been refilled. */ -uint32_t -bucket_millis_empty(int tokens_before, uint32_t last_empty_time, - int tokens_after, int milliseconds_elapsed, - const struct timeval *tvnow) -{ - uint32_t result = 0, refilled; - if (tokens_before <= 0 && tokens_after > tokens_before) { - refilled = msec_since_midnight(tvnow); - result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) % - (86400L * 1000L)); - if (result > (uint32_t)milliseconds_elapsed) - result = (uint32_t)milliseconds_elapsed; - } - return result; -} - -/** Check if a bucket which had <b>tokens_before</b> tokens and which got - * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run - * out of tokens, and if so, note the milliseconds since midnight in - * <b>timestamp_var</b> for the next TB_EMPTY event. */ -void -connection_buckets_note_empty_ts(uint32_t *timestamp_var, - int tokens_before, size_t tokens_removed, - const struct timeval *tvnow) -{ - if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed) - *timestamp_var = msec_since_midnight(tvnow); + /* Remember these bytes towards accounting. */ + if (accounting_is_enabled(get_options())) { + if (now > last_recorded_accounting_at && last_recorded_accounting_at) { + accounting_add_bytes(num_read, num_written, + (int)(now - last_recorded_accounting_at)); + } else { + accounting_add_bytes(num_read, num_written, 0); + } + last_recorded_accounting_at = now; + } } -/** Last time at which the global or relay buckets were emptied in msec - * since midnight. */ -static uint32_t global_relayed_read_emptied = 0, - global_relayed_write_emptied = 0, - global_read_emptied = 0, - global_write_emptied = 0; - /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes * onto <b>conn</b>. Decrement buckets appropriately. */ static void @@ -3098,45 +3070,54 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ - /* If one or more of our token buckets ran dry just now, note the - * timestamp for TB_EMPTY events. */ - if (get_options()->TestingEnableTbEmptyEvent) { - struct timeval tvnow; - tor_gettimeofday_cached(&tvnow); - if (connection_counts_as_relayed_traffic(conn, now)) { - connection_buckets_note_empty_ts(&global_relayed_read_emptied, - global_relayed_read_bucket, num_read, &tvnow); - connection_buckets_note_empty_ts(&global_relayed_write_emptied, - global_relayed_write_bucket, num_written, &tvnow); - } - connection_buckets_note_empty_ts(&global_read_emptied, - global_read_bucket, num_read, &tvnow); - connection_buckets_note_empty_ts(&global_write_emptied, - global_write_bucket, num_written, &tvnow); - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { - or_connection_t *or_conn = TO_OR_CONN(conn); - connection_buckets_note_empty_ts(&or_conn->read_emptied_time, - or_conn->read_bucket, num_read, &tvnow); - connection_buckets_note_empty_ts(&or_conn->write_emptied_time, - or_conn->write_bucket, num_written, &tvnow); - } + unsigned flags = 0; + if (connection_counts_as_relayed_traffic(conn, now)) { + flags = token_bucket_rw_dec(&global_relayed_bucket, num_read, num_written); } + flags |= token_bucket_rw_dec(&global_bucket, num_read, num_written); - if (connection_counts_as_relayed_traffic(conn, now)) { - global_relayed_read_bucket -= (int)num_read; - global_relayed_write_bucket -= (int)num_written; + if (flags & TB_WRITE) { + write_buckets_last_empty_at = now; } - global_read_bucket -= (int)num_read; - global_write_bucket -= (int)num_written; if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { - TO_OR_CONN(conn)->read_bucket -= (int)num_read; - TO_OR_CONN(conn)->write_bucket -= (int)num_written; + or_connection_t *or_conn = TO_OR_CONN(conn); + token_bucket_rw_dec(&or_conn->bucket, num_read, num_written); } } +/** + * Mark <b>conn</b> as needing to stop reading because bandwidth has been + * exhausted. If <b>is_global_bw</b>, it is closing because global bandwidth + * limit has been exhausted. Otherwise, it is closing because its own + * bandwidth limit has been exhausted. + */ +void +connection_read_bw_exhausted(connection_t *conn, bool is_global_bw) +{ + (void)is_global_bw; + conn->read_blocked_on_bw = 1; + connection_stop_reading(conn); + reenable_blocked_connection_schedule(); +} + +/** + * Mark <b>conn</b> as needing to stop reading because write bandwidth has + * been exhausted. If <b>is_global_bw</b>, it is closing because global + * bandwidth limit has been exhausted. Otherwise, it is closing because its + * own bandwidth limit has been exhausted. +*/ +void +connection_write_bw_exhausted(connection_t *conn, bool is_global_bw) +{ + (void)is_global_bw; + conn->write_blocked_on_bw = 1; + connection_stop_writing(conn); + reenable_blocked_connection_schedule(); +} + /** If we have exhausted our global buckets, or the buckets for conn, * stop reading. */ -static void +void connection_consider_empty_read_buckets(connection_t *conn) { const char *reason; @@ -3144,26 +3125,28 @@ connection_consider_empty_read_buckets(connection_t *conn) if (!connection_is_rate_limited(conn)) return; /* Always okay. */ - if (global_read_bucket <= 0) { + int is_global = 1; + + if (token_bucket_rw_get_read(&global_bucket) <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && - global_relayed_read_bucket <= 0) { + token_bucket_rw_get_read(&global_relayed_bucket) <= 0) { reason = "global relayed read bucket exhausted. Pausing."; } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && - TO_OR_CONN(conn)->read_bucket <= 0) { + token_bucket_rw_get_read(&TO_OR_CONN(conn)->bucket) <= 0) { reason = "connection read bucket exhausted. Pausing."; + is_global = false; } else return; /* all good, no need to stop it */ LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); - conn->read_blocked_on_bw = 1; - connection_stop_reading(conn); + connection_read_bw_exhausted(conn, is_global); } /** If we have exhausted our global buckets, or the buckets for conn, * stop writing. */ -static void +void connection_consider_empty_write_buckets(connection_t *conn) { const char *reason; @@ -3171,233 +3154,166 @@ connection_consider_empty_write_buckets(connection_t *conn) if (!connection_is_rate_limited(conn)) return; /* Always okay. */ - if (global_write_bucket <= 0) { + bool is_global = true; + if (token_bucket_rw_get_write(&global_bucket) <= 0) { reason = "global write bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && - global_relayed_write_bucket <= 0) { + token_bucket_rw_get_write(&global_relayed_bucket) <= 0) { reason = "global relayed write bucket exhausted. Pausing."; } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && - TO_OR_CONN(conn)->write_bucket <= 0) { + token_bucket_rw_get_write(&TO_OR_CONN(conn)->bucket) <= 0) { reason = "connection write bucket exhausted. Pausing."; + is_global = false; } else return; /* all good, no need to stop it */ LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); - conn->write_blocked_on_bw = 1; - connection_stop_writing(conn); + connection_write_bw_exhausted(conn, is_global); } -/** Initialize the global read bucket to options-\>BandwidthBurst. */ +/** Initialize the global buckets to the values configured in the + * options */ void connection_bucket_init(void) { const or_options_t *options = get_options(); - /* start it at max traffic */ - global_read_bucket = (int)options->BandwidthBurst; - global_write_bucket = (int)options->BandwidthBurst; + const uint32_t now_ts = monotime_coarse_get_stamp(); + token_bucket_rw_init(&global_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst, + now_ts); if (options->RelayBandwidthRate) { - global_relayed_read_bucket = (int)options->RelayBandwidthBurst; - global_relayed_write_bucket = (int)options->RelayBandwidthBurst; + token_bucket_rw_init(&global_relayed_bucket, + (int32_t)options->RelayBandwidthRate, + (int32_t)options->RelayBandwidthBurst, + now_ts); } else { - global_relayed_read_bucket = (int)options->BandwidthBurst; - global_relayed_write_bucket = (int)options->BandwidthBurst; + token_bucket_rw_init(&global_relayed_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst, + now_ts); } + + reenable_blocked_connection_init(options); } -/** Refill a single <b>bucket</b> called <b>name</b> with bandwidth rate per - * second <b>rate</b> and bandwidth burst <b>burst</b>, assuming that - * <b>milliseconds_elapsed</b> milliseconds have passed since the last - * call. */ -static void -connection_bucket_refill_helper(int *bucket, int rate, int burst, - int milliseconds_elapsed, - const char *name) +/** Update the global connection bucket settings to a new value. */ +void +connection_bucket_adjust(const or_options_t *options) { - int starting_bucket = *bucket; - if (starting_bucket < burst && milliseconds_elapsed > 0) { - int64_t incr = (((int64_t)rate) * milliseconds_elapsed) / 1000; - if ((burst - starting_bucket) < incr) { - *bucket = burst; /* We would overflow the bucket; just set it to - * the maximum. */ - } else { - *bucket += (int)incr; - if (*bucket > burst || *bucket < starting_bucket) { - /* If we overflow the burst, or underflow our starting bucket, - * cap the bucket value to burst. */ - /* XXXX this might be redundant now, but it doesn't show up - * in profiles. Remove it after analysis. */ - *bucket = burst; - } - } - log_debug(LD_NET,"%s now %d.", name, *bucket); + token_bucket_rw_adjust(&global_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst); + if (options->RelayBandwidthRate) { + token_bucket_rw_adjust(&global_relayed_bucket, + (int32_t)options->RelayBandwidthRate, + (int32_t)options->RelayBandwidthBurst); + } else { + token_bucket_rw_adjust(&global_relayed_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst); } } -/** Time has passed; increment buckets appropriately. */ -void -connection_bucket_refill(int milliseconds_elapsed, time_t now) +/** + * Cached value of the last coarse-timestamp when we refilled the + * global buckets. + */ +static uint32_t last_refilled_global_buckets_ts=0; +/** + * Refill the token buckets for a single connection <b>conn</b>, and the + * global token buckets as appropriate. Requires that <b>now_ts</b> is + * the time in coarse timestamp units. + */ +static void +connection_bucket_refill_single(connection_t *conn, uint32_t now_ts) { - const or_options_t *options = get_options(); - smartlist_t *conns = get_connection_array(); - int bandwidthrate, bandwidthburst, relayrate, relayburst; + /* Note that we only check for equality here: the underlying + * token bucket functions can handle moving backwards in time if they + * need to. */ + if (now_ts != last_refilled_global_buckets_ts) { + token_bucket_rw_refill(&global_bucket, now_ts); + token_bucket_rw_refill(&global_relayed_bucket, now_ts); + last_refilled_global_buckets_ts = now_ts; + } - int prev_global_read = global_read_bucket; - int prev_global_write = global_write_bucket; - int prev_relay_read = global_relayed_read_bucket; - int prev_relay_write = global_relayed_write_bucket; - struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */ + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { + or_connection_t *or_conn = TO_OR_CONN(conn); + token_bucket_rw_refill(&or_conn->bucket, now_ts); + } +} - bandwidthrate = (int)options->BandwidthRate; - bandwidthburst = (int)options->BandwidthBurst; +/** + * Event to re-enable all connections that were previously blocked on read or + * write. + */ +static mainloop_event_t *reenable_blocked_connections_ev = NULL; - if (options->RelayBandwidthRate) { - relayrate = (int)options->RelayBandwidthRate; - relayburst = (int)options->RelayBandwidthBurst; - } else { - relayrate = bandwidthrate; - relayburst = bandwidthburst; - } - - tor_assert(milliseconds_elapsed >= 0); - - write_buckets_empty_last_second = - global_relayed_write_bucket <= 0 || global_write_bucket <= 0; - - /* refill the global buckets */ - connection_bucket_refill_helper(&global_read_bucket, - bandwidthrate, bandwidthburst, - milliseconds_elapsed, - "global_read_bucket"); - connection_bucket_refill_helper(&global_write_bucket, - bandwidthrate, bandwidthburst, - milliseconds_elapsed, - "global_write_bucket"); - connection_bucket_refill_helper(&global_relayed_read_bucket, - relayrate, relayburst, - milliseconds_elapsed, - "global_relayed_read_bucket"); - connection_bucket_refill_helper(&global_relayed_write_bucket, - relayrate, relayburst, - milliseconds_elapsed, - "global_relayed_write_bucket"); - - /* If buckets were empty before and have now been refilled, tell any - * interested controllers. */ - if (get_options()->TestingEnableTbEmptyEvent) { - uint32_t global_read_empty_time, global_write_empty_time, - relay_read_empty_time, relay_write_empty_time; - tor_gettimeofday_cached(&tvnow); - global_read_empty_time = bucket_millis_empty(prev_global_read, - global_read_emptied, global_read_bucket, - milliseconds_elapsed, &tvnow); - global_write_empty_time = bucket_millis_empty(prev_global_write, - global_write_emptied, global_write_bucket, - milliseconds_elapsed, &tvnow); - control_event_tb_empty("GLOBAL", global_read_empty_time, - global_write_empty_time, milliseconds_elapsed); - relay_read_empty_time = bucket_millis_empty(prev_relay_read, - global_relayed_read_emptied, - global_relayed_read_bucket, - milliseconds_elapsed, &tvnow); - relay_write_empty_time = bucket_millis_empty(prev_relay_write, - global_relayed_write_emptied, - global_relayed_write_bucket, - milliseconds_elapsed, &tvnow); - control_event_tb_empty("RELAY", relay_read_empty_time, - relay_write_empty_time, milliseconds_elapsed); - } - - /* refill the per-connection buckets */ - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (connection_speaks_cells(conn)) { - or_connection_t *or_conn = TO_OR_CONN(conn); - int orbandwidthrate = or_conn->bandwidthrate; - int orbandwidthburst = or_conn->bandwidthburst; - - int prev_conn_read = or_conn->read_bucket; - int prev_conn_write = or_conn->write_bucket; - - if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { - connection_bucket_refill_helper(&or_conn->read_bucket, - orbandwidthrate, - orbandwidthburst, - milliseconds_elapsed, - "or_conn->read_bucket"); - } - if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { - connection_bucket_refill_helper(&or_conn->write_bucket, - orbandwidthrate, - orbandwidthburst, - milliseconds_elapsed, - "or_conn->write_bucket"); - } +/** True iff reenable_blocked_connections_ev is currently scheduled. */ +static int reenable_blocked_connections_is_scheduled = 0; - /* If buckets were empty before and have now been refilled, tell any - * interested controllers. */ - if (get_options()->TestingEnableTbEmptyEvent) { - char *bucket; - uint32_t conn_read_empty_time, conn_write_empty_time; - tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT, - U64_PRINTF_ARG(or_conn->base_.global_identifier)); - conn_read_empty_time = bucket_millis_empty(prev_conn_read, - or_conn->read_emptied_time, - or_conn->read_bucket, - milliseconds_elapsed, &tvnow); - conn_write_empty_time = bucket_millis_empty(prev_conn_write, - or_conn->write_emptied_time, - or_conn->write_bucket, - milliseconds_elapsed, &tvnow); - control_event_tb_empty(bucket, conn_read_empty_time, - conn_write_empty_time, - milliseconds_elapsed); - tor_free(bucket); - } - } +/** Delay after which to run reenable_blocked_connections_ev. */ +static struct timeval reenable_blocked_connections_delay; - if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */ - && global_read_bucket > 0 /* and we're allowed to read */ - && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_read_bucket > 0) /* even if we're relayed traffic */ - && (!connection_speaks_cells(conn) || - conn->state != OR_CONN_STATE_OPEN || - TO_OR_CONN(conn)->read_bucket > 0)) { - /* and either a non-cell conn or a cell conn with non-empty bucket */ - LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, - "waking up conn (fd %d) for read", (int)conn->s)); - conn->read_blocked_on_bw = 0; +/** + * Re-enable all connections that were previously blocked on read or write. + * This event is scheduled after enough time has elapsed to be sure + * that the buckets will refill when the connections have something to do. + */ +static void +reenable_blocked_connections_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) { + if (conn->read_blocked_on_bw == 1) { connection_start_reading(conn); + conn->read_blocked_on_bw = 0; } - - if (conn->write_blocked_on_bw == 1 - && global_write_bucket > 0 /* and we're allowed to write */ - && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0) /* even if it's relayed traffic */ - && (!connection_speaks_cells(conn) || - conn->state != OR_CONN_STATE_OPEN || - TO_OR_CONN(conn)->write_bucket > 0)) { - LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, - "waking up conn (fd %d) for write", (int)conn->s)); - conn->write_blocked_on_bw = 0; + if (conn->write_blocked_on_bw == 1) { connection_start_writing(conn); + conn->write_blocked_on_bw = 0; } } SMARTLIST_FOREACH_END(conn); + + reenable_blocked_connections_is_scheduled = 0; } -/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we - * should add another pile of tokens to it? +/** + * Initialize the mainloop event that we use to wake up connections that + * find themselves blocked on bandwidth. */ -static int -connection_bucket_should_increase(int bucket, or_connection_t *conn) +static void +reenable_blocked_connection_init(const or_options_t *options) { - tor_assert(conn); - - if (conn->base_.state != OR_CONN_STATE_OPEN) - return 0; /* only open connections play the rate limiting game */ - if (bucket >= conn->bandwidthburst) - return 0; + if (! reenable_blocked_connections_ev) { + reenable_blocked_connections_ev = + mainloop_event_new(reenable_blocked_connections_cb, NULL); + reenable_blocked_connections_is_scheduled = 0; + } + time_t sec = options->TokenBucketRefillInterval / 1000; + int msec = (options->TokenBucketRefillInterval % 1000); + reenable_blocked_connections_delay.tv_sec = sec; + reenable_blocked_connections_delay.tv_usec = msec * 1000; +} - return 1; +/** + * Called when we have blocked a connection for being low on bandwidth: + * schedule an event to reenable such connections, if it is not already + * scheduled. + */ +static void +reenable_blocked_connection_schedule(void) +{ + if (reenable_blocked_connections_is_scheduled) + return; + if (BUG(reenable_blocked_connections_ev == NULL)) { + reenable_blocked_connection_init(get_options()); + } + mainloop_event_schedule(reenable_blocked_connections_ev, + &reenable_blocked_connections_delay); + reenable_blocked_connections_is_scheduled = 1; } /** Read bytes from conn-\>s and process them. @@ -3418,7 +3334,9 @@ connection_handle_read_impl(connection_t *conn) if (conn->marked_for_close) return 0; /* do nothing */ - conn->timestamp_lastread = approx_time(); + conn->timestamp_last_read_allowed = approx_time(); + + connection_bucket_refill_single(conn, monotime_coarse_get_stamp()); switch (conn->type) { case CONN_TYPE_OR_LISTENER: @@ -3525,8 +3443,7 @@ int connection_handle_read(connection_t *conn) { int res; - - tor_gettimeofday_cache_clear(); + update_current_time(time(NULL)); res = connection_handle_read_impl(conn); return res; } @@ -3678,25 +3595,15 @@ connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read, /* change *max_to_read */ *max_to_read = at_most - n_read; - /* Update edge_conn->n_read and ocirc->n_read_circ_bw */ + /* Update edge_conn->n_read */ if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - circuit_t *circ = circuit_get_by_edge_conn(edge_conn); - origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) edge_conn->n_read += (int)n_read; else edge_conn->n_read = UINT32_MAX; - - if (circ && CIRCUIT_IS_ORIGIN(circ)) { - ocirc = TO_ORIGIN_CIRCUIT(circ); - if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read)) - ocirc->n_read_circ_bw += (int)n_read; - else - ocirc->n_read_circ_bw = UINT32_MAX; - } } /* If CONN_BW events are enabled, update conn->n_read_conn_bw for @@ -3782,7 +3689,6 @@ connection_outbuf_too_full(connection_t *conn) * This should help fix slow upload rates. */ static void - update_send_buffer_size(tor_socket_t sock) { #ifdef _WIN32 @@ -3819,7 +3725,7 @@ update_send_buffer_size(tor_socket_t sock) * when libevent tells us that conn wants to write, or below * from connection_buf_add() when an entire TLS record is ready. * - * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf + * Update <b>conn</b>-\>timestamp_last_write_allowed to now, and call flush_buf * or flush_buf_tls appropriately. If it succeeds and there are no more * more bytes on <b>conn</b>-\>outbuf, then call connection_finished_flushing * on it too. @@ -3852,7 +3758,9 @@ connection_handle_write_impl(connection_t *conn, int force) return 0; } - conn->timestamp_lastwritten = now; + conn->timestamp_last_write_allowed = now; + + connection_bucket_refill_single(conn, monotime_coarse_get_stamp()); /* Sometimes, "writable" means "connected". */ if (connection_state_is_connecting(conn)) { @@ -3967,8 +3875,7 @@ connection_handle_write_impl(connection_t *conn, int force) /* Make sure to avoid a loop if the receive buckets are empty. */ log_debug(LD_NET,"wanted read."); if (!connection_is_reading(conn)) { - connection_stop_writing(conn); - conn->write_blocked_on_bw = 1; + connection_write_bw_exhausted(conn, true); /* we'll start reading again when we get more tokens in our * read bucket; then we'll start writing again too. */ @@ -4014,22 +3921,12 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - circuit_t *circ = circuit_get_by_edge_conn(edge_conn); - origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) edge_conn->n_written += (int)n_written; else edge_conn->n_written = UINT32_MAX; - - if (circ && CIRCUIT_IS_ORIGIN(circ)) { - ocirc = TO_ORIGIN_CIRCUIT(circ); - if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written)) - ocirc->n_written_circ_bw += (int)n_written; - else - ocirc->n_written_circ_bw = UINT32_MAX; - } } /* If CONN_BW events are enabled, update conn->n_written_conn_bw for @@ -4089,7 +3986,7 @@ int connection_handle_write(connection_t *conn, int force) { int res; - tor_gettimeofday_cache_clear(); + update_current_time(time(NULL)); conn->in_connection_handle_write = 1; res = connection_handle_write_impl(conn, force); conn->in_connection_handle_write = 0; @@ -4528,8 +4425,6 @@ alloc_http_authenticator(const char *authenticator) static void client_check_address_changed(tor_socket_t sock) { - struct sockaddr_storage out_sockaddr; - socklen_t out_addr_len = (socklen_t) sizeof(out_sockaddr); tor_addr_t out_addr, iface_addr; tor_addr_t **last_interface_ip_ptr; sa_family_t family; @@ -4537,13 +4432,12 @@ client_check_address_changed(tor_socket_t sock) if (!outgoing_addrs) outgoing_addrs = smartlist_new(); - if (getsockname(sock, (struct sockaddr*)&out_sockaddr, &out_addr_len)<0) { + if (tor_addr_from_getsockname(&out_addr, sock) < 0) { int e = tor_socket_errno(sock); log_warn(LD_NET, "getsockname() to check for address change failed: %s", tor_socket_strerror(e)); return; } - tor_addr_from_sockaddr(&out_addr, (struct sockaddr*)&out_sockaddr, NULL); family = tor_addr_family(&out_addr); if (family == AF_INET) @@ -5331,6 +5225,11 @@ connection_free_all(void) tor_free(last_interface_ipv4); tor_free(last_interface_ipv6); + last_recorded_accounting_at = 0; + + mainloop_event_free(reenable_blocked_connections_ev); + reenable_blocked_connections_is_scheduled = 0; + memset(&reenable_blocked_connections_delay, 0, sizeof(struct timeval)); } /** Log a warning, and possibly emit a control event, that <b>received</b> came @@ -5338,10 +5237,10 @@ connection_free_all(void) * that we had more faith in and therefore the warning level should have higher * severity. */ -void -clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted, - log_domain_mask_t domain, const char *received, - const char *source) +MOCK_IMPL(void, +clock_skew_warning, (const connection_t *conn, long apparent_skew, int trusted, + log_domain_mask_t domain, const char *received, + const char *source)) { char dbuf[64]; char *ext_source = NULL, *warn = NULL; diff --git a/src/or/connection.h b/src/or/connection.h index 6bc5a7cfd0..ad3129c9d8 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -122,7 +122,13 @@ void connection_mark_all_noncontrol_connections(void); ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); void connection_bucket_init(void); -void connection_bucket_refill(int seconds_elapsed, time_t now); +void connection_bucket_adjust(const or_options_t *options); +void connection_bucket_refill_all(time_t now, + uint32_t now_ts); +void connection_read_bw_exhausted(connection_t *conn, bool is_global_bw); +void connection_write_bw_exhausted(connection_t *conn, bool is_global_bw); +void connection_consider_empty_read_buckets(connection_t *conn); +void connection_consider_empty_write_buckets(connection_t *conn); int connection_handle_read(connection_t *conn); @@ -248,9 +254,10 @@ void assert_connection_ok(connection_t *conn, time_t now); int connection_or_nonopen_was_started_here(or_connection_t *conn); void connection_dump_buffer_mem_stats(int severity); -void clock_skew_warning(const connection_t *conn, long apparent_skew, - int trusted, log_domain_mask_t domain, - const char *received, const char *source); +MOCK_DECL(void, clock_skew_warning, + (const connection_t *conn, long apparent_skew, int trusted, + log_domain_mask_t domain, const char *received, + const char *source)); /** Check if a connection is on the way out so the OOS handler doesn't try * to kill more than it needs. */ @@ -272,13 +279,6 @@ void connection_check_oos(int n_socks, int failed); STATIC void connection_free_minimal(connection_t *conn); /* Used only by connection.c and test*.c */ -uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time, - int tokens_after, int milliseconds_elapsed, - const struct timeval *tvnow); -void connection_buckets_note_empty_ts(uint32_t *timestamp_var, - int tokens_before, - size_t tokens_removed, - const struct timeval *tvnow); MOCK_DECL(STATIC int,connection_connect_sockaddr, (connection_t *conn, const struct sockaddr *sa, diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 96b4700e4b..88e346e9b8 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -70,6 +70,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_util.h" #include "dns.h" #include "dnsserv.h" #include "directory.h" @@ -429,6 +430,12 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason) if (circ && !circ->marked_for_close) { log_debug(LD_EDGE,"Sending end on conn (fd "TOR_SOCKET_T_FORMAT").", conn->base_.s); + + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ); + connection_half_edge_add(conn, origin_circ); + } + connection_edge_send_command(conn, RELAY_COMMAND_END, payload, payload_len); /* We'll log warn if the connection was an hidden service and couldn't be @@ -445,6 +452,236 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason) return 0; } +/** + * Helper function for bsearch. + * + * As per smartlist_bsearch, return < 0 if key preceeds member, + * > 0 if member preceeds key, and 0 if they are equal. + * + * This is equivalent to subtraction of the values of key - member + * (why does no one ever say that explicitly?). + */ +static int +connection_half_edge_compare_bsearch(const void *key, const void **member) +{ + const half_edge_t *e2; + tor_assert(key); + tor_assert(member && *(half_edge_t**)member); + e2 = *(const half_edge_t **)member; + + return *(const streamid_t*)key - e2->stream_id; +} + +/** Total number of half_edge_t objects allocated */ +static size_t n_half_conns_allocated = 0; + +/** + * Add a half-closed connection to the list, to watch for activity. + * + * These connections are removed from the list upon receiving an end + * cell. + */ +STATIC void +connection_half_edge_add(const edge_connection_t *conn, + origin_circuit_t *circ) +{ + half_edge_t *half_conn = NULL; + int insert_at = 0; + int ignored; + + /* Double-check for re-insertion. This should not happen, + * but this check is cheap compared to the sort anyway */ + if (connection_half_edge_find_stream_id(circ->half_streams, + conn->stream_id)) { + log_warn(LD_BUG, "Duplicate stream close for stream %d on circuit %d", + conn->stream_id, circ->global_identifier); + return; + } + + half_conn = tor_malloc_zero(sizeof(half_edge_t)); + ++n_half_conns_allocated; + + if (!circ->half_streams) { + circ->half_streams = smartlist_new(); + } + + half_conn->stream_id = conn->stream_id; + + // How many sendme's should I expect? + half_conn->sendmes_pending = + (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT; + + // Is there a connected cell pending? + half_conn->connected_pending = conn->base_.state == + AP_CONN_STATE_CONNECT_WAIT; + + /* Data should only arrive if we're not waiting on a resolved cell. + * It can arrive after waiting on connected, because of optimistic + * data. */ + if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { + // How many more data cells can arrive on this id? + half_conn->data_pending = conn->deliver_window; + } + + insert_at = smartlist_bsearch_idx(circ->half_streams, &half_conn->stream_id, + connection_half_edge_compare_bsearch, + &ignored); + smartlist_insert(circ->half_streams, insert_at, half_conn); +} + +/** Release space held by <b>he</b> */ +void +half_edge_free_(half_edge_t *he) +{ + if (!he) + return; + --n_half_conns_allocated; + tor_free(he); +} + +/** Return the number of bytes devoted to storing info on half-open streams. */ +size_t +half_streams_get_total_allocation(void) +{ + return n_half_conns_allocated * sizeof(half_edge_t); +} + +/** + * Find a stream_id_t in the list in O(lg(n)). + * + * Returns NULL if the list is empty or element is not found. + * Returns a pointer to the element if found. + */ +STATIC half_edge_t * +connection_half_edge_find_stream_id(const smartlist_t *half_conns, + streamid_t stream_id) +{ + if (!half_conns) + return NULL; + + return smartlist_bsearch(half_conns, &stream_id, + connection_half_edge_compare_bsearch); +} + +/** + * Check if this stream_id is in a half-closed state. If so, + * check if it still has data cells pending, and decrement that + * window if so. + * + * Return 1 if the data window was not empty. + * Return 0 otherwise. + */ +int +connection_half_edge_is_valid_data(const smartlist_t *half_conns, + streamid_t stream_id) +{ + half_edge_t *half = connection_half_edge_find_stream_id(half_conns, + stream_id); + + if (!half) + return 0; + + if (half->data_pending > 0) { + half->data_pending--; + return 1; + } + + return 0; +} + +/** + * Check if this stream_id is in a half-closed state. If so, + * check if it still has a connected cell pending, and decrement + * that window if so. + * + * Return 1 if the connected window was not empty. + * Return 0 otherwise. + */ +int +connection_half_edge_is_valid_connected(const smartlist_t *half_conns, + streamid_t stream_id) +{ + half_edge_t *half = connection_half_edge_find_stream_id(half_conns, + stream_id); + + if (!half) + return 0; + + if (half->connected_pending) { + half->connected_pending = 0; + return 1; + } + + return 0; +} + +/** + * Check if this stream_id is in a half-closed state. If so, + * check if it still has sendme cells pending, and decrement that + * window if so. + * + * Return 1 if the sendme window was not empty. + * Return 0 otherwise. + */ +int +connection_half_edge_is_valid_sendme(const smartlist_t *half_conns, + streamid_t stream_id) +{ + half_edge_t *half = connection_half_edge_find_stream_id(half_conns, + stream_id); + + if (!half) + return 0; + + if (half->sendmes_pending > 0) { + half->sendmes_pending--; + return 1; + } + + return 0; +} + +/** + * Check if this stream_id is in a half-closed state. If so, remove + * it from the list. No other data should come after the END cell. + * + * Return 1 if stream_id was in half-closed state. + * Return 0 otherwise. + */ +int +connection_half_edge_is_valid_end(smartlist_t *half_conns, + streamid_t stream_id) +{ + half_edge_t *half; + int found, remove_idx; + + if (!half_conns) + return 0; + + remove_idx = smartlist_bsearch_idx(half_conns, &stream_id, + connection_half_edge_compare_bsearch, + &found); + if (!found) + return 0; + + half = smartlist_get(half_conns, remove_idx); + smartlist_del_keeporder(half_conns, remove_idx); + half_edge_free(half); + return 1; +} + +/** + * Streams that were used to send a RESOLVE cell are closed + * when they get the RESOLVED, without an end. So treat + * a RESOLVED just like an end, and remove from the list. + */ +int +connection_half_edge_is_valid_resolved(smartlist_t *half_conns, + streamid_t stream_id) +{ + return connection_half_edge_is_valid_end(half_conns, stream_id); +} + /** An error has just occurred on an operation on an edge connection * <b>conn</b>. Extract the errno; convert it to an end reason, and send an * appropriate relay end cell to the other end of the connection's circuit. @@ -611,6 +848,12 @@ static smartlist_t *pending_entry_connections = NULL; static int untried_pending_connections = 0; +/** + * Mainloop event to tell us to scan for pending connections that can + * be attached. + */ +static mainloop_event_t *attach_pending_entry_connections_ev = NULL; + /** Common code to connection_(ap|exit)_about_to_close. */ static void connection_edge_about_to_close(edge_connection_t *edge_conn) @@ -739,7 +982,7 @@ connection_ap_expire_beginning(void) /* if it's an internal linked connection, don't yell its status. */ severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port) ? LOG_INFO : LOG_NOTICE; - seconds_idle = (int)( now - base_conn->timestamp_lastread ); + seconds_idle = (int)( now - base_conn->timestamp_last_read_allowed ); seconds_since_born = (int)( now - base_conn->timestamp_created ); if (base_conn->state == AP_CONN_STATE_OPEN) @@ -825,7 +1068,7 @@ connection_ap_expire_beginning(void) mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ)); /* give our stream another 'cutoff' seconds to try */ - conn->base_.timestamp_lastread += cutoff; + conn->base_.timestamp_last_read_allowed += cutoff; if (entry_conn->num_socks_retries < 250) /* avoid overflow */ entry_conn->num_socks_retries++; /* move it back into 'pending' state, and try to attach. */ @@ -956,6 +1199,14 @@ connection_ap_attach_pending(int retry) untried_pending_connections = 0; } +static void +attach_pending_entry_connections_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + connection_ap_attach_pending(0); +} + /** Mark <b>entry_conn</b> as needing to get attached to a circuit. * * And <b>entry_conn</b> must be in AP_CONN_STATE_CIRCUIT_WAIT, @@ -973,9 +1224,13 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn, if (conn->marked_for_close) return; - if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) + if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) { pending_entry_connections = smartlist_new(); - + } + if (PREDICT_UNLIKELY(NULL == attach_pending_entry_connections_ev)) { + attach_pending_entry_connections_ev = mainloop_event_postloop_new( + attach_pending_entry_connections_cb, NULL); + } if (PREDICT_UNLIKELY(smartlist_contains(pending_entry_connections, entry_conn))) { log_warn(LD_BUG, "What?? pending_entry_connections already contains %p! " @@ -999,14 +1254,7 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn, untried_pending_connections = 1; smartlist_add(pending_entry_connections, entry_conn); - /* Work-around for bug 19969: we handle pending_entry_connections at - * the end of run_main_loop_once(), but in many cases that function will - * take a very long time, if ever, to finish its call to event_base_loop(). - * - * So the fix is to tell it right now that it ought to finish its loop at - * its next available opportunity. - */ - tell_event_loop_to_run_external_code(); + mainloop_event_activate(attach_pending_entry_connections_ev); } /** Mark <b>entry_conn</b> as no longer waiting for a circuit. */ @@ -1135,7 +1383,7 @@ connection_ap_detach_retriable(entry_connection_t *conn, int reason) { control_event_stream_status(conn, STREAM_EVENT_FAILED_RETRIABLE, reason); - ENTRY_TO_CONN(conn)->timestamp_lastread = time(NULL); + ENTRY_TO_CONN(conn)->timestamp_last_read_allowed = time(NULL); /* Roll back path bias use state so that we probe the circuit * if nothing else succeeds on it */ @@ -2584,6 +2832,11 @@ get_unique_stream_id_by_circ(origin_circuit_t *circ) for (tmpconn = circ->p_streams; tmpconn; tmpconn=tmpconn->next_stream) if (tmpconn->stream_id == test_stream_id) goto again; + + if (connection_half_edge_find_stream_id(circ->half_streams, + test_stream_id)) + goto again; + return test_stream_id; } @@ -3518,16 +3771,24 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) n_stream->deliver_window = STREAMWINDOW_START; if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { + int ret; tor_free(address); /* We handle this circuit and stream in this function for all supported * hidden service version. */ - return handle_hs_exit_conn(circ, n_stream); + ret = handle_hs_exit_conn(circ, n_stream); + + if (ret == 0) { + /* This was a valid cell. Count it as delivered + overhead. */ + circuit_read_valid_data(origin_circ, rh.length); + } + return ret; } tor_strlower(address); n_stream->base_.address = address; n_stream->base_.state = EXIT_CONN_STATE_RESOLVEFAILED; /* default to failed, change in dns_resolve if it turns out not to fail */ + /* If we're hibernating or shutting down, we refuse to open new streams. */ if (we_are_hibernating()) { relay_send_end_cell_from_edge(rh.stream_id, circ, END_STREAM_REASON_HIBERNATING, NULL); @@ -4168,4 +4429,5 @@ connection_edge_free_all(void) untried_pending_connections = 0; smartlist_free(pending_entry_connections); pending_entry_connections = NULL; + mainloop_event_free(attach_pending_entry_connections_ev); } diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index c6583d3845..8bef97dc41 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -122,6 +122,22 @@ void connection_ap_warn_and_unmark_if_pending_circ( entry_connection_t *entry_conn, const char *where); +int connection_half_edge_is_valid_data(const smartlist_t *half_conns, + streamid_t stream_id); +int connection_half_edge_is_valid_sendme(const smartlist_t *half_conns, + streamid_t stream_id); +int connection_half_edge_is_valid_connected(const smartlist_t *half_conns, + streamid_t stream_id); +int connection_half_edge_is_valid_end(smartlist_t *half_conns, + streamid_t stream_id); +int connection_half_edge_is_valid_resolved(smartlist_t *half_conns, + streamid_t stream_id); + +size_t half_streams_get_total_allocation(void); +void half_edge_free_(half_edge_t *he); +#define half_edge_free(he) \ + FREE_AND_NULL(half_edge_t, half_edge_free_, (he)) + /** @name Begin-cell flags * * These flags are used in RELAY_BEGIN cells to change the default behavior @@ -191,7 +207,12 @@ STATIC void connection_ap_handshake_rewrite(entry_connection_t *conn, rewrite_result_t *out); STATIC int connection_ap_process_http_connect(entry_connection_t *conn); +struct half_edge_t; +STATIC void connection_half_edge_add(const edge_connection_t *conn, + origin_circuit_t *circ); +STATIC struct half_edge_t *connection_half_edge_find_stream_id( + const smartlist_t *half_conns, + streamid_t stream_id); #endif /* defined(CONNECTION_EDGE_PRIVATE) */ #endif /* !defined(TOR_CONNECTION_EDGE_H) */ - diff --git a/src/or/connection_or.c b/src/or/connection_or.c index c2ca7078e0..99c0ac077f 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -39,6 +39,8 @@ #include "connection.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "dirserv.h" #include "entrynodes.h" #include "geoip.h" @@ -794,18 +796,10 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, (int)options->BandwidthBurst, 1, INT32_MAX); } - conn->bandwidthrate = rate; - conn->bandwidthburst = burst; - if (reset) { /* set up the token buckets to be full */ - conn->read_bucket = conn->write_bucket = burst; - return; + token_bucket_rw_adjust(&conn->bucket, rate, burst); + if (reset) { + token_bucket_rw_reset(&conn->bucket, monotime_coarse_get_stamp()); } - /* If the new token bucket is smaller, take out the extra tokens. - * (If it's larger, don't -- the buckets can grow to reach the cap.) */ - if (conn->read_bucket > burst) - conn->read_bucket = burst; - if (conn->write_bucket > burst) - conn->write_bucket = burst; } /** Either our set of relays or our per-conn rate limits have changed. diff --git a/src/or/conscache.c b/src/or/conscache.c index e25ac5f40b..51dc9d621f 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -5,6 +5,7 @@ #include "config.h" #include "conscache.h" +#include "crypto_util.h" #include "storagedir.h" #define CCE_MAGIC 0x17162253 diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 02b905a520..323f4f9ca0 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -99,6 +99,14 @@ static const compress_method_t compress_diffs_with[] = { #endif }; +/** + * Event for rescanning the cache. + */ +static mainloop_event_t *consdiffmgr_rescan_ev = NULL; + +static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg); +static void mark_cdm_cache_dirty(void); + /** How many different methods will we try to use for diff compression? */ STATIC unsigned n_diff_compression_methods(void) @@ -372,7 +380,9 @@ cdm_cache_init(void) } else { consdiffmgr_set_cache_flags(); } - cdm_cache_dirty = 1; + consdiffmgr_rescan_ev = + mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL); + mark_cdm_cache_dirty(); cdm_cache_loaded = 0; } @@ -1095,6 +1105,24 @@ consdiffmgr_rescan(void) cdm_cache_dirty = 0; } +/** Callback wrapper for consdiffmgr_rescan */ +static void +consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + consdiffmgr_rescan(); +} + +/** Mark the cache as dirty, and schedule a rescan event. */ +static void +mark_cdm_cache_dirty(void) +{ + cdm_cache_dirty = 1; + tor_assert(consdiffmgr_rescan_ev); + mainloop_event_activate(consdiffmgr_rescan_ev); +} + /** * Helper: compare two files by their from-valid-after and valid-after labels, * trying to sort in ascending order by from-valid-after (when present) and @@ -1219,6 +1247,7 @@ consdiffmgr_free_all(void) memset(latest_consensus, 0, sizeof(latest_consensus)); consensus_cache_free(cons_diff_cache); cons_diff_cache = NULL; + mainloop_event_free(consdiffmgr_rescan_ev); } /* ===== @@ -1750,7 +1779,7 @@ consensus_compress_worker_replyfn(void *work_) compress_consensus_with, job->out, "consensus"); - cdm_cache_dirty = 1; + mark_cdm_cache_dirty(); unsigned u; consensus_flavor_t f = job->flavor; diff --git a/src/or/control.c b/src/or/control.c index 3f677ba348..1115ef7b1d 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,3 +1,4 @@ + /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -52,6 +53,8 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" #include "dnsserv.h" @@ -76,15 +79,13 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" -#include "shared_random.h" +#include "shared_random_client.h" #ifndef _WIN32 #include <pwd.h> #include <sys/resource.h> #endif -#include <event2/event.h> - #include "crypto_s2k.h" #include "procmon.h" @@ -112,6 +113,10 @@ static int disable_log_messages = 0; #define EVENT_IS_INTERESTING(e) \ (!! (global_event_mask & EVENT_MASK_(e))) +/** Macro: true if any event from the bitfield 'e' is interesting. */ +#define ANY_EVENT_IS_INTERESTING(e) \ + (!! (global_event_mask & (e))) + /** If we're using cookie-type authentication, how long should our cookies be? */ #define AUTHENTICATION_COOKIE_LEN 32 @@ -216,9 +221,10 @@ static void orconn_target_get_name(char *buf, size_t len, static int get_cached_network_liveness(void); static void set_cached_network_liveness(int liveness); -static void flush_queued_events_cb(evutil_socket_t fd, short what, void *arg); +static void flush_queued_events_cb(mainloop_event_t *event, void *arg); static char * download_status_to_string(const download_status_t *dl); +static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w); /** Given a control event code for a message event, return the corresponding * log severity. */ @@ -259,6 +265,8 @@ clear_circ_bw_fields(void) continue; ocirc = TO_ORIGIN_CIRCUIT(circ); ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; + ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; } SMARTLIST_FOREACH_END(circ); } @@ -271,6 +279,7 @@ control_update_global_event_mask(void) smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; old_mask = global_event_mask; + int any_old_per_sec_events = control_any_per_second_event_enabled(); global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, @@ -288,10 +297,13 @@ control_update_global_event_mask(void) * we want to hear...*/ control_adjust_event_log_severity(); + /* Macro: true if ev was false before and is true now. */ +#define NEWLY_ENABLED(ev) \ + (! (old_mask & (ev)) && (new_mask & (ev))) + /* ...then, if we've started logging stream or circ bw, clear the * appropriate fields. */ - if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) && - (new_mask & EVENT_STREAM_BANDWIDTH_USED)) { + if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) { SMARTLIST_FOREACH(conns, connection_t *, conn, { if (conn->type == CONN_TYPE_AP) { @@ -300,10 +312,18 @@ control_update_global_event_mask(void) } }); } - if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) && - (new_mask & EVENT_CIRC_BANDWIDTH_USED)) { + if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) { clear_circ_bw_fields(); } + if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) { + uint64_t r, w; + control_get_bytes_rw_last_sec(&r, &w); + } + if (any_old_per_sec_events != control_any_per_second_event_enabled()) { + reschedule_per_second_timer(); + } + +#undef NEWLY_ENABLED } /** Adjust the log severities that result in control_event_logmsg being called @@ -352,6 +372,65 @@ control_event_is_interesting(int event) return EVENT_IS_INTERESTING(event); } +/** Return true if any event that needs to fire once a second is enabled. */ +int +control_any_per_second_event_enabled(void) +{ + return ANY_EVENT_IS_INTERESTING( + EVENT_MASK_(EVENT_BANDWIDTH_USED) | + EVENT_MASK_(EVENT_CELL_STATS) | + EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) | + EVENT_MASK_(EVENT_CONN_BW) | + EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED) + ); +} + +/* The value of 'get_bytes_read()' the previous time that + * control_get_bytes_rw_last_sec() as called. */ +static uint64_t stats_prev_n_read = 0; +/* The value of 'get_bytes_written()' the previous time that + * control_get_bytes_rw_last_sec() as called. */ +static uint64_t stats_prev_n_written = 0; + +/** + * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read + * and written by Tor since the last call to this function. + * + * Call this only from the main thread. + */ +static void +control_get_bytes_rw_last_sec(uint64_t *n_read, + uint64_t *n_written) +{ + const uint64_t stats_n_bytes_read = get_bytes_read(); + const uint64_t stats_n_bytes_written = get_bytes_written(); + + *n_read = stats_n_bytes_read - stats_prev_n_read; + *n_written = stats_n_bytes_written - stats_prev_n_written; + stats_prev_n_read = stats_n_bytes_read; + stats_prev_n_written = stats_n_bytes_written; +} + +/** + * Run all the controller events (if any) that are scheduled to trigger once + * per second. + */ +void +control_per_second_events(void) +{ + if (!control_any_per_second_event_enabled()) + return; + + uint64_t bytes_read, bytes_written; + control_get_bytes_rw_last_sec(&bytes_read, &bytes_written); + control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); + + control_event_stream_bandwidth_used(); + control_event_conn_bandwidth_used(); + control_event_circ_bandwidth_used(); + control_event_circuit_cell_stats(); +} + /** Append a NUL-terminated string <b>s</b> to the end of * <b>conn</b>-\>outbuf. */ @@ -691,7 +770,7 @@ static tor_mutex_t *queued_control_events_lock = NULL; /** An event that should fire in order to flush the contents of * queued_control_events. */ -static struct event *flush_queued_events_event = NULL; +static mainloop_event_t *flush_queued_events_event = NULL; void control_initialize_event_queue(void) @@ -703,9 +782,8 @@ control_initialize_event_queue(void) if (flush_queued_events_event == NULL) { struct event_base *b = tor_libevent_get_base(); if (b) { - flush_queued_events_event = tor_event_new(b, - -1, 0, flush_queued_events_cb, - NULL); + flush_queued_events_event = + mainloop_event_new(flush_queued_events_cb, NULL); tor_assert(flush_queued_events_event); } } @@ -781,7 +859,7 @@ queue_control_event_string,(uint16_t event, char *msg)) */ if (activate_event) { tor_assert(flush_queued_events_event); - event_active(flush_queued_events_event, EV_READ, 1); + mainloop_event_activate(flush_queued_events_event); } } @@ -806,6 +884,9 @@ queued_event_free_(queued_event_t *ev) static void queued_events_flush_all(int force) { + /* Make sure that we get all the pending log events, if there are any. */ + flush_pending_log_callbacks(); + if (PREDICT_UNLIKELY(queued_control_events == NULL)) { return; } @@ -863,10 +944,9 @@ queued_events_flush_all(int force) /** Libevent callback: Flushes pending events to controllers that are * interested in them. */ static void -flush_queued_events_cb(evutil_socket_t fd, short what, void *arg) +flush_queued_events_cb(mainloop_event_t *event, void *arg) { - (void) fd; - (void) what; + (void) event; (void) arg; queued_events_flush_all(0); } @@ -1218,7 +1298,6 @@ static const struct control_event_t control_event_table[] = { { EVENT_CONF_CHANGED, "CONF_CHANGED"}, { EVENT_CONN_BW, "CONN_BW" }, { EVENT_CELL_STATS, "CELL_STATS" }, - { EVENT_TB_EMPTY, "TB_EMPTY" }, { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, { EVENT_HS_DESC, "HS_DESC" }, @@ -1784,24 +1863,24 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "process/pid")) { int myPid = -1; - #ifdef _WIN32 +#ifdef _WIN32 myPid = _getpid(); - #else +#else myPid = getpid(); - #endif +#endif tor_asprintf(answer, "%d", myPid); } else if (!strcmp(question, "process/uid")) { - #ifdef _WIN32 +#ifdef _WIN32 *answer = tor_strdup("-1"); - #else +#else int myUid = geteuid(); tor_asprintf(answer, "%d", myUid); #endif /* defined(_WIN32) */ } else if (!strcmp(question, "process/user")) { - #ifdef _WIN32 +#ifdef _WIN32 *answer = tor_strdup(""); - #else +#else int myUid = geteuid(); const struct passwd *myPwEntry = tor_getpwuid(myUid); @@ -1940,6 +2019,31 @@ getinfo_helper_listeners(control_connection_t *control_conn, return 0; } +/** Implementation helper for GETINFO: answers requests for information about + * the current time in both local and UTF forms. */ +STATIC int +getinfo_helper_current_time(control_connection_t *control_conn, + const char *question, + char **answer, const char **errmsg) +{ + (void)control_conn; + (void)errmsg; + + struct timeval now; + tor_gettimeofday(&now); + char timebuf[ISO_TIME_LEN+1]; + + if (!strcmp(question, "current-time/local")) + format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec); + else if (!strcmp(question, "current-time/utc")) + format_iso_time_nospace(timebuf, (time_t)now.tv_sec); + else + return 0; + + *answer = tor_strdup(timebuf); + return 0; +} + /** Implementation helper for GETINFO: knows the answers for questions about * directory information. */ STATIC int @@ -3082,6 +3186,9 @@ static const getinfo_item_t getinfo_items[] = { DOC("config/defaults", "List of default values for configuration options. " "See also config/names"), + PREFIX("current-time/", current_time, "Current time."), + DOC("current-time/local", "Current time on the local system."), + DOC("current-time/utc", "Current UTC time."), PREFIX("downloads/networkstatus/", downloads, "Download statuses for networkstatus objects"), DOC("downloads/networkstatus/ns", @@ -4499,7 +4606,7 @@ handle_control_add_onion(control_connection_t *conn, const char *body) { smartlist_t *args; - size_t arg_len; + int arg_len; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = getargs_helper("ADD_ONION", conn, body, 2, -1); if (!args) @@ -4520,7 +4627,7 @@ handle_control_add_onion(control_connection_t *conn, rend_auth_type_t auth_type = REND_NO_AUTH; /* Default to adding an anonymous hidden service if no flag is given */ int non_anonymous = 0; - for (size_t i = 1; i < arg_len; i++) { + for (int i = 1; i < arg_len; i++) { static const char *port_prefix = "Port="; static const char *flags_prefix = "Flags="; static const char *max_s_prefix = "MaxStreams="; @@ -5817,8 +5924,6 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, int control_event_stream_bandwidth(edge_connection_t *edge_conn) { - circuit_t *circ; - origin_circuit_t *ocirc; struct timeval now; char tbuf[ISO_TIME_USEC_LEN+1]; if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { @@ -5834,12 +5939,6 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) (unsigned long)edge_conn->n_written, tbuf); - circ = circuit_get_by_edge_conn(edge_conn); - if (circ && CIRCUIT_IS_ORIGIN(circ)) { - ocirc = TO_ORIGIN_CIRCUIT(circ); - ocirc->n_read_circ_bw += edge_conn->n_read; - ocirc->n_written_circ_bw += edge_conn->n_written; - } edge_conn->n_written = edge_conn->n_read = 0; } @@ -5902,13 +6001,20 @@ control_event_circ_bandwidth_used(void) tor_gettimeofday(&now); format_iso_time_nospace_usec(tbuf, &now); send_control_event(EVENT_CIRC_BANDWIDTH_USED, - "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu " - "TIME=%s\r\n", + "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s " + "DELIVERED_READ=%lu OVERHEAD_READ=%lu " + "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n", ocirc->global_identifier, (unsigned long)ocirc->n_read_circ_bw, (unsigned long)ocirc->n_written_circ_bw, - tbuf); + tbuf, + (unsigned long)ocirc->n_delivered_read_circ_bw, + (unsigned long)ocirc->n_overhead_read_circ_bw, + (unsigned long)ocirc->n_delivered_written_circ_bw, + (unsigned long)ocirc->n_overhead_written_circ_bw); ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; + ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; } SMARTLIST_FOREACH_END(circ); @@ -6093,28 +6199,6 @@ control_event_circuit_cell_stats(void) return 0; } -/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty - * for <b>read_empty_time</b> millis, the write bucket was empty for - * <b>write_empty_time</b> millis, and buckets were last refilled - * <b>milliseconds_elapsed</b> millis ago. Only emit TB_EMPTY event if - * either read or write bucket have been empty before. */ -int -control_event_tb_empty(const char *bucket, uint32_t read_empty_time, - uint32_t write_empty_time, - int milliseconds_elapsed) -{ - if (get_options()->TestingEnableTbEmptyEvent && - EVENT_IS_INTERESTING(EVENT_TB_EMPTY) && - (read_empty_time > 0 || write_empty_time > 0)) { - send_control_event(EVENT_TB_EMPTY, - "650 TB_EMPTY %s READ=%d WRITTEN=%d " - "LAST=%d\r\n", - bucket, read_empty_time, write_empty_time, - milliseconds_elapsed); - } - return 0; -} - /* about 5 minutes worth. */ #define N_BW_EVENTS_TO_CACHE 300 /* Index into cached_bw_events to next write. */ @@ -6202,7 +6286,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int event; /* Don't even think of trying to add stuff to a buffer from a cpuworker - * thread. */ + * thread. (See #25987 for plan to fix.) */ if (! in_main_thread()) return; @@ -6248,6 +6332,23 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) } } +/** + * Logging callback: called when there is a queued pending log callback. + */ +void +control_event_logmsg_pending(void) +{ + if (! in_main_thread()) { + /* We can't handle this case yet, since we're using a + * mainloop_event_t to invoke queued_events_flush_all. We ought to + * use a different mechanism instead: see #25987. + **/ + return; + } + tor_assert(flush_queued_events_event); + mainloop_event_activate(flush_queued_events_event); +} + /** Called whenever we receive new router descriptors: tell any * interested control connections. <b>routers</b> is a list of * routerinfo_t's. @@ -7029,6 +7130,8 @@ control_event_bootstrap_problem(const char *warn, const char *reason, if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) dowarn = 1; + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ if (we_are_hibernating()) dowarn = 0; @@ -7598,20 +7701,31 @@ control_event_hs_descriptor_upload_failed(const char *id_digest, void control_free_all(void) { + smartlist_t *queued_events = NULL; + + stats_prev_n_read = stats_prev_n_written = 0; + if (authentication_cookie) /* Free the auth cookie */ tor_free(authentication_cookie); if (detached_onion_services) { /* Free the detached onion services */ SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp)); smartlist_free(detached_onion_services); } - if (queued_control_events) { - SMARTLIST_FOREACH(queued_control_events, queued_event_t *, ev, - queued_event_free(ev)); - smartlist_free(queued_control_events); + + if (queued_control_events_lock) { + tor_mutex_acquire(queued_control_events_lock); + flush_queued_event_pending = 0; + queued_events = queued_control_events; queued_control_events = NULL; + tor_mutex_release(queued_control_events_lock); + } + if (queued_events) { + SMARTLIST_FOREACH(queued_events, queued_event_t *, ev, + queued_event_free(ev)); + smartlist_free(queued_events); } if (flush_queued_events_event) { - tor_event_free(flush_queued_events_event); + mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; @@ -7621,7 +7735,6 @@ control_free_all(void) global_event_mask = 0; disable_log_messages = 0; memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); - flush_queued_event_pending = 0; } #ifdef TOR_UNIT_TESTS @@ -7632,3 +7745,4 @@ control_testing_set_global_event_mask(uint64_t mask) global_event_mask = mask; } #endif /* defined(TOR_UNIT_TESTS) */ + diff --git a/src/or/control.h b/src/or/control.h index 28ffeaed86..92cbf866dd 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -40,6 +40,9 @@ int connection_control_process_inbuf(control_connection_t *conn); #define EVENT_NS 0x000F int control_event_is_interesting(int event); +void control_per_second_events(void); +int control_any_per_second_event_enabled(void); + int control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t e, int reason); int control_event_circuit_purpose_changed(origin_circuit_t *circ, @@ -59,10 +62,8 @@ int control_event_circ_bandwidth_used(void); int control_event_conn_bandwidth(connection_t *conn); int control_event_conn_bandwidth_used(void); int control_event_circuit_cell_stats(void); -int control_event_tb_empty(const char *bucket, uint32_t read_empty_time, - uint32_t write_empty_time, - int milliseconds_elapsed); void control_event_logmsg(int severity, uint32_t domain, const char *msg); +void control_event_logmsg_pending(void); int control_event_descriptors_changed(smartlist_t *routers); int control_event_address_mapped(const char *from, const char *to, time_t expires, const char *error, @@ -194,7 +195,7 @@ void control_free_all(void); #define EVENT_CONF_CHANGED 0x0019 #define EVENT_CONN_BW 0x001A #define EVENT_CELL_STATS 0x001B -#define EVENT_TB_EMPTY 0x001C +/* UNUSED : 0x001C */ #define EVENT_CIRC_BANDWIDTH_USED 0x001D #define EVENT_TRANSPORT_LAUNCHED 0x0020 #define EVENT_HS_DESC 0x0021 @@ -314,6 +315,10 @@ STATIC int getinfo_helper_dir( control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); +STATIC int getinfo_helper_current_time( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); #endif /* defined(CONTROL_PRIVATE) */ diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 50761dd4d3..15ef6869cf 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -24,14 +24,14 @@ #include "connection_or.h" #include "config.h" #include "cpuworker.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "main.h" #include "onion.h" #include "rephist.h" #include "router.h" #include "workqueue.h" -#include <event2/event.h> - static void queue_pending_tasks(void); typedef struct worker_state_s { @@ -69,22 +69,12 @@ worker_state_free_void(void *arg) static replyqueue_t *replyqueue = NULL; static threadpool_t *threadpool = NULL; -static struct event *reply_event = NULL; static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; static int total_pending_tasks = 0; static int max_pending_tasks = 128; -static void -replyqueue_process_cb(evutil_socket_t sock, short events, void *arg) -{ - replyqueue_t *rq = arg; - (void) sock; - (void) events; - replyqueue_process(rq); -} - /** Initialize the cpuworker subsystem. It is OK to call this more than once * during Tor's lifetime. */ @@ -94,14 +84,6 @@ cpu_init(void) if (!replyqueue) { replyqueue = replyqueue_new(0); } - if (!reply_event) { - reply_event = tor_event_new(tor_libevent_get_base(), - replyqueue_get_socket(replyqueue), - EV_READ|EV_PERSIST, - replyqueue_process_cb, - replyqueue); - event_add(reply_event, NULL); - } if (!threadpool) { /* In our threadpool implementation, half the threads are permissive and @@ -115,7 +97,12 @@ cpu_init(void) worker_state_new, worker_state_free_void, NULL); + + int r = threadpool_register_reply_event(threadpool, NULL); + + tor_assert(r == 0); } + /* Total voodoo. Can we make this more sensible? */ max_pending_tasks = get_num_cpus(get_options()) * 64; crypto_seed_weak_rng(&request_sample_rng); @@ -547,7 +534,7 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ, return 0; } - if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) + if (!channel_is_client(circ->p_chan)) rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); should_time = should_time_request(onionskin->handshake_type); diff --git a/src/or/dircollate.c b/src/or/dirauth/dircollate.c index ce4534ff6c..dec6f75154 100644 --- a/src/or/dircollate.c +++ b/src/or/dirauth/dircollate.c @@ -25,7 +25,6 @@ #include "dircollate.h" #include "dirvote.h" -static void dircollator_collate_by_rsa(dircollator_t *dc); static void dircollator_collate_by_ed25519(dircollator_t *dc); /** Hashtable entry mapping a pair of digests (actually an ed25519 key and an @@ -208,49 +207,18 @@ dircollator_add_vote(dircollator_t *dc, networkstatus_t *v) void dircollator_collate(dircollator_t *dc, int consensus_method) { + (void) consensus_method; + tor_assert(!dc->is_collated); dc->all_rsa_sha1_lst = smartlist_new(); - if (consensus_method < MIN_METHOD_FOR_ED25519_ID_VOTING) - dircollator_collate_by_rsa(dc); - else - dircollator_collate_by_ed25519(dc); + dircollator_collate_by_ed25519(dc); smartlist_sort_digests(dc->all_rsa_sha1_lst); dc->is_collated = 1; } /** - * Collation function for RSA-only consensuses: collate the votes for each - * entry in <b>dc</b> by their RSA keys. - * - * The rule is: - * If an RSA identity key is listed by more than half of the authorities, - * include that identity, and treat all descriptors with that RSA identity - * as describing the same router. - */ -static void -dircollator_collate_by_rsa(dircollator_t *dc) -{ - const int total_authorities = dc->n_authorities; - - DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) { - int n = 0, i; - for (i = 0; i < dc->n_votes; ++i) { - if (vrs_lst[i] != NULL) - ++n; - } - - if (n <= total_authorities / 2) - continue; - - smartlist_add(dc->all_rsa_sha1_lst, (char *)k); - } DIGESTMAP_FOREACH_END; - - dc->by_collated_rsa_sha1 = dc->by_rsa_sha1; -} - -/** * Collation function for ed25519 consensuses: collate the votes for each * entry in <b>dc</b> by ed25519 key and by RSA key. * diff --git a/src/or/dircollate.h b/src/or/dirauth/dircollate.h index 0584b2fe06..0584b2fe06 100644 --- a/src/or/dircollate.h +++ b/src/or/dirauth/dircollate.h diff --git a/src/or/dirvote.c b/src/or/dirauth/dirvote.c index 7023d4f951..bf05286b3d 100644 --- a/src/or/dirvote.c +++ b/src/or/dirauth/dirvote.c @@ -9,9 +9,10 @@ #include "dircollate.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "microdesc.h" #include "networkstatus.h" +#include "nodelist.h" +#include "parsecommon.h" #include "policies.h" #include "protover.h" #include "rephist.h" @@ -21,7 +22,11 @@ #include "routerparse.h" #include "entrynodes.h" /* needed for guardfraction methods */ #include "torcert.h" -#include "shared_random_state.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random_state.h" /** * \file dirvote.c @@ -92,6 +97,30 @@ static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); /* ===== + * Certificate functions + * ===== */ + +/** Allocate and return a new authority_cert_t with the same contents as + * <b>cert</b>. */ +STATIC authority_cert_t * +authority_cert_dup(authority_cert_t *cert) +{ + authority_cert_t *out = tor_malloc(sizeof(authority_cert_t)); + tor_assert(cert); + + memcpy(out, cert, sizeof(authority_cert_t)); + /* Now copy pointed-to things. */ + out->cache_info.signed_descriptor_body = + tor_strndup(cert->cache_info.signed_descriptor_body, + cert->cache_info.signed_descriptor_len); + out->cache_info.saved_location = SAVED_NOWHERE; + out->identity_key = crypto_pk_dup_key(cert->identity_key); + out->signing_key = crypto_pk_dup_key(cert->signing_key); + + return out; +} + +/* ===== * Voting * =====*/ @@ -347,10 +376,73 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, return status; } +/** Set *<b>timing_out</b> to the intervals at which we would like to vote. + * Note that these aren't the intervals we'll use to vote; they're the ones + * that we'll vote to use. */ +static void +dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) +{ + const or_options_t *options = get_options(); + + tor_assert(timing_out); + + timing_out->vote_interval = options->V3AuthVotingInterval; + timing_out->n_intervals_valid = options->V3AuthNIntervalsValid; + timing_out->vote_delay = options->V3AuthVoteDelay; + timing_out->dist_delay = options->V3AuthDistDelay; +} + /* ===== * Consensus generation * ===== */ +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +static int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && fast_memeq(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + /** Given a vote <b>vote</b> (not a consensus!), return its associated * networkstatus_voter_info_t. */ static networkstatus_voter_info_t * @@ -363,20 +455,6 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } -/** Return the signature made by <b>voter</b> using the algorithm - * <b>alg</b>, or NULL if none is found. */ -document_signature_t * -voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, - digest_algorithm_t alg) -{ - if (!voter->sigs) - return NULL; - SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, - if (sig->alg == alg) - return sig); - return NULL; -} - /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -549,12 +627,12 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, tor_assert(most); - /* If we're producing "a" lines, vote on potential alternative (sets - * of) OR port(s) in the winning routerstatuses. + /* Vote on potential alternative (sets of) OR port(s) in the winning + * routerstatuses. * * XXX prop186 There's at most one alternative OR port (_the_ IPv6 * port) for now. */ - if (consensus_method >= MIN_METHOD_FOR_A_LINES && best_alt_orport_out) { + if (best_alt_orport_out) { smartlist_t *alt_orports = smartlist_new(); const tor_addr_port_t *most_alt_orport = NULL; @@ -664,13 +742,6 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - if (method == MIN_METHOD_FOR_ED25519_ID_IN_MD) { - /* This method was broken due to buggy code accidentally left in - * dircollate.c; do not actually use it. - */ - return 0; - } - return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); } @@ -1343,7 +1414,7 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes) * behavior, and make the new behavior conditional on a new-enough * consensus_method. **/ -char * +STATIC char * networkstatus_compute_consensus(smartlist_t *votes, int total_authorities, crypto_pk_t *identity_key, @@ -1463,19 +1534,14 @@ networkstatus_compute_consensus(smartlist_t *votes, n_versioning_servers); client_versions = compute_consensus_versions_list(combined_client_versions, n_versioning_clients); - if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) { - packages = compute_consensus_package_lines(votes); - } else { - packages = tor_strdup(""); - } + packages = compute_consensus_package_lines(votes); SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp)); SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp)); smartlist_free(combined_server_versions); smartlist_free(combined_client_versions); - if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) - smartlist_add_strdup(flags, "NoEdConsensus"); + smartlist_add_strdup(flags, "NoEdConsensus"); smartlist_sort_strings(flags); smartlist_uniq_strings(flags); @@ -1524,7 +1590,7 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(flaglist); } - if (consensus_method >= MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS) { + { int num_dirauth = get_n_authorities(V3_DIRINFO); int idx; for (idx = 0; idx < 4; ++idx) { @@ -1544,7 +1610,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_strdup(chunks, "\n"); } - if (consensus_method >= MIN_METHOD_FOR_SHARED_RANDOM) { + { int num_dirauth = get_n_authorities(V3_DIRINFO); /* Default value of this is 2/3 of the total number of authorities. For * instance, if we have 9 dirauth, the default value is 6. The following @@ -1609,7 +1675,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(dir_sources); } - if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW) { + { char *max_unmeasured_param = NULL; /* XXXX Extract this code into a common function. Or don't! see #19011 */ if (params) { @@ -1871,7 +1937,6 @@ networkstatus_compute_consensus(smartlist_t *votes, continue; if (ed_consensus > 0) { - tor_assert(consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING); if (ed_consensus <= total_authorities / 2) { log_warn(LD_BUG, "Not enough entries had ed_consensus set; how " "can we have a consensus of %d?", ed_consensus); @@ -1898,10 +1963,8 @@ networkstatus_compute_consensus(smartlist_t *votes, rs_out.published_on = rs->status.published_on; rs_out.dir_port = rs->status.dir_port; rs_out.or_port = rs->status.or_port; - if (consensus_method >= MIN_METHOD_FOR_A_LINES) { - tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr); - rs_out.ipv6_orport = alt_orport.port; - } + tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr); + rs_out.ipv6_orport = alt_orport.port; rs_out.has_bandwidth = 0; rs_out.has_exitsummary = 0; @@ -1931,8 +1994,7 @@ networkstatus_compute_consensus(smartlist_t *votes, } else if (!strcmp(fl, "Unnamed")) { if (is_unnamed) smartlist_add(chosen_flags, (char*)fl); - } else if (!strcmp(fl, "NoEdConsensus") && - consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) { + } else if (!strcmp(fl, "NoEdConsensus")) { if (ed_consensus <= total_authorities/2) smartlist_add(chosen_flags, (char*)fl); } else { @@ -1959,8 +2021,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Starting with consensus method 24, we don't list servers * that are not valid in a consensus. See Proposal 272 */ - if (!is_valid && - consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES) + if (!is_valid) continue; /* Pick the version. */ @@ -1981,8 +2042,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* If it's a guard and we have enough guardfraction votes, calculate its consensus guardfraction value. */ - if (is_guard && num_guardfraction_inputs > 2 && - consensus_method >= MIN_METHOD_FOR_GUARDFRACTION) { + if (is_guard && num_guardfraction_inputs > 2) { rs_out.has_guardfraction = 1; rs_out.guardfraction_percentage = median_uint32(measured_guardfraction, num_guardfraction_inputs); @@ -1999,8 +2059,7 @@ networkstatus_compute_consensus(smartlist_t *votes, rs_out.has_bandwidth = 1; rs_out.bw_is_unmeasured = 1; rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths); - if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW && - n_authorities_measuring_bandwidth > 2) { + if (n_authorities_measuring_bandwidth > 2) { /* Cap non-measured bandwidths. */ if (rs_out.bandwidth_kb > max_unmeasured_bw_kb) { rs_out.bandwidth_kb = max_unmeasured_bw_kb; @@ -2140,8 +2199,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Now the weight line. */ if (rs_out.has_bandwidth) { char *guardfraction_str = NULL; - int unmeasured = rs_out.bw_is_unmeasured && - consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW; + int unmeasured = rs_out.bw_is_unmeasured; /* If we have guardfraction info, include it in the 'w' line. */ if (rs_out.has_guardfraction) { @@ -2400,7 +2458,7 @@ compute_consensus_package_lines(smartlist_t *votes) * new signature is verifiable.) Return the number of signatures added or * changed, or -1 if the document signed by <b>sigs</b> isn't the same * document as <b>target</b>. */ -int +STATIC int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char *source, @@ -2484,7 +2542,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target, continue; } - old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + old_sig = networkstatus_get_voter_sig_by_alg(target_voter, sig->alg); /* If the target already has a good signature from this voter, then skip * this one. */ @@ -2592,7 +2650,7 @@ networkstatus_format_signatures(networkstatus_t *consensus, * corresponding to the signatures on <b>consensuses</b>, which must contain * exactly one FLAV_NS consensus, and no more than one consensus for each * other flavor. */ -char * +STATIC char * networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; @@ -2697,219 +2755,16 @@ get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, return signatures; } -/** Release all storage held in <b>s</b>. */ -void -ns_detached_signatures_free_(ns_detached_signatures_t *s) -{ - if (!s) - return; - if (s->signatures) { - STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { - SMARTLIST_FOREACH(sigs, document_signature_t *, sig, - document_signature_free(sig)); - smartlist_free(sigs); - } STRMAP_FOREACH_END; - strmap_free(s->signatures, NULL); - strmap_free(s->digests, tor_free_); - } - - tor_free(s); -} - -/* ===== - * Certificate functions - * ===== */ - -/** Allocate and return a new authority_cert_t with the same contents as - * <b>cert</b>. */ -authority_cert_t * -authority_cert_dup(authority_cert_t *cert) -{ - authority_cert_t *out = tor_malloc(sizeof(authority_cert_t)); - tor_assert(cert); - - memcpy(out, cert, sizeof(authority_cert_t)); - /* Now copy pointed-to things. */ - out->cache_info.signed_descriptor_body = - tor_strndup(cert->cache_info.signed_descriptor_body, - cert->cache_info.signed_descriptor_len); - out->cache_info.saved_location = SAVED_NOWHERE; - out->identity_key = crypto_pk_dup_key(cert->identity_key); - out->signing_key = crypto_pk_dup_key(cert->signing_key); - - return out; -} - -/* ===== - * Vote scheduling - * ===== */ - -/** Set *<b>timing_out</b> to the intervals at which we would like to vote. - * Note that these aren't the intervals we'll use to vote; they're the ones - * that we'll vote to use. */ -void -dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) -{ - const or_options_t *options = get_options(); - - tor_assert(timing_out); - - timing_out->vote_interval = options->V3AuthVotingInterval; - timing_out->n_intervals_valid = options->V3AuthNIntervalsValid; - timing_out->vote_delay = options->V3AuthVoteDelay; - timing_out->dist_delay = options->V3AuthDistDelay; -} - -/** Return the start of the next interval of size <b>interval</b> (in - * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always - * starts a fresh interval, and if the last interval of a day would be - * truncated to less than half its size, it is rolled into the - * previous interval. */ -time_t -dirvote_get_start_of_next_interval(time_t now, int interval, int offset) -{ - struct tm tm; - time_t midnight_today=0; - time_t midnight_tomorrow; - time_t next; - - tor_gmtime_r(&now, &tm); - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - - if (tor_timegm(&tm, &midnight_today) < 0) { - log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight."); - } - midnight_tomorrow = midnight_today + (24*60*60); - - next = midnight_today + ((now-midnight_today)/interval + 1)*interval; - - /* Intervals never cross midnight. */ - if (next > midnight_tomorrow) - next = midnight_tomorrow; - - /* If the interval would only last half as long as it's supposed to, then - * skip over to the next day. */ - if (next + interval/2 > midnight_tomorrow) - next = midnight_tomorrow; - - next += offset; - if (next - interval > now) - next -= interval; - - return next; -} - -/* Populate and return a new voting_schedule_t that can be used to schedule - * voting. The object is allocated on the heap and it's the responsibility of - * the caller to free it. Can't fail. */ -static voting_schedule_t * -get_voting_schedule(const or_options_t *options, time_t now, int severity) -{ - int interval, vote_delay, dist_delay; - time_t start; - time_t end; - networkstatus_t *consensus; - voting_schedule_t *new_voting_schedule; - - new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t)); - - consensus = networkstatus_get_live_consensus(now); - - if (consensus) { - interval = (int)( consensus->fresh_until - consensus->valid_after ); - vote_delay = consensus->vote_seconds; - dist_delay = consensus->dist_seconds; - } else { - interval = options->TestingV3AuthInitialVotingInterval; - vote_delay = options->TestingV3AuthInitialVoteDelay; - dist_delay = options->TestingV3AuthInitialDistDelay; - } - - tor_assert(interval > 0); - - if (vote_delay + dist_delay > interval/2) - vote_delay = dist_delay = interval / 4; - - start = new_voting_schedule->interval_starts = - dirvote_get_start_of_next_interval(now,interval, - options->TestingV3AuthVotingStartOffset); - end = dirvote_get_start_of_next_interval(start+1, interval, - options->TestingV3AuthVotingStartOffset); - - tor_assert(end > start); - - new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2); - new_voting_schedule->voting_ends = start - dist_delay; - new_voting_schedule->fetch_missing_votes = - start - dist_delay - (vote_delay/2); - new_voting_schedule->voting_starts = start - dist_delay - vote_delay; - - { - char tbuf[ISO_TIME_LEN+1]; - format_iso_time(tbuf, new_voting_schedule->interval_starts); - tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: " - "consensus_set=%d, interval=%d", - tbuf, consensus?1:0, interval); - } - - return new_voting_schedule; -} - -#define voting_schedule_free(s) \ - FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s)) - -/** Frees a voting_schedule_t. This should be used instead of the generic - * tor_free. */ -static void -voting_schedule_free_(voting_schedule_t *voting_schedule_to_free) -{ - if (!voting_schedule_to_free) - return; - tor_free(voting_schedule_to_free); -} - -static voting_schedule_t voting_schedule; - -/* Using the time <b>now</b>, return the next voting valid-after time. */ +/** + * Entry point: Take whatever voting actions are pending as of <b>now</b>. + * + * Return the time at which the next action should be taken. + */ time_t -dirvote_get_next_valid_after_time(void) -{ - /* This is a safe guard in order to make sure that the voting schedule - * static object is at least initialized. Using this function with a zeroed - * voting schedule can lead to bugs. */ - if (tor_mem_is_zero((const char *) &voting_schedule, - sizeof(voting_schedule))) { - dirvote_recalculate_timing(get_options(), time(NULL)); - voting_schedule.created_on_demand = 1; - } - return voting_schedule.interval_starts; -} - -/** Set voting_schedule to hold the timing for the next vote we should be - * doing. All type of tor do that because HS subsystem needs the timing as - * well to function properly. */ -void -dirvote_recalculate_timing(const or_options_t *options, time_t now) -{ - voting_schedule_t *new_voting_schedule; - - /* get the new voting schedule */ - new_voting_schedule = get_voting_schedule(options, now, LOG_INFO); - tor_assert(new_voting_schedule); - - /* Fill in the global static struct now */ - memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule)); - voting_schedule_free(new_voting_schedule); -} - -/** Entry point: Take whatever voting actions are pending as of <b>now</b>. */ -void dirvote_act(const or_options_t *options, time_t now) { if (!authdir_mode_v3(options)) - return; + return TIME_MAX; tor_assert_nonfatal(voting_schedule.voting_starts); /* If we haven't initialized this object through this codeflow, we need to * recalculate the timings to match our vote. The reason to do that is if we @@ -2923,35 +2778,43 @@ dirvote_act(const or_options_t *options, time_t now) "Mine is %s.", keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN)); tor_free(keys); - dirvote_recalculate_timing(options, now); + voting_schedule_recalculate_timing(options, now); } - if (voting_schedule.voting_starts < now && !voting_schedule.have_voted) { + +#define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \ + if (! voting_schedule.done_field) { \ + if (voting_schedule.when_field > now) { \ + return voting_schedule.when_field; \ + } else { +#define ENDIF \ + } \ + } + + IF_TIME_FOR_NEXT_ACTION(voting_starts, have_voted) { log_notice(LD_DIR, "Time to vote."); dirvote_perform_vote(); voting_schedule.have_voted = 1; - } - if (voting_schedule.fetch_missing_votes < now && - !voting_schedule.have_fetched_missing_votes) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(fetch_missing_votes, have_fetched_missing_votes) { log_notice(LD_DIR, "Time to fetch any votes that we're missing."); dirvote_fetch_missing_votes(); voting_schedule.have_fetched_missing_votes = 1; - } - if (voting_schedule.voting_ends < now && - !voting_schedule.have_built_consensus) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(voting_ends, have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; - } - if (voting_schedule.fetch_missing_signatures < now && - !voting_schedule.have_fetched_missing_signatures) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(fetch_missing_signatures, + have_fetched_missing_signatures) { log_notice(LD_DIR, "Time to fetch any signatures that we're missing."); dirvote_fetch_missing_signatures(); voting_schedule.have_fetched_missing_signatures = 1; - } - if (voting_schedule.interval_starts < now && - !voting_schedule.have_published_consensus) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(interval_starts, + have_published_consensus) { log_notice(LD_DIR, "Time to publish the consensus and discard old votes"); dirvote_publish_consensus(); dirvote_clear_votes(0); @@ -2961,8 +2824,15 @@ dirvote_act(const or_options_t *options, time_t now) networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); /* XXXX We will want to try again later if we haven't got enough * signatures yet. Implement this if it turns out to ever happen. */ - dirvote_recalculate_timing(options, now); - } + voting_schedule_recalculate_timing(options, now); + return voting_schedule.voting_starts; + } ENDIF + + tor_assert_nonfatal_unreached(); + return now + 1; + +#undef ENDIF +#undef IF_TIME_FOR_NEXT_ACTION } /** A vote networkstatus_t and its unparsed body: held around so we can @@ -3826,7 +3696,7 @@ dirvote_get_vote(const char *fp, int flags) /** Construct and return a new microdescriptor from a routerinfo <b>ri</b> * according to <b>consensus_method</b>. **/ -microdesc_t * +STATIC microdesc_t * dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) { microdesc_t *result = NULL; @@ -3843,8 +3713,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "onion-key\n%s", key); - if (consensus_method >= MIN_METHOD_FOR_NTOR_KEY && - ri->onion_curve25519_pkey) { + if (ri->onion_curve25519_pkey) { char kbuf[128]; base64_encode(kbuf, sizeof(kbuf), (const char*)ri->onion_curve25519_pkey->public_key, @@ -3854,8 +3723,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) /* We originally put a lines in the micrdescriptors, but then we worked out * that we needed them in the microdesc consensus. See #20916. */ - if (consensus_method >= MIN_METHOD_FOR_A_LINES && - consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC && + if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC && !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); @@ -3866,8 +3734,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); - if (consensus_method >= MIN_METHOD_FOR_P6_LINES && - ri->ipv6_exit_policy) { + if (ri->ipv6_exit_policy) { /* XXXX+++ This doesn't match proposal 208, which says these should * be taken unchanged from the routerinfo. That's bogosity, IMO: * the proposal should have said to do this instead.*/ @@ -3877,11 +3744,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) tor_free(p6); } - if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) { + { char idbuf[ED25519_BASE64_LEN+1]; const char *keytype; - if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_IN_MD && - ri->cache_info.signing_key_cert && + if (ri->cache_info.signing_key_cert && ri->cache_info.signing_key_cert->signing_key_included) { keytype = "ed25519"; ed25519_public_to_base64(idbuf, @@ -3925,7 +3791,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer * in <b>out</b>. Return -1 on failure and the number of characters written * on success. */ -ssize_t +static ssize_t dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len, const microdesc_t *md, int consensus_method_low, @@ -3959,13 +3825,7 @@ static const struct consensus_method_range_t { int low; int high; } microdesc_consensus_methods[] = { - {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_A_LINES - 1}, - {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1}, - {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1}, - {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1}, - {MIN_METHOD_FOR_ID_HASH_IN_MD, MIN_METHOD_FOR_ED25519_ID_IN_MD - 1}, - {MIN_METHOD_FOR_ED25519_ID_IN_MD, - MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, + {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; @@ -4039,49 +3899,692 @@ dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now, return result; } -/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with - * the digest algorithm <b>alg</b>, decode it and copy it into - * <b>digest256_out</b> and return 0. Otherwise return -1. */ -int -vote_routerstatus_find_microdesc_hash(char *digest256_out, - const vote_routerstatus_t *vrs, - int method, - digest_algorithm_t alg) +/** Parse and extract all SR commits from <b>tokens</b> and place them in + * <b>ns</b>. */ +static void +extract_shared_random_commits(networkstatus_t *ns, const smartlist_t *tokens) { - /* XXXX only returns the sha256 method. */ - const vote_microdesc_hash_t *h; - char mstr[64]; - size_t mlen; - char dstr[64]; + smartlist_t *chunks = NULL; - tor_snprintf(mstr, sizeof(mstr), "%d", method); - mlen = strlen(mstr); - tor_snprintf(dstr, sizeof(dstr), " %s=", - crypto_digest_algorithm_get_name(alg)); + tor_assert(ns); + tor_assert(tokens); + /* Commits are only present in a vote. */ + tor_assert(ns->type == NS_TYPE_VOTE); - for (h = vrs->microdesc; h; h = h->next) { - const char *cp = h->microdesc_hash_line; - size_t num_len; - /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in - * the first part. */ - while (1) { - num_len = strspn(cp, "1234567890"); - if (num_len == mlen && fast_memeq(mstr, cp, mlen)) { - /* This is the line. */ - char buf[BASE64_DIGEST256_LEN+1]; - /* XXXX ignores extraneous stuff if the digest is too long. This - * seems harmless enough, right? */ - cp = strstr(cp, dstr); - if (!cp) - return -1; - cp += strlen(dstr); - strlcpy(buf, cp, sizeof(buf)); - return digest256_from_base64(digest256_out, buf); + ns->sr_info.commits = smartlist_new(); + + smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT); + /* It's normal that a vote might contain no commits even if it participates + * in the SR protocol. Don't treat it as an error. */ + if (commits == NULL) { + goto end; + } + + /* Parse the commit. We do NO validation of number of arguments or ordering + * for forward compatibility, it's the parse commit job to inform us if it's + * supported or not. */ + chunks = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) { + /* Extract all arguments and put them in the chunks list. */ + for (int i = 0; i < tok->n_args; i++) { + smartlist_add(chunks, tok->args[i]); + } + sr_commit_t *commit = sr_parse_commit(chunks); + smartlist_clear(chunks); + if (commit == NULL) { + /* Get voter identity so we can warn that this dirauth vote contains + * commit we can't parse. */ + networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); + tor_assert(voter); + log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.", + escaped(tok->object_body), + hex_str(voter->identity_digest, + sizeof(voter->identity_digest))); + /* Commitment couldn't be parsed. Continue onto the next commit because + * this one could be unsupported for instance. */ + continue; + } + /* Add newly created commit object to the vote. */ + smartlist_add(ns->sr_info.commits, commit); + } SMARTLIST_FOREACH_END(tok); + + end: + smartlist_free(chunks); + smartlist_free(commits); +} + +/* Using the given directory tokens in tokens, parse the shared random commits + * and put them in the given vote document ns. + * + * This also sets the SR participation flag if present in the vote. */ +void +dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens) +{ + /* Does this authority participates in the SR protocol? */ + directory_token_t *tok = find_opt_by_keyword(tokens, K_SR_FLAG); + if (tok) { + ns->sr_info.participate = 1; + /* Get the SR commitments and reveals from the vote. */ + extract_shared_random_commits(ns, tokens); + } +} + +/* For the given vote, free the shared random commits if any. */ +void +dirvote_clear_commits(networkstatus_t *ns) +{ + tor_assert(ns->type == NS_TYPE_VOTE); + + if (ns->sr_info.commits) { + SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c, + sr_commit_free(c)); + smartlist_free(ns->sr_info.commits); + } +} + +/* The given url is the /tor/status-vote GET directory request. Populates the + * items list with strings that we can compress on the fly and dir_items with + * cached_dir_t objects that have a precompressed deflated version. */ +void +dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items, + smartlist_t *dir_items) +{ + int current; + + url += strlen("/tor/status-vote/"); + current = !strcmpstart(url, "current/"); + url = strchr(url, '/'); + tor_assert(url); + ++url; + if (!strcmp(url, "consensus")) { + const char *item; + tor_assert(!current); /* we handle current consensus specially above, + * since it wants to be spooled. */ + if ((item = dirvote_get_pending_consensus(FLAV_NS))) + smartlist_add(items, (char*)item); + } else if (!current && !strcmp(url, "consensus-signatures")) { + /* XXXX the spec says that we should implement + * current/consensus-signatures too. It doesn't seem to be needed, + * though. */ + const char *item; + if ((item=dirvote_get_pending_detached_signatures())) + smartlist_add(items, (char*)item); + } else if (!strcmp(url, "authority")) { + const cached_dir_t *d; + int flags = DGV_BY_ID | + (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); + if ((d=dirvote_get_vote(NULL, flags))) + smartlist_add(dir_items, (cached_dir_t*)d); + } else { + const cached_dir_t *d; + smartlist_t *fps = smartlist_new(); + int flags; + if (!strcmpstart(url, "d/")) { + url += 2; + flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS; + } else { + flags = DGV_BY_ID | + (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); + } + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); + SMARTLIST_FOREACH(fps, char *, fp, { + if ((d = dirvote_get_vote(fp, flags))) + smartlist_add(dir_items, (cached_dir_t*)d); + tor_free(fp); + }); + smartlist_free(fps); + } +} + +/** Get the best estimate of a router's bandwidth for dirauth purposes, + * preferring measured to advertised values if available. */ +static uint32_t +dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri) +{ + uint32_t bw_kb = 0; + /* + * Yeah, measured bandwidths in measured_bw_line_t are (implicitly + * signed) longs and the ones router_get_advertised_bandwidth() returns + * are uint32_t. + */ + long mbw_kb = 0; + + if (ri) { + /* + * * First try to see if we have a measured bandwidth; don't bother with + * as_of_out here, on the theory that a stale measured bandwidth is still + * better to trust than an advertised one. + */ + if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest, + &mbw_kb, NULL)) { + /* Got one! */ + bw_kb = (uint32_t)mbw_kb; + } else { + /* If not, fall back to advertised */ + bw_kb = router_get_advertised_bandwidth(ri) / 1000; + } + } + + return bw_kb; +} + +/** Helper for sorting: compares two routerinfos first by address, and then by + * descending order of "usefulness". (An authority is more useful than a + * non-authority; a running router is more useful than a non-running router; + * and a router with more bandwidth is more useful than one with less.) + **/ +static int +compare_routerinfo_by_ip_and_bw_(const void **a, const void **b) +{ + routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; + int first_is_auth, second_is_auth; + uint32_t bw_kb_first, bw_kb_second; + const node_t *node_first, *node_second; + int first_is_running, second_is_running; + + /* we return -1 if first should appear before second... that is, + * if first is a better router. */ + if (first->addr < second->addr) + return -1; + else if (first->addr > second->addr) + return 1; + + /* Potentially, this next bit could cause k n lg n memeq calls. But in + * reality, we will almost never get here, since addresses will usually be + * different. */ + + first_is_auth = + router_digest_is_trusted_dir(first->cache_info.identity_digest); + second_is_auth = + router_digest_is_trusted_dir(second->cache_info.identity_digest); + + if (first_is_auth && !second_is_auth) + return -1; + else if (!first_is_auth && second_is_auth) + return 1; + + node_first = node_get_by_id(first->cache_info.identity_digest); + node_second = node_get_by_id(second->cache_info.identity_digest); + first_is_running = node_first && node_first->is_running; + second_is_running = node_second && node_second->is_running; + + if (first_is_running && !second_is_running) + return -1; + else if (!first_is_running && second_is_running) + return 1; + + bw_kb_first = dirserv_get_bandwidth_for_router_kb(first); + bw_kb_second = dirserv_get_bandwidth_for_router_kb(second); + + if (bw_kb_first > bw_kb_second) + return -1; + else if (bw_kb_first < bw_kb_second) + return 1; + + /* They're equal! Compare by identity digest, so there's a + * deterministic order and we avoid flapping. */ + return fast_memcmp(first->cache_info.identity_digest, + second->cache_info.identity_digest, + DIGEST_LEN); +} + +/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t + * whose keys are the identity digests of those routers that we're going to + * exclude for Sybil-like appearance. */ +static digestmap_t * +get_possible_sybil_list(const smartlist_t *routers) +{ + const or_options_t *options = get_options(); + digestmap_t *omit_as_sybil; + smartlist_t *routers_by_ip = smartlist_new(); + uint32_t last_addr; + int addr_count; + /* Allow at most this number of Tor servers on a single IP address, ... */ + int max_with_same_addr = options->AuthDirMaxServersPerAddr; + if (max_with_same_addr <= 0) + max_with_same_addr = INT_MAX; + + smartlist_add_all(routers_by_ip, routers); + smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_); + omit_as_sybil = digestmap_new(); + + last_addr = 0; + addr_count = 0; + SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) { + if (last_addr != ri->addr) { + last_addr = ri->addr; + addr_count = 1; + } else if (++addr_count > max_with_same_addr) { + digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri); + } + } SMARTLIST_FOREACH_END(ri); + + smartlist_free(routers_by_ip); + return omit_as_sybil; +} + +/** Given a platform string as in a routerinfo_t (possibly null), return a + * newly allocated version string for a networkstatus document, or NULL if the + * platform doesn't give a Tor version. */ +static char * +version_from_platform(const char *platform) +{ + if (platform && !strcmpstart(platform, "Tor ")) { + const char *eos = find_whitespace(platform+4); + if (eos && !strcmpstart(eos, " (r")) { + /* XXXX Unify this logic with the other version extraction + * logic in routerparse.c. */ + eos = find_whitespace(eos+1); + } + if (eos) { + return tor_strndup(platform, eos-platform); + } + } + return NULL; +} + +/** Given a (possibly empty) list of config_line_t, each line of which contains + * a list of comma-separated version numbers surrounded by optional space, + * allocate and return a new string containing the version numbers, in order, + * separated by commas. Used to generate Recommended(Client|Server)?Versions + */ +char * +format_recommended_version_list(const config_line_t *ln, int warn) +{ + smartlist_t *versions; + char *result; + versions = smartlist_new(); + for ( ; ln; ln = ln->next) { + smartlist_split_string(versions, ln->value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + } + + /* Handle the case where a dirauth operator has accidentally made some + * versions space-separated instead of comma-separated. */ + smartlist_t *more_versions = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(versions, char *, v) { + if (strchr(v, ' ')) { + if (warn) + log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. " + "(These are supposed to be comma-separated; I'll pretend you " + "used commas instead.)", escaped(v)); + SMARTLIST_DEL_CURRENT(versions, v); + smartlist_split_string(more_versions, v, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(v); + } + } SMARTLIST_FOREACH_END(v); + smartlist_add_all(versions, more_versions); + smartlist_free(more_versions); + + /* Check to make sure everything looks like a version. */ + if (warn) { + SMARTLIST_FOREACH_BEGIN(versions, const char *, v) { + tor_version_t ver; + if (tor_version_parse(v, &ver) < 0) { + log_warn(LD_DIRSERV, "Recommended version %s does not look valid. " + " (I'll include it anyway, since you told me to.)", + escaped(v)); } - if (num_len == 0 || cp[num_len] != ',') - break; - cp += num_len + 1; + } SMARTLIST_FOREACH_END(v); + } + + sort_version_list(versions, 1); + result = smartlist_join_strings(versions,",",0,NULL); + SMARTLIST_FOREACH(versions,char *,s,tor_free(s)); + smartlist_free(versions); + return result; +} + +/** If there are entries in <b>routers</b> with exactly the same ed25519 keys, + * remove the older one. If they are exactly the same age, remove the one + * with the greater descriptor digest. May alter the order of the list. */ +static void +routers_make_ed_keys_unique(smartlist_t *routers) +{ + routerinfo_t *ri2; + digest256map_t *by_ed_key = digest256map_new(); + + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + ri->omit_from_vote = 0; + if (ri->cache_info.signing_key_cert == NULL) + continue; /* No ed key */ + const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey; + if ((ri2 = digest256map_get(by_ed_key, pk))) { + /* Duplicate; must omit one. Set the omit_from_vote flag in whichever + * one has the earlier published_on. */ + const time_t ri_pub = ri->cache_info.published_on; + const time_t ri2_pub = ri2->cache_info.published_on; + if (ri2_pub < ri_pub || + (ri2_pub == ri_pub && + fast_memcmp(ri->cache_info.signed_descriptor_digest, + ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { + digest256map_set(by_ed_key, pk, ri); + ri2->omit_from_vote = 1; + } else { + ri->omit_from_vote = 1; + } + } else { + /* Add to map */ + digest256map_set(by_ed_key, pk, ri); } + } SMARTLIST_FOREACH_END(ri); + + digest256map_free(by_ed_key, NULL); + + /* Now remove every router where the omit_from_vote flag got set. */ + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { + if (ri->omit_from_vote) { + SMARTLIST_DEL_CURRENT(routers, ri); + } + } SMARTLIST_FOREACH_END(ri); +} + +/** Routerstatus <b>rs</b> is part of a group of routers that are on + * too narrow an IP-space. Clear out its flags since we don't want it be used + * because of its Sybil-like appearance. + * + * Leave its BadExit flag alone though, since if we think it's a bad exit, + * we want to vote that way in case all the other authorities are voting + * Running and Exit. + */ +static void +clear_status_flags_on_sybil(routerstatus_t *rs) +{ + rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = + rs->is_flagged_running = rs->is_named = rs->is_valid = + rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0; + /* FFFF we might want some mechanism to check later on if we + * missed zeroing any flags: it's easy to add a new flag but + * forget to add it to this clause. */ +} + +/** Return a new networkstatus_t* containing our current opinion. (For v3 + * authorities) */ +networkstatus_t * +dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, + authority_cert_t *cert) +{ + const or_options_t *options = get_options(); + networkstatus_t *v3_out = NULL; + uint32_t addr; + char *hostname = NULL, *client_versions = NULL, *server_versions = NULL; + const char *contact; + smartlist_t *routers, *routerstatuses; + char identity_digest[DIGEST_LEN]; + char signing_key_digest[DIGEST_LEN]; + int listbadexits = options->AuthDirListBadExits; + routerlist_t *rl = router_get_routerlist(); + time_t now = time(NULL); + time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; + networkstatus_voter_info_t *voter = NULL; + vote_timing_t timing; + digestmap_t *omit_as_sybil = NULL; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; + + tor_assert(private_key); + tor_assert(cert); + + if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { + log_err(LD_BUG, "Error computing signing key digest"); + return NULL; } - return -1; + if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) { + log_err(LD_BUG, "Error computing identity key digest"); + return NULL; + } + if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { + log_warn(LD_NET, "Couldn't resolve my hostname"); + return NULL; + } + if (!hostname || !strchr(hostname, '.')) { + tor_free(hostname); + hostname = tor_dup_ip(addr); + } + + if (options->VersioningAuthoritativeDir) { + client_versions = + format_recommended_version_list(options->RecommendedClientVersions, 0); + server_versions = + format_recommended_version_list(options->RecommendedServerVersions, 0); + } + + contact = get_options()->ContactInfo; + if (!contact) + contact = "(none)"; + + /* + * Do this so dirserv_compute_performance_thresholds() and + * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. + */ + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } else { + /* + * No bandwidths file; clear the measured bandwidth cache in case we had + * one last time around. + */ + if (dirserv_get_measured_bw_cache_size() > 0) { + dirserv_clear_measured_bw_cache(); + } + } + + /* precompute this part, since we need it to decide what "stable" + * means. */ + SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + dirserv_set_router_is_running(ri, now); + }); + + routers = smartlist_new(); + smartlist_add_all(routers, rl->routers); + routers_make_ed_keys_unique(routers); + /* After this point, don't use rl->routers; use 'routers' instead. */ + routers_sort_by_identity(routers); + omit_as_sybil = get_possible_sybil_list(routers); + + DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) { + (void) ignore; + rep_hist_make_router_pessimal(sybil_id, now); + } DIGESTMAP_FOREACH_END; + + /* Count how many have measured bandwidths so we know how to assign flags; + * this must come before dirserv_compute_performance_thresholds() */ + dirserv_count_measured_bws(routers); + + dirserv_compute_performance_thresholds(omit_as_sybil); + + routerstatuses = smartlist_new(); + microdescriptors = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + /* If it has a protover list and contains a protocol name greater than + * MAX_PROTOCOL_NAME_LENGTH, skip it. */ + if (ri->protocol_list && + protover_contains_long_protocol_names(ri->protocol_list)) { + continue; + } + if (ri->cache_info.published_on >= cutoff) { + routerstatus_t *rs; + vote_routerstatus_t *vrs; + node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); + if (!node) + continue; + + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + set_routerstatus_from_routerinfo(rs, node, ri, now, + listbadexits); + + if (ri->cache_info.signing_key_cert) { + memcpy(vrs->ed25519_id, + ri->cache_info.signing_key_cert->signing_key.pubkey, + ED25519_PUBKEY_LEN); + } + + if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) + clear_status_flags_on_sybil(rs); + + if (!vote_on_reachability) + rs->is_flagged_running = 0; + + vrs->version = version_from_platform(ri->platform); + if (ri->protocol_list) { + vrs->protocols = tor_strdup(ri->protocol_list); + } else { + vrs->protocols = tor_strdup( + protover_compute_for_old_tor(vrs->version)); + } + vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now, + microdescriptors); + + smartlist_add(routerstatuses, vrs); + } + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); + + /* Apply guardfraction information to routerstatuses. */ + if (options->GuardfractionFile) { + dirserv_read_guardfraction_file(options->GuardfractionFile, + routerstatuses); + } + + /* This pass through applies the measured bw lines to the routerstatuses */ + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } else { + /* + * No bandwidths file; clear the measured bandwidth cache in case we had + * one last time around. + */ + if (dirserv_get_measured_bw_cache_size() > 0) { + dirserv_clear_measured_bw_cache(); + } + } + + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); + + v3_out->type = NS_TYPE_VOTE; + dirvote_get_preferred_voting_intervals(&timing); + v3_out->published = now; + { + char tbuf[ISO_TIME_LEN+1]; + networkstatus_t *current_consensus = + networkstatus_get_live_consensus(now); + long last_consensus_interval; /* only used to pick a valid_after */ + if (current_consensus) + last_consensus_interval = current_consensus->fresh_until - + current_consensus->valid_after; + else + last_consensus_interval = options->TestingV3AuthInitialVotingInterval; + v3_out->valid_after = + voting_schedule_get_start_of_next_interval(now, + (int)last_consensus_interval, + options->TestingV3AuthVotingStartOffset); + format_iso_time(tbuf, v3_out->valid_after); + log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " + "consensus_set=%d, last_interval=%d", + tbuf, current_consensus?1:0, (int)last_consensus_interval); + } + v3_out->fresh_until = v3_out->valid_after + timing.vote_interval; + v3_out->valid_until = v3_out->valid_after + + (timing.vote_interval * timing.n_intervals_valid); + v3_out->vote_seconds = timing.vote_delay; + v3_out->dist_seconds = timing.dist_delay; + tor_assert(v3_out->vote_seconds > 0); + tor_assert(v3_out->dist_seconds > 0); + tor_assert(timing.n_intervals_valid > 0); + + v3_out->client_versions = client_versions; + v3_out->server_versions = server_versions; + + /* These are hardwired, to avoid disaster. */ + v3_out->recommended_relay_protocols = + tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=4 Microdesc=1-2 Relay=2"); + v3_out->recommended_client_protocols = + tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=4 Microdesc=1-2 Relay=2"); + v3_out->required_client_protocols = + tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=4 Microdesc=1-2 Relay=2"); + v3_out->required_relay_protocols = + tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=3-4 Microdesc=1 Relay=1-2"); + + /* We are not allowed to vote to require anything we don't have. */ + tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL)); + tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL)); + + /* We should not recommend anything we don't have. */ + tor_assert_nonfatal(protover_all_supported( + v3_out->recommended_relay_protocols, NULL)); + tor_assert_nonfatal(protover_all_supported( + v3_out->recommended_client_protocols, NULL)); + + v3_out->package_lines = smartlist_new(); + { + config_line_t *cl; + for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { + if (validate_recommended_package_line(cl->value)) + smartlist_add_strdup(v3_out->package_lines, cl->value); + } + } + + v3_out->known_flags = smartlist_new(); + smartlist_split_string(v3_out->known_flags, + "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (vote_on_reachability) + smartlist_add_strdup(v3_out->known_flags, "Running"); + if (listbadexits) + smartlist_add_strdup(v3_out->known_flags, "BadExit"); + smartlist_sort_strings(v3_out->known_flags); + + if (options->ConsensusParams) { + v3_out->net_params = smartlist_new(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->nickname = tor_strdup(options->Nickname); + memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_new(); + voter->address = hostname; + voter->addr = addr; + voter->dir_port = router_get_advertised_dir_port(options, 0); + voter->or_port = router_get_advertised_or_port(options); + voter->contact = tor_strdup(contact); + if (options->V3AuthUseLegacyKey) { + authority_cert_t *c = get_my_v3_legacy_cert(); + if (c) { + if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { + log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); + memset(voter->legacy_id_digest, 0, DIGEST_LEN); + } + } + } + + v3_out->voters = smartlist_new(); + smartlist_add(v3_out->voters, voter); + v3_out->cert = authority_cert_dup(cert); + v3_out->routerstatus_list = routerstatuses; + /* Note: networkstatus_digest is unset; it won't get set until we actually + * format the vote. */ + + return v3_out; } diff --git a/src/or/dirvote.h b/src/or/dirauth/dirvote.h index deeb27bfe1..b69bbbf5d9 100644 --- a/src/or/dirvote.h +++ b/src/or/dirauth/dirvote.h @@ -12,8 +12,6 @@ #ifndef TOR_DIRVOTE_H #define TOR_DIRVOTE_H -#include "testsupport.h" - /* * Ideally, assuming synced clocks, we should only need 1 second for each of: * - Vote @@ -56,57 +54,11 @@ #define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0 /** The lowest consensus method that we currently support. */ -#define MIN_SUPPORTED_CONSENSUS_METHOD 13 +#define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ #define MAX_SUPPORTED_CONSENSUS_METHOD 28 -/** Lowest consensus method where microdesc consensuses omit any entry - * with no microdesc. */ -#define MIN_METHOD_FOR_MANDATORY_MICRODESC 13 - -/** Lowest consensus method that contains "a" lines. */ -#define MIN_METHOD_FOR_A_LINES 14 - -/** Lowest consensus method where microdescs may include a "p6" line. */ -#define MIN_METHOD_FOR_P6_LINES 15 - -/** Lowest consensus method where microdescs may include an onion-key-ntor - * line */ -#define MIN_METHOD_FOR_NTOR_KEY 16 - -/** Lowest consensus method that ensures that authorities output an - * Unmeasured=1 flag for unmeasured bandwidths */ -#define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17 - -/** Lowest consensus method where authorities may include an "id" line in - * microdescriptors. */ -#define MIN_METHOD_FOR_ID_HASH_IN_MD 18 - -/** Lowest consensus method where we include "package" lines*/ -#define MIN_METHOD_FOR_PACKAGE_LINES 19 - -/** Lowest consensus method where authorities may include - * GuardFraction information in microdescriptors. */ -#define MIN_METHOD_FOR_GUARDFRACTION 20 - -/** Lowest consensus method where authorities may include an "id" line for - * ed25519 identities in microdescriptors. (Broken; see - * consensus_method_is_supported() for more info.) */ -#define MIN_METHOD_FOR_ED25519_ID_IN_MD 21 - -/** Lowest consensus method where authorities vote on ed25519 ids and ensure - * ed25519 id consistency. */ -#define MIN_METHOD_FOR_ED25519_ID_VOTING 22 - -/** Lowest consensus method where authorities may include a shared random - * value(s). */ -#define MIN_METHOD_FOR_SHARED_RANDOM 23 - -/** Lowest consensus method where authorities drop all nodes that don't get - * the Valid flag. */ -#define MIN_METHOD_FOR_EXCLUDING_INVALID_NODES 24 - /** Lowest consensus method where authorities vote on required/recommended * protocols. */ #define MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS 25 @@ -132,74 +84,27 @@ * get confused with the above macros.) */ #define DEFAULT_MAX_UNMEASURED_BW_KB 20 +/* Directory Get Vote (DGV) flags for dirvote_get_vote(). */ +#define DGV_BY_ID 1 +#define DGV_INCLUDE_PENDING 2 +#define DGV_INCLUDE_PREVIOUS 4 + +/* + * Public API. Used outside of the dirauth subsystem. + * + * We need to nullify them if the module is disabled. + */ +#ifdef HAVE_MODULE_DIRAUTH + +time_t dirvote_act(const or_options_t *options, time_t now); void dirvote_free_all(void); -/* vote manipulation */ -char *networkstatus_compute_consensus(smartlist_t *votes, - int total_authorities, - crypto_pk_t *identity_key, - crypto_pk_t *signing_key, - const char *legacy_identity_key_digest, - crypto_pk_t *legacy_signing_key, - consensus_flavor_t flavor); -int networkstatus_add_detached_signatures(networkstatus_t *target, - ns_detached_signatures_t *sigs, - const char *source, - int severity, - const char **msg_out); -char *networkstatus_get_detached_signatures(smartlist_t *consensuses); -void ns_detached_signatures_free_(ns_detached_signatures_t *s); -#define ns_detached_signatures_free(s) \ - FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s)) - -/* cert manipulation */ -authority_cert_t *authority_cert_dup(authority_cert_t *cert); - -/* vote scheduling */ - -/** Scheduling information for a voting interval. */ -typedef struct { - /** When do we generate and distribute our vote for this interval? */ - time_t voting_starts; - /** When do we send an HTTP request for any votes that we haven't - * been posted yet?*/ - time_t fetch_missing_votes; - /** When do we give up on getting more votes and generate a consensus? */ - time_t voting_ends; - /** When do we send an HTTP request for any signatures we're expecting to - * see on the consensus? */ - time_t fetch_missing_signatures; - /** When do we publish the consensus? */ - time_t interval_starts; - - /* True iff we have generated and distributed our vote. */ - int have_voted; - /* True iff we've requested missing votes. */ - int have_fetched_missing_votes; - /* True iff we have built a consensus and sent the signatures around. */ - int have_built_consensus; - /* True iff we've fetched missing signatures. */ - int have_fetched_missing_signatures; - /* True iff we have published our consensus. */ - int have_published_consensus; - - /* True iff this voting schedule was set on demand meaning not through the - * normal vote operation of a dirauth or when a consensus is set. This only - * applies to a directory authority that needs to recalculate the voting - * timings only for the first vote even though this object was initilized - * prior to voting. */ - int created_on_demand; -} voting_schedule_t; - -void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, - int interval, - int offset); -void dirvote_recalculate_timing(const or_options_t *options, time_t now); -void dirvote_act(const or_options_t *options, time_t now); -time_t dirvote_get_next_valid_after_time(void); - -/* invoked on timers and by outside triggers. */ +void dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens); +void dirvote_clear_commits(networkstatus_t *ns); +void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items, + smartlist_t *dir_items); + +/* Storing signatures and votes functions */ struct pending_vote_t * dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out); @@ -207,15 +112,82 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg_out); +#else /* HAVE_MODULE_DIRAUTH */ + +static inline time_t +dirvote_act(const or_options_t *options, time_t now) +{ + (void) options; + (void) now; + return TIME_MAX; +} + +static inline void +dirvote_free_all(void) +{ +} + +static inline void +dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens) +{ + (void) ns; + (void) tokens; +} + +static inline void +dirvote_clear_commits(networkstatus_t *ns) +{ + (void) ns; +} + +static inline void +dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items, + smartlist_t *dir_items) +{ + (void) url; + (void) items; + (void) dir_items; +} + +static inline struct pending_vote_t * +dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) +{ + (void) vote_body; + /* If the dirauth module is disabled, this should NEVER be called else we + * failed to safeguard the dirauth module. */ + tor_assert_nonfatal_unreached(); + + /* We need to send out an error code. */ + *status_out = 400; + *msg_out = "No directory authority support"; + return NULL; +} + +static inline int +dirvote_add_signatures(const char *detached_signatures_body, const char *source, + const char **msg_out) +{ + (void) detached_signatures_body; + (void) source; + (void) msg_out; + /* If the dirauth module is disabled, this should NEVER be called else we + * failed to safeguard the dirauth module. */ + tor_assert_nonfatal_unreached(); + return 0; +} + +#endif /* HAVE_MODULE_DIRAUTH */ + /* Item access */ MOCK_DECL(const char*, dirvote_get_pending_consensus, (consensus_flavor_t flav)); MOCK_DECL(const char*, dirvote_get_pending_detached_signatures, (void)); - -#define DGV_BY_ID 1 -#define DGV_INCLUDE_PENDING 2 -#define DGV_INCLUDE_PREVIOUS 4 const cached_dir_t *dirvote_get_vote(const char *fp, int flags); + +/* + * API used _only_ by the dirauth subsystem. + */ + void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, routerinfo_t *ri, time_t now, @@ -224,26 +196,18 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, authority_cert_t *cert); -microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri, - int consensus_method); -ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, - const microdesc_t *md, - int consensus_method_low, - int consensus_method_high); vote_microdesc_hash_t *dirvote_format_all_microdesc_vote_lines( const routerinfo_t *ri, time_t now, smartlist_t *microdescriptors_out); -int vote_routerstatus_find_microdesc_hash(char *digest256_out, - const vote_routerstatus_t *vrs, - int method, - digest_algorithm_t alg); -document_signature_t *voter_get_sig_by_algorithm( - const networkstatus_voter_info_t *voter, - digest_algorithm_t alg); - +/* + * Exposed functions for unit tests. + */ #ifdef DIRVOTE_PRIVATE + +/* Cert manipulation */ +STATIC authority_cert_t *authority_cert_dup(authority_cert_t *cert); STATIC int32_t dirvote_get_intermediate_param_value( const smartlist_t *param_list, const char *keyword, @@ -258,6 +222,25 @@ STATIC int networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, int64_t M, int64_t E, int64_t D, int64_t T, int64_t weight_scale); +STATIC +char *networkstatus_compute_consensus(smartlist_t *votes, + int total_authorities, + crypto_pk_t *identity_key, + crypto_pk_t *signing_key, + const char *legacy_identity_key_digest, + crypto_pk_t *legacy_signing_key, + consensus_flavor_t flavor); +STATIC +int networkstatus_add_detached_signatures(networkstatus_t *target, + ns_detached_signatures_t *sigs, + const char *source, + int severity, + const char **msg_out); +STATIC +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); +STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri, + int consensus_method); + #endif /* defined(DIRVOTE_PRIVATE) */ #endif /* !defined(TOR_DIRVOTE_H) */ diff --git a/src/or/dirauth/mode.h b/src/or/dirauth/mode.h new file mode 100644 index 0000000000..8a0d3142f1 --- /dev/null +++ b/src/or/dirauth/mode.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file mode.h + * \brief Standalone header file for directory authority mode. + **/ + +#ifndef TOR_DIRAUTH_MODE_H +#define TOR_DIRAUTH_MODE_H + +#ifdef HAVE_MODULE_DIRAUTH + +#include "router.h" + +/* Return true iff we believe ourselves to be a v3 authoritative directory + * server. */ +static inline int +authdir_mode_v3(const or_options_t *options) +{ + return authdir_mode(options) && options->V3AuthoritativeDir != 0; +} + +#else /* HAVE_MODULE_DIRAUTH */ + +/* Without the dirauth module, we can't be a v3 directory authority, ever. */ + +static inline int +authdir_mode_v3(const or_options_t *options) +{ + (void) options; + return 0; +} + +#endif /* HAVE_MODULE_DIRAUTH */ + +#endif /* TOR_MODE_H */ + diff --git a/src/or/shared_random.c b/src/or/dirauth/shared_random.c index 13416d6bc7..6dd1f330e0 100644 --- a/src/or/shared_random.c +++ b/src/or/dirauth/shared_random.c @@ -91,13 +91,19 @@ #include "shared_random.h" #include "config.h" #include "confparse.h" -#include "dirvote.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "networkstatus.h" -#include "routerkeys.h" #include "router.h" +#include "routerkeys.h" #include "routerlist.h" +#include "shared_random_client.h" #include "shared_random_state.h" #include "util.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" /* String prefix of shared random values in votes/consensuses. */ static const char previous_srv_str[] = "shared-rand-previous-value"; @@ -498,20 +504,6 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase) return vote_line; } -/* Convert a given srv object to a string for the control port. This doesn't - * fail and the srv object MUST be valid. */ -static char * -srv_to_control_string(const sr_srv_t *srv) -{ - char *srv_str; - char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1]; - tor_assert(srv); - - sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv); - tor_asprintf(&srv_str, "%s", srv_hash_encoded); - return srv_str; -} - /* Return a heap allocated string that contains the given <b>srv</b> string * representation formatted for a networkstatus document using the * <b>key</b> as the start of the line. This doesn't return NULL. */ @@ -874,27 +866,6 @@ get_majority_srv_from_votes(const smartlist_t *votes, int current) return the_srv; } -/* Encode the given shared random value and put it in dst. Destination - * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */ -void -sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv) -{ - int ret; - /* Extra byte for the NULL terminated char. */ - char buf[SR_SRV_VALUE_BASE64_LEN + 1]; - - tor_assert(dst); - tor_assert(srv); - tor_assert(dst_len >= sizeof(buf)); - - ret = base64_encode(buf, sizeof(buf), (const char *) srv->value, - sizeof(srv->value), 0); - /* Always expect the full length without the NULL byte. */ - tor_assert(ret == (sizeof(buf) - 1)); - tor_assert(ret <= (int) dst_len); - strlcpy(dst, buf, dst_len); -} - /* Free a commit object. */ void sr_commit_free_(sr_commit_t *commit) @@ -1036,55 +1007,6 @@ sr_compute_srv(void) tor_free(reveals); } -/* Parse a list of arguments from a SRV value either from a vote, consensus - * or from our disk state and return a newly allocated srv object. NULL is - * returned on error. - * - * The arguments' order: - * num_reveals, value - */ -sr_srv_t * -sr_parse_srv(const smartlist_t *args) -{ - char *value; - int ok, ret; - uint64_t num_reveals; - sr_srv_t *srv = NULL; - - tor_assert(args); - - if (smartlist_len(args) < 2) { - goto end; - } - - /* First argument is the number of reveal values */ - num_reveals = tor_parse_uint64(smartlist_get(args, 0), - 10, 0, UINT64_MAX, &ok, NULL); - if (!ok) { - goto end; - } - /* Second and last argument is the shared random value it self. */ - value = smartlist_get(args, 1); - if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) { - goto end; - } - - srv = tor_malloc_zero(sizeof(*srv)); - srv->num_reveals = num_reveals; - /* We subtract one byte from the srclen because the function ignores the - * '=' character in the given buffer. This is broken but it's a documented - * behavior of the implementation. */ - ret = base64_decode((char *) srv->value, sizeof(srv->value), value, - SR_SRV_VALUE_BASE64_LEN - 1); - if (ret != sizeof(srv->value)) { - tor_free(srv); - srv = NULL; - goto end; - } - end: - return srv; -} - /* Parse a commit from a vote or from our disk state and return a newly * allocated commit object. NULL is returned on error. * @@ -1333,7 +1255,7 @@ sr_act_post_consensus(const networkstatus_t *consensus) } /* Prepare our state so that it's ready for the next voting period. */ - sr_state_update(dirvote_get_next_valid_after_time()); + sr_state_update(voting_schedule_get_next_valid_after_time()); } /* Initialize shared random subsystem. This MUST be called early in the boot @@ -1352,84 +1274,6 @@ sr_save_and_cleanup(void) sr_cleanup(); } -/* Return the current SRV string representation for the control port. Return a - * newly allocated string on success containing the value else "" if not found - * or if we don't have a valid consensus yet. */ -char * -sr_get_current_for_control(void) -{ - char *srv_str; - const networkstatus_t *c = networkstatus_get_latest_consensus(); - if (c && c->sr_info.current_srv) { - srv_str = srv_to_control_string(c->sr_info.current_srv); - } else { - srv_str = tor_strdup(""); - } - return srv_str; -} - -/* Return the previous SRV string representation for the control port. Return - * a newly allocated string on success containing the value else "" if not - * found or if we don't have a valid consensus yet. */ -char * -sr_get_previous_for_control(void) -{ - char *srv_str; - const networkstatus_t *c = networkstatus_get_latest_consensus(); - if (c && c->sr_info.previous_srv) { - srv_str = srv_to_control_string(c->sr_info.previous_srv); - } else { - srv_str = tor_strdup(""); - } - return srv_str; -} - -/* Return current shared random value from the latest consensus. Caller can - * NOT keep a reference to the returned pointer. Return NULL if none. */ -const sr_srv_t * -sr_get_current(const networkstatus_t *ns) -{ - const networkstatus_t *consensus; - - /* Use provided ns else get a live one */ - if (ns) { - consensus = ns; - } else { - consensus = networkstatus_get_live_consensus(approx_time()); - } - /* Ideally we would never be asked for an SRV without a live consensus. Make - * sure this assumption is correct. */ - tor_assert_nonfatal(consensus); - - if (consensus) { - return consensus->sr_info.current_srv; - } - return NULL; -} - -/* Return previous shared random value from the latest consensus. Caller can - * NOT keep a reference to the returned pointer. Return NULL if none. */ -const sr_srv_t * -sr_get_previous(const networkstatus_t *ns) -{ - const networkstatus_t *consensus; - - /* Use provided ns else get a live one */ - if (ns) { - consensus = ns; - } else { - consensus = networkstatus_get_live_consensus(approx_time()); - } - /* Ideally we would never be asked for an SRV without a live consensus. Make - * sure this assumption is correct. */ - tor_assert_nonfatal(consensus); - - if (consensus) { - return consensus->sr_info.previous_srv; - } - return NULL; -} - #ifdef TOR_UNIT_TESTS /* Set the global value of number of SRV agreements so the test can play diff --git a/src/or/shared_random.h b/src/or/dirauth/shared_random.h index 675a8d8b06..1778ce8f09 100644 --- a/src/or/shared_random.h +++ b/src/or/dirauth/shared_random.h @@ -101,21 +101,48 @@ typedef struct sr_commit_t { /* API */ -/* Public methods: */ +/* Public methods used _outside_ of the module. + * + * We need to nullify them if the module is disabled. */ +#ifdef HAVE_MODULE_DIRAUTH int sr_init(int save_to_disk); void sr_save_and_cleanup(void); void sr_act_post_consensus(const networkstatus_t *consensus); + +#else /* HAVE_MODULE_DIRAUTH */ + +static inline int +sr_init(int save_to_disk) +{ + (void) save_to_disk; + /* Always return success. */ + return 0; +} + +static inline void +sr_save_and_cleanup(void) +{ +} + +static inline void +sr_act_post_consensus(const networkstatus_t *consensus) +{ + (void) consensus; +} + +#endif /* HAVE_MODULE_DIRAUTH */ + +/* Public methods used only by dirauth code. */ + void sr_handle_received_commits(smartlist_t *commits, crypto_pk_t *voter_key); sr_commit_t *sr_parse_commit(const smartlist_t *args); -sr_srv_t *sr_parse_srv(const smartlist_t *args); char *sr_get_string_for_vote(void); char *sr_get_string_for_consensus(const smartlist_t *votes, int32_t num_srv_agreements); void sr_commit_free_(sr_commit_t *commit); #define sr_commit_free(sr) FREE_AND_NULL(sr_commit_t, sr_commit_free_, (sr)) -void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv); /* Private methods (only used by shared_random_state.c): */ static inline @@ -128,12 +155,6 @@ void sr_compute_srv(void); sr_commit_t *sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert); -char *sr_get_current_for_control(void); -char *sr_get_previous_for_control(void); - -const sr_srv_t *sr_get_current(const networkstatus_t *ns); -const sr_srv_t *sr_get_previous(const networkstatus_t *ns); - #ifdef SHARED_RANDOM_PRIVATE /* Encode */ diff --git a/src/or/shared_random_state.c b/src/or/dirauth/shared_random_state.c index 7bac8e9482..fc0e4e5630 100644 --- a/src/or/shared_random_state.c +++ b/src/or/dirauth/shared_random_state.c @@ -11,13 +11,16 @@ #define SHARED_RANDOM_STATE_PRIVATE #include "or.h" -#include "shared_random.h" #include "config.h" #include "confparse.h" -#include "dirvote.h" +#include "crypto_util.h" +#include "dirauth/dirvote.h" #include "networkstatus.h" #include "router.h" +#include "shared_random.h" +#include "shared_random_client.h" #include "shared_random_state.h" +#include "voting_schedule.h" /* Default filename of the shared random state on disk. */ static const char default_fname[] = "sr-state"; @@ -53,10 +56,6 @@ DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t); VAR(#member, conftype, member, initvalue) /* Our persistent state magic number. */ #define SR_DISK_STATE_MAGIC 0x98AB1254 -/* Each protocol phase has 12 rounds */ -#define SHARED_RANDOM_N_ROUNDS 12 -/* Number of phase we have in a protocol. */ -#define SHARED_RANDOM_N_PHASES 2 static int disk_state_validate_cb(void *old_state, void *state, void *default_state, @@ -115,81 +114,6 @@ get_phase_str(sr_phase_t phase) return the_string; } - -/* Return the voting interval of the tor vote subsystem. */ -static int -get_voting_interval(void) -{ - int interval; - networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL)); - - if (consensus) { - interval = (int)(consensus->fresh_until - consensus->valid_after); - } else { - /* Same for both a testing and real network. We voluntarily ignore the - * InitialVotingInterval since it complexifies things and it doesn't - * affect the SR protocol. */ - interval = get_options()->V3AuthVotingInterval; - } - tor_assert(interval > 0); - return interval; -} - -/* Given the time <b>now</b>, return the start time of the current round of - * the SR protocol. For example, if it's 23:47:08, the current round thus - * started at 23:47:00 for a voting interval of 10 seconds. */ -STATIC time_t -get_start_time_of_current_round(void) -{ - const or_options_t *options = get_options(); - int voting_interval = get_voting_interval(); - /* First, get the start time of the next round */ - time_t next_start = dirvote_get_next_valid_after_time(); - /* Now roll back next_start by a voting interval to find the start time of - the current round. */ - time_t curr_start = dirvote_get_start_of_next_interval( - next_start - voting_interval - 1, - voting_interval, - options->TestingV3AuthVotingStartOffset); - return curr_start; -} - -/** Return the start time of the current SR protocol run. For example, if the - * time is 23/06/2017 23:47:08 and a full SR protocol run is 24 hours, this - * function should return 23/06/2017 00:00:00. */ -time_t -sr_state_get_start_time_of_current_protocol_run(time_t now) -{ - int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; - int voting_interval = get_voting_interval(); - /* Find the time the current round started. */ - time_t beginning_of_current_round = get_start_time_of_current_round(); - - /* Get current SR protocol round */ - int current_round = (now / voting_interval) % total_rounds; - - /* Get start time by subtracting the time elapsed from the beginning of the - protocol run */ - time_t time_elapsed_since_start_of_run = current_round * voting_interval; - return beginning_of_current_round - time_elapsed_since_start_of_run; -} - -/** Return the time (in seconds) it takes to complete a full SR protocol phase - * (e.g. the commit phase). */ -unsigned int -sr_state_get_phase_duration(void) -{ - return SHARED_RANDOM_N_ROUNDS * get_voting_interval(); -} - -/** Return the time (in seconds) it takes to complete a full SR protocol run */ -unsigned int -sr_state_get_protocol_run_duration(void) -{ - int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; - return total_protocol_rounds * get_voting_interval(); -} - /* Return the time we should expire the state file created at <b>now</b>. * We expire the state file in the beginning of the next protocol run. */ STATIC time_t @@ -1370,7 +1294,7 @@ sr_state_init(int save_to_disk, int read_from_disk) /* We have a state in memory, let's make sure it's updated for the current * and next voting round. */ { - time_t valid_after = dirvote_get_next_valid_after_time(); + time_t valid_after = voting_schedule_get_next_valid_after_time(); sr_state_update(valid_after); } return 0; diff --git a/src/or/shared_random_state.h b/src/or/dirauth/shared_random_state.h index fdbbf4919a..60a326f86c 100644 --- a/src/or/shared_random_state.h +++ b/src/or/dirauth/shared_random_state.h @@ -121,16 +121,11 @@ int sr_state_is_initialized(void); void sr_state_save(void); void sr_state_free_all(void); -time_t sr_state_get_start_time_of_current_protocol_run(time_t now); -unsigned int sr_state_get_phase_duration(void); -unsigned int sr_state_get_protocol_run_duration(void); - #ifdef SHARED_RANDOM_STATE_PRIVATE STATIC int disk_state_load_from_disk_impl(const char *fname); STATIC sr_phase_t get_sr_protocol_phase(time_t valid_after); -STATIC time_t get_start_time_of_current_round(void); STATIC time_t get_state_valid_until_time(time_t now); STATIC const char *get_phase_str(sr_phase_t phase); diff --git a/src/or/directory.c b/src/or/directory.c index ddb86cd9a4..efb6cd0e94 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -18,9 +18,10 @@ #include "consdiffmgr.h" #include "control.h" #include "compat.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "entrynodes.h" #include "geoip.h" #include "hs_cache.h" @@ -41,7 +42,6 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" -#include "shared_random.h" #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #if !defined(OpenBSD) @@ -49,6 +49,10 @@ #endif #endif +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random.h" + /** * \file directory.c * \brief Code to send and fetch information from directory authorities and @@ -794,9 +798,9 @@ directory_choose_address_routerstatus(const routerstatus_t *status, * Use the preferred address and port if they are reachable, otherwise, * use the alternate address and port (if any). */ - have_or = fascist_firewall_choose_address_rs(status, - FIREWALL_OR_CONNECTION, 0, - use_or_ap); + fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0, + use_or_ap); + have_or = tor_addr_port_is_valid_ap(use_or_ap, 0); } /* DirPort connections @@ -805,9 +809,9 @@ directory_choose_address_routerstatus(const routerstatus_t *status, indirection == DIRIND_ANON_DIRPORT || (indirection == DIRIND_ONEHOP && !directory_must_use_begindir(options))) { - have_dir = fascist_firewall_choose_address_rs(status, - FIREWALL_DIR_CONNECTION, 0, - use_dir_ap); + fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0, + use_dir_ap); + have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0); } /* We rejected all addresses in the relay's status. This means we can't @@ -2438,7 +2442,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * and the date header. (We used to check now-date_header, but that's * inaccurate if we spend a lot of time downloading.) */ - apparent_skew = conn->base_.timestamp_lastwritten - date_header; + apparent_skew = conn->base_.timestamp_last_write_allowed - date_header; if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) { int trusted = router_digest_is_trusted_dir(conn->identity_digest); clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP, @@ -4439,59 +4443,15 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; { - int current; ssize_t body_len = 0; ssize_t estimated_len = 0; + int lifetime = 60; /* XXXX?? should actually use vote intervals. */ /* This smartlist holds strings that we can compress on the fly. */ smartlist_t *items = smartlist_new(); /* This smartlist holds cached_dir_t objects that have a precompressed * deflated version. */ smartlist_t *dir_items = smartlist_new(); - int lifetime = 60; /* XXXX?? should actually use vote intervals. */ - url += strlen("/tor/status-vote/"); - current = !strcmpstart(url, "current/"); - url = strchr(url, '/'); - tor_assert(url); - ++url; - if (!strcmp(url, "consensus")) { - const char *item; - tor_assert(!current); /* we handle current consensus specially above, - * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus(FLAV_NS))) - smartlist_add(items, (char*)item); - } else if (!current && !strcmp(url, "consensus-signatures")) { - /* XXXX the spec says that we should implement - * current/consensus-signatures too. It doesn't seem to be needed, - * though. */ - const char *item; - if ((item=dirvote_get_pending_detached_signatures())) - smartlist_add(items, (char*)item); - } else if (!strcmp(url, "authority")) { - const cached_dir_t *d; - int flags = DGV_BY_ID | - (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); - if ((d=dirvote_get_vote(NULL, flags))) - smartlist_add(dir_items, (cached_dir_t*)d); - } else { - const cached_dir_t *d; - smartlist_t *fps = smartlist_new(); - int flags; - if (!strcmpstart(url, "d/")) { - url += 2; - flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS; - } else { - flags = DGV_BY_ID | - (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); - } - dir_split_resource_into_fingerprints(url, fps, NULL, - DSR_HEX|DSR_SORT_UNIQ); - SMARTLIST_FOREACH(fps, char *, fp, { - if ((d = dirvote_get_vote(fp, flags))) - smartlist_add(dir_items, (cached_dir_t*)d); - tor_free(fp); - }); - smartlist_free(fps); - } + dirvote_dirreq_get_status_vote(url, items, dir_items); if (!smartlist_len(dir_items) && !smartlist_len(items)) { write_short_http_response(conn, 404, "Not found"); goto vote_done; @@ -5219,6 +5179,9 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->base_.type == CONN_TYPE_DIR); + if (conn->base_.marked_for_close) + return 0; + /* Note that we have finished writing the directory response. For direct * connections this means we're done; for tunneled connections it's only * an intermediate step. */ @@ -5302,84 +5265,71 @@ connection_dir_finished_connecting(dir_connection_t *conn) /** Decide which download schedule we want to use based on descriptor type * in <b>dls</b> and <b>options</b>. - * Then return a list of int pointers defining download delays in seconds. + * + * Then, return the initial delay for that download schedule, in seconds. + * * Helper function for download_status_increment_failure(), * download_status_reset(), and download_status_increment_attempt(). */ -STATIC const smartlist_t * -find_dl_schedule(const download_status_t *dls, const or_options_t *options) +STATIC int +find_dl_min_delay(const download_status_t *dls, const or_options_t *options) { + tor_assert(dls); + tor_assert(options); + switch (dls->schedule) { case DL_SCHED_GENERIC: /* Any other directory document */ if (dir_server_mode(options)) { /* A directory authority or directory mirror */ - return options->TestingServerDownloadSchedule; + return options->TestingServerDownloadInitialDelay; } else { - return options->TestingClientDownloadSchedule; + return options->TestingClientDownloadInitialDelay; } case DL_SCHED_CONSENSUS: if (!networkstatus_consensus_can_use_multiple_directories(options)) { /* A public relay */ - return options->TestingServerConsensusDownloadSchedule; + return options->TestingServerConsensusDownloadInitialDelay; } else { /* A client or bridge */ if (networkstatus_consensus_is_bootstrapping(time(NULL))) { /* During bootstrapping */ if (!networkstatus_consensus_can_use_extra_fallbacks(options)) { /* A bootstrapping client without extra fallback directories */ - return - options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + return options-> + ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay; } else if (dls->want_authority) { /* A bootstrapping client with extra fallback directories, but * connecting to an authority */ return - options->ClientBootstrapConsensusAuthorityDownloadSchedule; + options->ClientBootstrapConsensusAuthorityDownloadInitialDelay; } else { /* A bootstrapping client connecting to extra fallback directories */ return - options->ClientBootstrapConsensusFallbackDownloadSchedule; + options->ClientBootstrapConsensusFallbackDownloadInitialDelay; } } else { /* A client with a reasonably live consensus, with or without * certificates */ - return options->TestingClientConsensusDownloadSchedule; + return options->TestingClientConsensusDownloadInitialDelay; } } case DL_SCHED_BRIDGE: if (options->UseBridges && num_bridges_usable(0) > 0) { /* A bridge client that is sure that one or more of its bridges are * running can afford to wait longer to update bridge descriptors. */ - return options->TestingBridgeDownloadSchedule; + return options->TestingBridgeDownloadInitialDelay; } else { /* A bridge client which might have no running bridges, must try to * get bridge descriptors straight away. */ - return options->TestingBridgeBootstrapDownloadSchedule; + return options->TestingBridgeBootstrapDownloadInitialDelay; } default: tor_assert(0); } /* Impossible, but gcc will fail with -Werror without a `return`. */ - return NULL; -} - -/** Decide which minimum delay step we want to use based on - * descriptor type in <b>dls</b> and <b>options</b>. - * Helper function for download_status_schedule_get_delay(). */ -STATIC int -find_dl_min_delay(download_status_t *dls, const or_options_t *options) -{ - tor_assert(dls); - tor_assert(options); - - /* - * For now, just use the existing schedule config stuff and pick the - * first/last entries off to get min/max delay for backoff purposes - */ - const smartlist_t *schedule = find_dl_schedule(dls, options); - tor_assert(schedule != NULL && smartlist_len(schedule) >= 2); - return *(int *)(smartlist_get(schedule, 0)); + return 0; } /** As next_random_exponential_delay() below, but does not compute a random @@ -5636,10 +5586,9 @@ download_status_increment_attempt(download_status_t *dls, const char *item, static time_t download_status_get_initial_delay_from_now(const download_status_t *dls) { - const smartlist_t *schedule = find_dl_schedule(dls, get_options()); /* We use constant initial delays, even in exponential backoff * schedules. */ - return time(NULL) + *(int *)smartlist_get(schedule, 0); + return time(NULL) + find_dl_min_delay(dls, get_options()); } /** Reset <b>dls</b> so that it will be considered downloadable @@ -5966,4 +5915,3 @@ dir_split_resource_into_spoolable(const char *resource, smartlist_free(fingerprints); return r; } - diff --git a/src/or/directory.h b/src/or/directory.h index aa4d29a5bb..5f5ff7eca6 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -259,9 +259,7 @@ STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); STATIC compression_level_t choose_compression_level(ssize_t n_bytes); -STATIC const smartlist_t *find_dl_schedule(const download_status_t *dls, - const or_options_t *options); -STATIC int find_dl_min_delay(download_status_t *dls, +STATIC int find_dl_min_delay(const download_status_t *dls, const or_options_t *options); STATIC int next_random_exponential_delay(int delay, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index ced40899d0..4e09c1c65b 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -18,7 +18,6 @@ #include "control.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "hibernate.h" #include "keypin.h" #include "main.h" @@ -33,6 +32,9 @@ #include "routerparse.h" #include "routerset.h" #include "torcert.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" /** * \file dirserv.c @@ -86,7 +88,6 @@ static const signed_descriptor_t *get_signed_descriptor_by_fp( int extrainfo); static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); -static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri); static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri); static int spooled_resource_lookup_body(const spooled_resource_t *spooled, @@ -258,11 +259,12 @@ dirserv_load_fingerprint_file(void) * identity to stop doing so. This is going to be essential for good identity * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could * just sign fake descriptors missing the Ed25519 key. But we won't actually - * be able to prevent that kind of thing until we're confident that there - * isn't actually a legit reason to downgrade to 0.2.5. So for now, we have - * to leave this #undef. + * be able to prevent that kind of thing until we're confident that there isn't + * actually a legit reason to downgrade to 0.2.5. Now we are not recommending + * 0.2.5 anymore so there is no reason to keep the #undef. */ -#undef DISABLE_DISABLING_ED25519 + +#define DISABLE_DISABLING_ED25519 /** Check whether <b>router</b> has a nickname/identity key combination that * we recognize from the fingerprint list, or an IP we automatically act on @@ -857,13 +859,13 @@ directory_remove_invalid(void) SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) { const char *msg = NULL; + const char *description; routerinfo_t *ent = node->ri; - char description[NODE_DESC_BUF_LEN]; uint32_t r; if (!ent) continue; r = dirserv_router_get_status(ent, &msg, LOG_INFO); - router_get_description(description, ent); + description = router_describe(ent); if (r & FP_REJECT) { log_info(LD_DIRSERV, "Router %s is now rejected: %s", description, msg?msg:""); @@ -920,7 +922,7 @@ list_single_server_status(const routerinfo_t *desc, int is_live) } /* DOCDOC running_long_enough_to_decide_unreachable */ -static inline int +int running_long_enough_to_decide_unreachable(void) { return time_of_process_start @@ -953,7 +955,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) tor_assert(node); if (router_is_me(router)) { - /* We always know if we are down ourselves. */ + /* We always know if we are shutting down or hibernating ourselves. */ answer = ! we_are_hibernating(); } else if (router->is_hibernating && (router->cache_info.published_on + @@ -1055,59 +1057,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, return 0; } -/** Given a (possibly empty) list of config_line_t, each line of which contains - * a list of comma-separated version numbers surrounded by optional space, - * allocate and return a new string containing the version numbers, in order, - * separated by commas. Used to generate Recommended(Client|Server)?Versions - */ -char * -format_recommended_version_list(const config_line_t *ln, int warn) -{ - smartlist_t *versions; - char *result; - versions = smartlist_new(); - for ( ; ln; ln = ln->next) { - smartlist_split_string(versions, ln->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - } - - /* Handle the case where a dirauth operator has accidentally made some - * versions space-separated instead of comma-separated. */ - smartlist_t *more_versions = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(versions, char *, v) { - if (strchr(v, ' ')) { - if (warn) - log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. " - "(These are supposed to be comma-separated; I'll pretend you " - "used commas instead.)", escaped(v)); - SMARTLIST_DEL_CURRENT(versions, v); - smartlist_split_string(more_versions, v, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - tor_free(v); - } - } SMARTLIST_FOREACH_END(v); - smartlist_add_all(versions, more_versions); - smartlist_free(more_versions); - - /* Check to make sure everything looks like a version. */ - if (warn) { - SMARTLIST_FOREACH_BEGIN(versions, const char *, v) { - tor_version_t ver; - if (tor_version_parse(v, &ver) < 0) { - log_warn(LD_DIRSERV, "Recommended version %s does not look valid. " - " (I'll include it anyway, since you told me to.)", - escaped(v)); - } - } SMARTLIST_FOREACH_END(v); - } - - sort_version_list(versions, 1); - result = smartlist_join_strings(versions,",",0,NULL); - SMARTLIST_FOREACH(versions,char *,s,tor_free(s)); - smartlist_free(versions); - return result; -} - /** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid, * not hibernating, having observed bw greater 0, and not too old. Else * return 0. @@ -1453,7 +1402,7 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router, * tests aren't instant. If we haven't been running long enough, * trust the relay. */ - if (stats_n_seconds_working > + if (get_uptime() > get_options()->MinUptimeHidServDirectoryV2 * 1.1) uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now), real_uptime(router, now)); @@ -1497,6 +1446,24 @@ router_counts_toward_thresholds(const node_t *node, time_t now, (have_mbw || !require_mbw); } +/** Look through the routerlist, and using the measured bandwidth cache count + * how many measured bandwidths we know. This is used to decide whether we + * ever trust advertised bandwidths for purposes of assigning flags. */ +void +dirserv_count_measured_bws(const smartlist_t *routers) +{ + /* Initialize this first */ + routers_with_measured_bw = 0; + + /* Iterate over the routerlist and count measured bandwidths */ + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { + /* Check if we know a measured bandwidth for this one */ + if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { + ++routers_with_measured_bw; + } + } SMARTLIST_FOREACH_END(ri); +} + /** Look through the routerlist, the Mean Time Between Failure history, and * the Weighted Fractional Uptime history, and use them to set thresholds for * the Stable, Fast, and Guard flags. Update the fields stable_uptime, @@ -1504,7 +1471,7 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits. * * Also, set the is_exit flag of each router appropriately. */ -static void +void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) { int n_active, n_active_nonexit, n_familiar; @@ -1735,7 +1702,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, } /** Clear and free the measured bandwidth cache */ -STATIC void +void dirserv_clear_measured_bw_cache(void) { if (mbw_cache) { @@ -1767,18 +1734,10 @@ dirserv_expire_measured_bw_cache(time_t now) } } -/** Get the current size of the measured bandwidth cache */ -STATIC int -dirserv_get_measured_bw_cache_size(void) -{ - if (mbw_cache) return digestmap_size(mbw_cache); - else return 0; -} - /** Query the cache by identity digest, return value indicates whether * we found it. The bw_out and as_of_out pointers receive the cached * bandwidth value and the time it was cached if not NULL. */ -STATIC int +int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, time_t *as_of_out) { @@ -1799,61 +1758,18 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, } /** Predicate wrapper for dirserv_query_measured_bw_cache() */ -STATIC int +int dirserv_has_measured_bw(const char *node_id) { return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL); } -/** Get the best estimate of a router's bandwidth for dirauth purposes, - * preferring measured to advertised values if available. */ - -static uint32_t -dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri) -{ - uint32_t bw_kb = 0; - /* - * Yeah, measured bandwidths in measured_bw_line_t are (implicitly - * signed) longs and the ones router_get_advertised_bandwidth() returns - * are uint32_t. - */ - long mbw_kb = 0; - - if (ri) { - /* - * * First try to see if we have a measured bandwidth; don't bother with - * as_of_out here, on the theory that a stale measured bandwidth is still - * better to trust than an advertised one. - */ - if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest, - &mbw_kb, NULL)) { - /* Got one! */ - bw_kb = (uint32_t)mbw_kb; - } else { - /* If not, fall back to advertised */ - bw_kb = router_get_advertised_bandwidth(ri) / 1000; - } - } - - return bw_kb; -} - -/** Look through the routerlist, and using the measured bandwidth cache count - * how many measured bandwidths we know. This is used to decide whether we - * ever trust advertised bandwidths for purposes of assigning flags. */ -static void -dirserv_count_measured_bws(const smartlist_t *routers) +/** Get the current size of the measured bandwidth cache */ +int +dirserv_get_measured_bw_cache_size(void) { - /* Initialize this first */ - routers_with_measured_bw = 0; - - /* Iterate over the routerlist and count measured bandwidths */ - SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { - /* Check if we know a measured bandwidth for this one */ - if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { - ++routers_with_measured_bw; - } - } SMARTLIST_FOREACH_END(ri); + if (mbw_cache) return digestmap_size(mbw_cache); + else return 0; } /** Return the bandwidth we believe for assigning flags; prefer measured @@ -1916,26 +1832,6 @@ dirserv_get_flag_thresholds_line(void) return result; } -/** Given a platform string as in a routerinfo_t (possibly null), return a - * newly allocated version string for a networkstatus document, or NULL if the - * platform doesn't give a Tor version. */ -static char * -version_from_platform(const char *platform) -{ - if (platform && !strcmpstart(platform, "Tor ")) { - const char *eos = find_whitespace(platform+4); - if (eos && !strcmpstart(eos, " (r")) { - /* XXXX Unify this logic with the other version extraction - * logic in routerparse.c. */ - eos = find_whitespace(eos+1); - } - if (eos) { - return tor_strndup(platform, eos-platform); - } - } - return NULL; -} - /** Helper: write the router-status information in <b>rs</b> into a newly * allocated character buffer. Use the same format as in network-status * documents. If <b>version</b> is non-NULL, add a "v" line for the platform. @@ -2124,145 +2020,6 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, return result; } -/** Helper for sorting: compares two routerinfos first by address, and then by - * descending order of "usefulness". (An authority is more useful than a - * non-authority; a running router is more useful than a non-running router; - * and a router with more bandwidth is more useful than one with less.) - **/ -static int -compare_routerinfo_by_ip_and_bw_(const void **a, const void **b) -{ - routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; - int first_is_auth, second_is_auth; - uint32_t bw_kb_first, bw_kb_second; - const node_t *node_first, *node_second; - int first_is_running, second_is_running; - - /* we return -1 if first should appear before second... that is, - * if first is a better router. */ - if (first->addr < second->addr) - return -1; - else if (first->addr > second->addr) - return 1; - - /* Potentially, this next bit could cause k n lg n memeq calls. But in - * reality, we will almost never get here, since addresses will usually be - * different. */ - - first_is_auth = - router_digest_is_trusted_dir(first->cache_info.identity_digest); - second_is_auth = - router_digest_is_trusted_dir(second->cache_info.identity_digest); - - if (first_is_auth && !second_is_auth) - return -1; - else if (!first_is_auth && second_is_auth) - return 1; - - node_first = node_get_by_id(first->cache_info.identity_digest); - node_second = node_get_by_id(second->cache_info.identity_digest); - first_is_running = node_first && node_first->is_running; - second_is_running = node_second && node_second->is_running; - - if (first_is_running && !second_is_running) - return -1; - else if (!first_is_running && second_is_running) - return 1; - - bw_kb_first = dirserv_get_bandwidth_for_router_kb(first); - bw_kb_second = dirserv_get_bandwidth_for_router_kb(second); - - if (bw_kb_first > bw_kb_second) - return -1; - else if (bw_kb_first < bw_kb_second) - return 1; - - /* They're equal! Compare by identity digest, so there's a - * deterministic order and we avoid flapping. */ - return fast_memcmp(first->cache_info.identity_digest, - second->cache_info.identity_digest, - DIGEST_LEN); -} - -/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t - * whose keys are the identity digests of those routers that we're going to - * exclude for Sybil-like appearance. */ -static digestmap_t * -get_possible_sybil_list(const smartlist_t *routers) -{ - const or_options_t *options = get_options(); - digestmap_t *omit_as_sybil; - smartlist_t *routers_by_ip = smartlist_new(); - uint32_t last_addr; - int addr_count; - /* Allow at most this number of Tor servers on a single IP address, ... */ - int max_with_same_addr = options->AuthDirMaxServersPerAddr; - if (max_with_same_addr <= 0) - max_with_same_addr = INT_MAX; - - smartlist_add_all(routers_by_ip, routers); - smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_); - omit_as_sybil = digestmap_new(); - - last_addr = 0; - addr_count = 0; - SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) { - if (last_addr != ri->addr) { - last_addr = ri->addr; - addr_count = 1; - } else if (++addr_count > max_with_same_addr) { - digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri); - } - } SMARTLIST_FOREACH_END(ri); - - smartlist_free(routers_by_ip); - return omit_as_sybil; -} - -/** If there are entries in <b>routers</b> with exactly the same ed25519 keys, - * remove the older one. If they are exactly the same age, remove the one - * with the greater descriptor digest. May alter the order of the list. */ -static void -routers_make_ed_keys_unique(smartlist_t *routers) -{ - routerinfo_t *ri2; - digest256map_t *by_ed_key = digest256map_new(); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - ri->omit_from_vote = 0; - if (ri->cache_info.signing_key_cert == NULL) - continue; /* No ed key */ - const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey; - if ((ri2 = digest256map_get(by_ed_key, pk))) { - /* Duplicate; must omit one. Set the omit_from_vote flag in whichever - * one has the earlier published_on. */ - const time_t ri_pub = ri->cache_info.published_on; - const time_t ri2_pub = ri2->cache_info.published_on; - if (ri2_pub < ri_pub || - (ri2_pub == ri_pub && - fast_memcmp(ri->cache_info.signed_descriptor_digest, - ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { - digest256map_set(by_ed_key, pk, ri); - ri2->omit_from_vote = 1; - } else { - ri->omit_from_vote = 1; - } - } else { - /* Add to map */ - digest256map_set(by_ed_key, pk, ri); - } - } SMARTLIST_FOREACH_END(ri); - - digest256map_free(by_ed_key, NULL); - - /* Now remove every router where the omit_from_vote flag got set. */ - SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { - if (ri->omit_from_vote) { - SMARTLIST_DEL_CURRENT(routers, ri); - } - } SMARTLIST_FOREACH_END(ri); -} - /** Extract status information from <b>ri</b> and from other authority * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is * set. @@ -2375,25 +2132,6 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs) } } -/** Routerstatus <b>rs</b> is part of a group of routers that are on - * too narrow an IP-space. Clear out its flags since we don't want it be used - * because of its Sybil-like appearance. - * - * Leave its BadExit flag alone though, since if we think it's a bad exit, - * we want to vote that way in case all the other authorities are voting - * Running and Exit. - */ -static void -clear_status_flags_on_sybil(routerstatus_t *rs) -{ - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_named = rs->is_valid = - rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0; - /* FFFF we might want some mechanism to check later on if we - * missed zeroing any flags: it's easy to add a new flag but - * forget to add it to this clause. */ -} - /** The guardfraction of the guard with identity fingerprint <b>guard_id</b> * is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for * this guard in <b>vote_routerstatuses</b>, and if we do, register the @@ -2710,17 +2448,38 @@ dirserv_read_guardfraction_file(const char *fname, /** * Helper function to parse out a line in the measured bandwidth file - * into a measured_bw_line_t output structure. Returns -1 on failure - * or 0 on success. + * into a measured_bw_line_t output structure. + * + * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete + * bw line, return -1 and warn, since we are after the headers and we should + * only parse bw lines. Return 0 otherwise. + * + * If <b>line_is_after_headers</b> is false then it means that we are not past + * the header block yet. If we encounter an incomplete bw line, return -1 but + * don't warn since there could be additional header lines coming. If we + * encounter a proper bw line, return 0 (and we got past the headers). */ STATIC int -measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line, + int line_is_after_headers) { char *line = tor_strdup(orig_line); char *cp = line; int got_bw = 0; int got_node_id = 0; char *strtok_state; /* lame sauce d'jour */ + + if (strlen(line) == 0) { + log_warn(LD_DIRSERV, "Empty line in bandwidth file"); + tor_free(line); + return -1; + } + + /* Remove end of line character, so that is not part of the token */ + if (line[strlen(line) - 1] == '\n') { + line[strlen(line) - 1] = '\0'; + } + cp = tor_strtok_r(cp, " \t", &strtok_state); if (!cp) { @@ -2782,6 +2541,13 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) if (got_bw && got_node_id) { tor_free(line); return 0; + } else if (line_is_after_headers == 0) { + /* There could be additional header lines, therefore do not give warnings + * but returns -1 since it's not a complete bw line. */ + log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; } else { log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", escaped(orig_line)); @@ -2830,6 +2596,11 @@ dirserv_read_measured_bandwidths(const char *from_file, int applied_lines = 0; time_t file_time, now; int ok; + /* This flag will be 1 only when the first successful bw measurement line + * has been encountered, so that measured_bw_line_parse don't give warnings + * if there are additional header lines, as introduced in Bandwidth List spec + * version 1.1.0 */ + int line_is_after_headers = 0; /* Initialise line, so that we can't possibly run off the end. */ memset(line, 0, sizeof(line)); @@ -2877,7 +2648,11 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; if (fgets(line, sizeof(line), fp) && strlen(line)) { - if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_parse(&parsed_line, line, + line_is_after_headers) != -1) { + /* This condition will be true when the first complete valid bw line + * has been encountered, which means the end of the header lines. */ + line_is_after_headers = 1; /* Also cache the line for dirserv_get_bandwidth_for_router() */ dirserv_cache_measured_bw(&parsed_line, file_time); if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) @@ -2896,294 +2671,6 @@ dirserv_read_measured_bandwidths(const char *from_file, return 0; } -/** Return a new networkstatus_t* containing our current opinion. (For v3 - * authorities) */ -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, - authority_cert_t *cert) -{ - const or_options_t *options = get_options(); - networkstatus_t *v3_out = NULL; - uint32_t addr; - char *hostname = NULL, *client_versions = NULL, *server_versions = NULL; - const char *contact; - smartlist_t *routers, *routerstatuses; - char identity_digest[DIGEST_LEN]; - char signing_key_digest[DIGEST_LEN]; - int listbadexits = options->AuthDirListBadExits; - routerlist_t *rl = router_get_routerlist(); - time_t now = time(NULL); - time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - networkstatus_voter_info_t *voter = NULL; - vote_timing_t timing; - digestmap_t *omit_as_sybil = NULL; - const int vote_on_reachability = running_long_enough_to_decide_unreachable(); - smartlist_t *microdescriptors = NULL; - - tor_assert(private_key); - tor_assert(cert); - - if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { - log_err(LD_BUG, "Error computing signing key digest"); - return NULL; - } - if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) { - log_err(LD_BUG, "Error computing identity key digest"); - return NULL; - } - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - return NULL; - } - if (!hostname || !strchr(hostname, '.')) { - tor_free(hostname); - hostname = tor_dup_ip(addr); - } - - if (options->VersioningAuthoritativeDir) { - client_versions = - format_recommended_version_list(options->RecommendedClientVersions, 0); - server_versions = - format_recommended_version_list(options->RecommendedServerVersions, 0); - } - - contact = get_options()->ContactInfo; - if (!contact) - contact = "(none)"; - - /* - * Do this so dirserv_compute_performance_thresholds() and - * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. - */ - if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); - } else { - /* - * No bandwidths file; clear the measured bandwidth cache in case we had - * one last time around. - */ - if (dirserv_get_measured_bw_cache_size() > 0) { - dirserv_clear_measured_bw_cache(); - } - } - - /* precompute this part, since we need it to decide what "stable" - * means. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - dirserv_set_router_is_running(ri, now); - }); - - routers = smartlist_new(); - smartlist_add_all(routers, rl->routers); - routers_make_ed_keys_unique(routers); - /* After this point, don't use rl->routers; use 'routers' instead. */ - routers_sort_by_identity(routers); - omit_as_sybil = get_possible_sybil_list(routers); - - DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) { - (void) ignore; - rep_hist_make_router_pessimal(sybil_id, now); - } DIGESTMAP_FOREACH_END; - - /* Count how many have measured bandwidths so we know how to assign flags; - * this must come before dirserv_compute_performance_thresholds() */ - dirserv_count_measured_bws(routers); - - dirserv_compute_performance_thresholds(omit_as_sybil); - - routerstatuses = smartlist_new(); - microdescriptors = smartlist_new(); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - /* If it has a protover list and contains a protocol name greater than - * MAX_PROTOCOL_NAME_LENGTH, skip it. */ - if (ri->protocol_list && - protover_contains_long_protocol_names(ri->protocol_list)) { - continue; - } - if (ri->cache_info.published_on >= cutoff) { - routerstatus_t *rs; - vote_routerstatus_t *vrs; - node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); - if (!node) - continue; - - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - set_routerstatus_from_routerinfo(rs, node, ri, now, - listbadexits); - - if (ri->cache_info.signing_key_cert) { - memcpy(vrs->ed25519_id, - ri->cache_info.signing_key_cert->signing_key.pubkey, - ED25519_PUBKEY_LEN); - } - - if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) - clear_status_flags_on_sybil(rs); - - if (!vote_on_reachability) - rs->is_flagged_running = 0; - - vrs->version = version_from_platform(ri->platform); - if (ri->protocol_list) { - vrs->protocols = tor_strdup(ri->protocol_list); - } else { - vrs->protocols = tor_strdup( - protover_compute_for_old_tor(vrs->version)); - } - vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now, - microdescriptors); - - smartlist_add(routerstatuses, vrs); - } - } SMARTLIST_FOREACH_END(ri); - - { - smartlist_t *added = - microdescs_add_list_to_cache(get_microdesc_cache(), - microdescriptors, SAVED_NOWHERE, 0); - smartlist_free(added); - smartlist_free(microdescriptors); - } - - smartlist_free(routers); - digestmap_free(omit_as_sybil, NULL); - - /* Apply guardfraction information to routerstatuses. */ - if (options->GuardfractionFile) { - dirserv_read_guardfraction_file(options->GuardfractionFile, - routerstatuses); - } - - /* This pass through applies the measured bw lines to the routerstatuses */ - if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, - routerstatuses); - } else { - /* - * No bandwidths file; clear the measured bandwidth cache in case we had - * one last time around. - */ - if (dirserv_get_measured_bw_cache_size() > 0) { - dirserv_clear_measured_bw_cache(); - } - } - - v3_out = tor_malloc_zero(sizeof(networkstatus_t)); - - v3_out->type = NS_TYPE_VOTE; - dirvote_get_preferred_voting_intervals(&timing); - v3_out->published = now; - { - char tbuf[ISO_TIME_LEN+1]; - networkstatus_t *current_consensus = - networkstatus_get_live_consensus(now); - long last_consensus_interval; /* only used to pick a valid_after */ - if (current_consensus) - last_consensus_interval = current_consensus->fresh_until - - current_consensus->valid_after; - else - last_consensus_interval = options->TestingV3AuthInitialVotingInterval; - v3_out->valid_after = - dirvote_get_start_of_next_interval(now, (int)last_consensus_interval, - options->TestingV3AuthVotingStartOffset); - format_iso_time(tbuf, v3_out->valid_after); - log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " - "consensus_set=%d, last_interval=%d", - tbuf, current_consensus?1:0, (int)last_consensus_interval); - } - v3_out->fresh_until = v3_out->valid_after + timing.vote_interval; - v3_out->valid_until = v3_out->valid_after + - (timing.vote_interval * timing.n_intervals_valid); - v3_out->vote_seconds = timing.vote_delay; - v3_out->dist_seconds = timing.dist_delay; - tor_assert(v3_out->vote_seconds > 0); - tor_assert(v3_out->dist_seconds > 0); - tor_assert(timing.n_intervals_valid > 0); - - v3_out->client_versions = client_versions; - v3_out->server_versions = server_versions; - - /* These are hardwired, to avoid disaster. */ - v3_out->recommended_relay_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 Microdesc=1-2 Relay=2"); - v3_out->recommended_client_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 Microdesc=1-2 Relay=2"); - v3_out->required_client_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 Microdesc=1-2 Relay=2"); - v3_out->required_relay_protocols = - tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=3-4 Microdesc=1 Relay=1-2"); - - /* We are not allowed to vote to require anything we don't have. */ - tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL)); - tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL)); - - /* We should not recommend anything we don't have. */ - tor_assert_nonfatal(protover_all_supported( - v3_out->recommended_relay_protocols, NULL)); - tor_assert_nonfatal(protover_all_supported( - v3_out->recommended_client_protocols, NULL)); - - v3_out->package_lines = smartlist_new(); - { - config_line_t *cl; - for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { - if (validate_recommended_package_line(cl->value)) - smartlist_add_strdup(v3_out->package_lines, cl->value); - } - } - - v3_out->known_flags = smartlist_new(); - smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (vote_on_reachability) - smartlist_add_strdup(v3_out->known_flags, "Running"); - if (listbadexits) - smartlist_add_strdup(v3_out->known_flags, "BadExit"); - smartlist_sort_strings(v3_out->known_flags); - - if (options->ConsensusParams) { - v3_out->net_params = smartlist_new(); - smartlist_split_string(v3_out->net_params, - options->ConsensusParams, NULL, 0, 0); - smartlist_sort_strings(v3_out->net_params); - } - - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup(options->Nickname); - memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); - voter->sigs = smartlist_new(); - voter->address = hostname; - voter->addr = addr; - voter->dir_port = router_get_advertised_dir_port(options, 0); - voter->or_port = router_get_advertised_or_port(options); - voter->contact = tor_strdup(contact); - if (options->V3AuthUseLegacyKey) { - authority_cert_t *c = get_my_v3_legacy_cert(); - if (c) { - if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { - log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); - memset(voter->legacy_id_digest, 0, DIGEST_LEN); - } - } - } - - v3_out->voters = smartlist_new(); - smartlist_add(v3_out->voters, voter); - v3_out->cert = authority_cert_dup(cert); - v3_out->routerstatus_list = routerstatuses; - /* Note: networkstatus_digest is unset; it won't get set until we actually - * format the vote. */ - - return v3_out; -} - /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t * pointers, adds copies of digests to fps_out, and doesn't use the * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other diff --git a/src/or/dirserv.h b/src/or/dirserv.h index cb7c628387..b243439cc2 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -157,6 +157,15 @@ void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); char *format_recommended_version_list(const config_line_t *line, int warn); int validate_recommended_package_line(const char *line); +int dirserv_query_measured_bw_cache_kb(const char *node_id, + long *bw_out, + time_t *as_of_out); +void dirserv_clear_measured_bw_cache(void); +int dirserv_has_measured_bw(const char *node_id); +int dirserv_get_measured_bw_cache_size(void); +void dirserv_count_measured_bws(const smartlist_t *routers); +int running_long_enough_to_decide_unreachable(void); +void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef DIRSERV_PRIVATE @@ -165,20 +174,15 @@ STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */ #define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ -STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line); +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line, + int line_is_after_headers); STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses); STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of); -STATIC void dirserv_clear_measured_bw_cache(void); STATIC void dirserv_expire_measured_bw_cache(time_t now); -STATIC int dirserv_get_measured_bw_cache_size(void); -STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, - long *bw_out, - time_t *as_of_out); -STATIC int dirserv_has_measured_bw(const char *node_id); STATIC int dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, diff --git a/src/or/dns.c b/src/or/dns.c index 411e2d5aa6..ba734ed900 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -56,6 +56,7 @@ #include "connection.h" #include "connection_edge.h" #include "control.h" +#include "crypto_rand.h" #include "dns.h" #include "main.h" #include "policies.h" diff --git a/src/or/dos.c b/src/or/dos.c index 2cb3470582..ee731accea 100644 --- a/src/or/dos.c +++ b/src/or/dos.c @@ -11,6 +11,7 @@ #include "or.h" #include "channel.h" #include "config.h" +#include "crypto_rand.h" #include "geoip.h" #include "main.h" #include "networkstatus.h" diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 54638810fa..27d760f1a8 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -118,11 +118,13 @@ #include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" +#include "circuituse.h" #include "circuitstats.h" #include "config.h" #include "confparse.h" #include "connection.h" #include "control.h" +#include "crypto_rand.h" #include "directory.h" #include "entrynodes.h" #include "main.h" @@ -432,14 +434,15 @@ get_guard_confirmed_min_lifetime(void) STATIC int get_n_primary_guards(void) { - const int n = get_options()->NumEntryGuards; - const int n_dir = get_options()->NumDirectoryGuards; - if (n > 5) { - return MAX(n_dir, n + n / 2); - } else if (n >= 1) { - return MAX(n_dir, n * 2); + /* If the user has explicitly configured the number of primary guards, do + * what the user wishes to do */ + const int configured_primaries = get_options()->NumPrimaryGuards; + if (configured_primaries) { + return configured_primaries; } + /* otherwise check for consensus parameter and if that's not set either, just + * use the default value. */ return networkstatus_get_param(NULL, "guard-n-primary-guards", DFLT_N_PRIMARY_GUARDS, 1, INT32_MAX); @@ -454,6 +457,9 @@ get_n_primary_guards_to_use(guard_usage_t usage) int configured; const char *param_name; int param_default; + + /* If the user has explicitly configured the amount of guards, use + that. Otherwise, fall back to the default value. */ if (usage == GUARD_USAGE_DIRGUARD) { configured = get_options()->NumDirectoryGuards; param_name = "guard-n-primary-dir-guards-to-use"; @@ -2335,7 +2341,7 @@ entry_guard_cancel(circuit_guard_state_t **guard_state_p) } /** - * Called by the circuit building module when a circuit has succeeded: + * Called by the circuit building module when a circuit has failed: * informs the guards code that the guard in *<b>guard_state_p</b> is * not working, and advances the state of the guard module. */ @@ -3474,12 +3480,18 @@ guards_update_all(void) used. */ const node_t * guards_choose_guard(cpath_build_state_t *state, - circuit_guard_state_t **guard_state_out) + uint8_t purpose, + circuit_guard_state_t **guard_state_out) { const node_t *r = NULL; const uint8_t *exit_id = NULL; entry_guard_restriction_t *rst = NULL; - if (state && (exit_id = build_state_get_exit_rsa_id(state))) { + + /* Only apply restrictions if we have a specific exit node in mind, and only + * if we are not doing vanguard circuits: we don't want to apply guard + * restrictions to vanguard circuits. */ + if (state && !circuit_should_use_vanguards(purpose) && + (exit_id = build_state_get_exit_rsa_id(state))) { /* We're building to a targeted exit node, so that node can't be * chosen as our guard for this circuit. Remember that fact in a * restriction. */ diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index d562498313..e8c91da41b 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -322,6 +322,7 @@ struct circuit_guard_state_t { /* Common entry points for old and new guard code */ int guards_update_all(void); const node_t *guards_choose_guard(cpath_build_state_t *state, + uint8_t purpose, circuit_guard_state_t **guard_state_out); const node_t *guards_choose_dirguard(uint8_t dir_purpose, circuit_guard_state_t **guard_state_out); diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index 16a250fa58..b842442caf 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -20,9 +20,11 @@ #include "or.h" #include "connection.h" #include "connection_or.h" -#include "ext_orport.h" #include "control.h" #include "config.h" +#include "crypto_rand.h" +#include "crypto_util.h" +#include "ext_orport.h" #include "main.h" #include "proto_ext_or.h" #include "util.h" diff --git a/src/or/geoip.c b/src/or/geoip.c index 0ff1c6ce0d..d59043a7f6 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -150,7 +150,7 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, idx = ((uintptr_t)idxplus1_)-1; } { - geoip_country_t *c = smartlist_get(geoip_countries, idx); + geoip_country_t *c = smartlist_get(geoip_countries, (int)idx); tor_assert(!strcasecmp(c->countrycode, country)); } @@ -628,8 +628,7 @@ geoip_note_client_seen(geoip_client_action_t action, /* Only remember statistics if the DoS mitigation subsystem is enabled. If * not, only if as entry guard or as bridge. */ if (!dos_enabled()) { - if (!options->EntryStatistics && - (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) { + if (!options->EntryStatistics && !should_record_bridge_info(options)) { return; } } @@ -1881,5 +1880,8 @@ geoip_free_all(void) clear_geoip_db(); tor_free(bridge_stats_extrainfo); + + memset(geoip_digest, 0, sizeof(geoip_digest)); + memset(geoip6_digest, 0, sizeof(geoip6_digest)); } diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 4dc35f68d0..d7d259470f 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -36,6 +36,7 @@ hibernating, phase 2: #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" #include "hibernate.h" #include "main.h" #include "router.h" @@ -51,6 +52,10 @@ static time_t hibernate_end_time = 0; * we aren't shutting down. */ static time_t shutdown_time = 0; +/** A timed event that we'll use when it's time to wake up from + * hibernation. */ +static mainloop_event_t *wakeup_event = NULL; + /** Possible accounting periods. */ typedef enum { UNIT_MONTH=1, UNIT_WEEK=2, UNIT_DAY=3, @@ -130,6 +135,8 @@ static time_t start_of_accounting_period_after(time_t now); static time_t start_of_accounting_period_containing(time_t now); static void accounting_set_wakeup_time(void); static void on_hibernate_state_change(hibernate_state_t prev_state); +static void hibernate_schedule_wakeup_event(time_t now, time_t end_time); +static void wakeup_event_callback(mainloop_event_t *ev, void *data); /** * Return the human-readable name for the hibernation state <b>state</b> @@ -297,7 +304,7 @@ accounting_get_end_time,(void)) return interval_end_time; } -/** Called from main.c to tell us that <b>seconds</b> seconds have +/** Called from connection.c to tell us that <b>seconds</b> seconds have * passed, <b>n_read</b> bytes have been read, and <b>n_written</b> * bytes have been written. */ void @@ -866,7 +873,7 @@ hibernate_end(hibernate_state_t new_state) hibernate_state = new_state; hibernate_end_time = 0; /* no longer hibernating */ - stats_n_seconds_working = 0; /* reset published uptime */ + reset_uptime(); /* reset published uptime */ } /** A wrapper around hibernate_begin, for when we get SIGINT. */ @@ -876,13 +883,26 @@ hibernate_begin_shutdown(void) hibernate_begin(HIBERNATE_STATE_EXITING, time(NULL)); } -/** Return true iff we are currently hibernating. */ +/** + * Return true iff we are currently hibernating -- that is, if we are in + * any non-live state. + */ MOCK_IMPL(int, we_are_hibernating,(void)) { return hibernate_state != HIBERNATE_STATE_LIVE; } +/** + * Return true iff we are currently _fully_ hibernating -- that is, if we are + * in a state where we expect to handle no network activity at all. + */ +MOCK_IMPL(int, +we_are_fully_hibernating,(void)) +{ + return hibernate_state == HIBERNATE_STATE_DORMANT; +} + /** If we aren't currently dormant, close all connections and become * dormant. */ static void @@ -935,6 +955,63 @@ hibernate_go_dormant(time_t now) or_state_mark_dirty(get_or_state(), get_options()->AvoidDiskWrites ? now+600 : 0); + + hibernate_schedule_wakeup_event(now, hibernate_end_time); +} + +/** + * Schedule a mainloop event at <b>end_time</b> to wake up from a dormant + * state. We can't rely on this happening from second_elapsed_callback, + * since second_elapsed_callback will be shut down when we're dormant. + * + * (Note that We might immediately go back to sleep after we set the next + * wakeup time.) + */ +static void +hibernate_schedule_wakeup_event(time_t now, time_t end_time) +{ + struct timeval delay = { 0, 0 }; + + if (now >= end_time) { + // In these cases we always wait at least a second, to avoid running + // the callback in a tight loop. + delay.tv_sec = 1; + } else { + delay.tv_sec = (end_time - now); + } + + if (!wakeup_event) { + wakeup_event = mainloop_event_postloop_new(wakeup_event_callback, NULL); + } + + mainloop_event_schedule(wakeup_event, &delay); +} + +/** + * Called at the end of the interval, or at the wakeup time of the current + * interval, to exit the dormant state. + **/ +static void +wakeup_event_callback(mainloop_event_t *ev, void *data) +{ + (void) ev; + (void) data; + + const time_t now = time(NULL); + accounting_run_housekeeping(now); + consider_hibernation(now); + if (hibernate_state != HIBERNATE_STATE_DORMANT) { + /* We woke up, so everything's great here */ + return; + } + + /* We're still dormant. */ + if (now < interval_wakeup_time) + hibernate_end_time = interval_wakeup_time; + else + hibernate_end_time = interval_end_time; + + hibernate_schedule_wakeup_event(now, hibernate_end_time); } /** Called when hibernate_end_time has arrived. */ @@ -1111,10 +1188,30 @@ getinfo_helper_accounting(control_connection_t *conn, static void on_hibernate_state_change(hibernate_state_t prev_state) { - (void)prev_state; /* Should we do something with this? */ control_event_server_status(LOG_NOTICE, "HIBERNATION_STATUS STATUS=%s", hibernate_state_to_string(hibernate_state)); + + /* We are changing hibernation state, this can affect the main loop event + * list. Rescan it to update the events state. We do this whatever the new + * hibernation state because they can each possibly affect an event. The + * initial state means we are booting up so we shouldn't scan here because + * at this point the events in the list haven't been initialized. */ + if (prev_state != HIBERNATE_STATE_INITIAL) { + rescan_periodic_events(get_options()); + } + + reschedule_per_second_timer(); +} + +/** Free all resources held by the accounting module */ +void +accounting_free_all(void) +{ + mainloop_event_free(wakeup_event); + hibernate_state = HIBERNATE_STATE_INITIAL; + hibernate_end_time = 0; + shutdown_time = 0; } #ifdef TOR_UNIT_TESTS diff --git a/src/or/hibernate.h b/src/or/hibernate.h index 85fb42864b..453969d052 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -25,11 +25,13 @@ void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); int accounting_record_bandwidth_usage(time_t now, or_state_t *state); void hibernate_begin_shutdown(void); MOCK_DECL(int, we_are_hibernating, (void)); +MOCK_DECL(int, we_are_fully_hibernating,(void)); void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, const char **errmsg); uint64_t get_accounting_max_total(void); +void accounting_free_all(void); #ifdef HIBERNATE_PRIVATE /** Possible values of hibernate_state */ diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c index df53efd32d..ecc845d17f 100644 --- a/src/or/hs_cache.c +++ b/src/or/hs_cache.c @@ -11,6 +11,7 @@ #include "or.h" #include "config.h" +#include "crypto_util.h" #include "hs_ident.h" #include "hs_common.h" #include "hs_client.h" diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h index a141634cc4..0d0085ffdc 100644 --- a/src/or/hs_cache.h +++ b/src/or/hs_cache.h @@ -11,7 +11,6 @@ #include <stdint.h> -#include "crypto.h" #include "crypto_ed25519.h" #include "hs_common.h" #include "hs_descriptor.h" diff --git a/src/or/hs_cell.c b/src/or/hs_cell.c index 5244cfa3dd..03273a44f9 100644 --- a/src/or/hs_cell.c +++ b/src/or/hs_cell.c @@ -8,6 +8,7 @@ #include "or.h" #include "config.h" +#include "crypto_util.h" #include "rendservice.h" #include "replaycache.h" #include "util.h" @@ -369,7 +370,7 @@ introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell, crypto_cipher_free(cipher); offset += encoded_enc_cell_len; /* Compute MAC from the above and put it in the buffer. This function will - * make the adjustment to the encryptled_len to ommit the MAC length. */ + * make the adjustment to the encrypted_len to omit the MAC length. */ compute_introduce_mac(encoded_cell, encoded_cell_len, encrypted, encrypted_len, keys.mac_key, sizeof(keys.mac_key), diff --git a/src/or/hs_circuit.c b/src/or/hs_circuit.c index 0a9999a190..d911f5fde4 100644 --- a/src/or/hs_circuit.c +++ b/src/or/hs_circuit.c @@ -13,6 +13,8 @@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "nodelist.h" #include "policies.h" #include "relay.h" @@ -193,11 +195,8 @@ register_intro_circ(const hs_service_intro_point_t *ip, tor_assert(circ); if (ip->base.is_only_legacy) { - uint8_t digest[DIGEST_LEN]; - if (BUG(crypto_pk_get_digest(ip->legacy_key, (char *) digest) < 0)) { - return; - } - hs_circuitmap_register_intro_circ_v2_service_side(circ, digest); + hs_circuitmap_register_intro_circ_v2_service_side(circ, + ip->legacy_key_digest); } else { hs_circuitmap_register_intro_circ_v3_service_side(circ, &ip->auth_key_kp.pubkey); @@ -683,22 +682,14 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip, origin_circuit_t * hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip) { - origin_circuit_t *circ = NULL; - tor_assert(ip); if (ip->base.is_only_legacy) { - uint8_t digest[DIGEST_LEN]; - if (BUG(crypto_pk_get_digest(ip->legacy_key, (char *) digest) < 0)) { - goto end; - } - circ = hs_circuitmap_get_intro_circ_v2_service_side(digest); + return hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest); } else { - circ = hs_circuitmap_get_intro_circ_v3_service_side( + return hs_circuitmap_get_intro_circ_v3_service_side( &ip->auth_key_kp.pubkey); } - end: - return circ; } /* Called when we fail building a rendezvous circuit at some point other than diff --git a/src/or/hs_circuit.h b/src/or/hs_circuit.h index 2f5beaa168..f69137e1d5 100644 --- a/src/or/hs_circuit.h +++ b/src/or/hs_circuit.h @@ -10,7 +10,6 @@ #define TOR_HS_CIRCUIT_H #include "or.h" -#include "crypto.h" #include "crypto_ed25519.h" #include "hs_service.h" diff --git a/src/or/hs_client.c b/src/or/hs_client.c index 77cbc11a89..8ecefc1209 100644 --- a/src/or/hs_client.c +++ b/src/or/hs_client.c @@ -9,31 +9,31 @@ #define HS_CLIENT_PRIVATE #include "or.h" -#include "hs_circuit.h" -#include "hs_ident.h" +#include "circpathbias.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" #include "connection_edge.h" #include "container.h" -#include "rendclient.h" -#include "hs_descriptor.h" +#include "crypto_rand.h" +#include "crypto_util.h" +#include "directory.h" #include "hs_cache.h" #include "hs_cell.h" -#include "hs_ident.h" -#include "config.h" -#include "directory.h" +#include "hs_circuit.h" #include "hs_client.h" #include "hs_control.h" -#include "router.h" -#include "routerset.h" -#include "circuitlist.h" -#include "circuituse.h" -#include "connection.h" -#include "nodelist.h" -#include "circpathbias.h" -#include "connection.h" +#include "hs_descriptor.h" +#include "hs_ident.h" #include "hs_ntor.h" -#include "circuitbuild.h" #include "networkstatus.h" +#include "nodelist.h" #include "reasons.h" +#include "rendclient.h" +#include "router.h" +#include "routerset.h" /* Return a human-readable string for the client fetch status code. */ static const char * @@ -1435,8 +1435,8 @@ hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident) * connection is considered "fresh" and can continue without being closed * too early. */ base_conn->timestamp_created = now; - base_conn->timestamp_lastread = now; - base_conn->timestamp_lastwritten = now; + base_conn->timestamp_last_read_allowed = now; + base_conn->timestamp_last_write_allowed = now; /* Change connection's state into waiting for a circuit. */ base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; diff --git a/src/or/hs_common.c b/src/or/hs_common.c index e586516c80..c42ef0c97b 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -15,6 +15,8 @@ #include "config.h" #include "circuitbuild.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "networkstatus.h" #include "nodelist.h" #include "hs_cache.h" @@ -28,9 +30,8 @@ #include "rendservice.h" #include "routerset.h" #include "router.h" -#include "routerset.h" -#include "shared_random.h" -#include "shared_random_state.h" +#include "shared_random_client.h" +#include "dirauth/shared_random_state.h" /* Trunnel */ #include "ed25519_cert.h" @@ -104,7 +105,7 @@ compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member) { const char *key = _key; const node_t *node = *_member; - return tor_memcmp(key, node->hsdir_index->fetch, DIGEST256_LEN); + return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN); } /* Helper function: The key is a digest that we compare to a node_t object @@ -115,7 +116,7 @@ compare_digest_to_store_first_hsdir_index(const void *_key, { const char *key = _key; const node_t *node = *_member; - return tor_memcmp(key, node->hsdir_index->store_first, DIGEST256_LEN); + return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN); } /* Helper function: The key is a digest that we compare to a node_t object @@ -126,7 +127,7 @@ compare_digest_to_store_second_hsdir_index(const void *_key, { const char *key = _key; const node_t *node = *_member; - return tor_memcmp(key, node->hsdir_index->store_second, DIGEST256_LEN); + return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN); } /* Helper function: Compare two node_t objects current hsdir_index. */ @@ -135,8 +136,8 @@ compare_node_fetch_hsdir_index(const void **a, const void **b) { const node_t *node1= *a; const node_t *node2 = *b; - return tor_memcmp(node1->hsdir_index->fetch, - node2->hsdir_index->fetch, + return tor_memcmp(node1->hsdir_index.fetch, + node2->hsdir_index.fetch, DIGEST256_LEN); } @@ -146,8 +147,8 @@ compare_node_store_first_hsdir_index(const void **a, const void **b) { const node_t *node1= *a; const node_t *node2 = *b; - return tor_memcmp(node1->hsdir_index->store_first, - node2->hsdir_index->store_first, + return tor_memcmp(node1->hsdir_index.store_first, + node2->hsdir_index.store_first, DIGEST256_LEN); } @@ -157,8 +158,8 @@ compare_node_store_second_hsdir_index(const void **a, const void **b) { const node_t *node1= *a; const node_t *node2 = *b; - return tor_memcmp(node1->hsdir_index->store_second, - node2->hsdir_index->store_second, + return tor_memcmp(node1->hsdir_index.store_second, + node2->hsdir_index.store_second, DIGEST256_LEN); } @@ -1289,18 +1290,15 @@ node_has_hsdir_index(const node_t *node) /* At this point, since the node has a desc, this node must also have an * hsdir index. If not, something went wrong, so BUG out. */ - if (BUG(node->hsdir_index == NULL)) { - return 0; - } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index->fetch, + if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch, DIGEST256_LEN))) { return 0; } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index->store_first, + if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first, DIGEST256_LEN))) { return 0; } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index->store_second, + if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second, DIGEST256_LEN))) { return 0; } @@ -1334,15 +1332,20 @@ hs_get_responsible_hsdirs(const ed25519_public_key_t *blinded_pk, sorted_nodes = smartlist_new(); + /* Make sure we actually have a live consensus */ + networkstatus_t *c = networkstatus_get_live_consensus(approx_time()); + if (!c || smartlist_len(c->routerstatus_list) == 0) { + log_warn(LD_REND, "No live consensus so we can't get the responsible " + "hidden service directories."); + goto done; + } + + /* Ensure the nodelist is fresh, since it contains the HSDir indices. */ + nodelist_ensure_freshness(c); + /* Add every node_t that support HSDir v3 for which we do have a valid * hsdir_index already computed for them for this consensus. */ { - networkstatus_t *c = networkstatus_get_latest_consensus(); - if (!c || smartlist_len(c->routerstatus_list) == 0) { - log_warn(LD_REND, "No valid consensus so we can't get the responsible " - "hidden service directories."); - goto done; - } SMARTLIST_FOREACH_BEGIN(c->routerstatus_list, const routerstatus_t *, rs) { /* Even though this node_t object won't be modified and should be const, * we can't add const object in a smartlist_t. */ diff --git a/src/or/hs_common.h b/src/or/hs_common.h index 83ba1b8599..ef7d5dca2b 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -156,19 +156,6 @@ typedef struct rend_service_port_config_t { char unix_addr[FLEXIBLE_ARRAY_MEMBER]; } rend_service_port_config_t; -/* Hidden service directory index used in a node_t which is set once we set - * the consensus. */ -typedef struct hsdir_index_t { - /* HSDir index to use when fetching a descriptor. */ - uint8_t fetch[DIGEST256_LEN]; - - /* HSDir index used by services to store their first and second - * descriptor. The first descriptor is chronologically older than the second - * one and uses older TP and SRV values. */ - uint8_t store_first[DIGEST256_LEN]; - uint8_t store_second[DIGEST256_LEN]; -} hsdir_index_t; - void hs_init(void); void hs_free_all(void); diff --git a/src/or/hs_control.c b/src/or/hs_control.c index 87b4e3fca8..6b9b95c6d8 100644 --- a/src/or/hs_control.c +++ b/src/or/hs_control.c @@ -8,6 +8,7 @@ #include "or.h" #include "control.h" +#include "crypto_util.h" #include "hs_common.h" #include "hs_control.h" #include "hs_descriptor.h" @@ -39,9 +40,8 @@ hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk, * can't pick a node without an hsdir_index. */ hsdir_node = node_get_by_id(hsdir_rs->identity_digest); tor_assert(hsdir_node); - tor_assert(hsdir_node->hsdir_index); /* This is a fetch event. */ - hsdir_index = hsdir_node->hsdir_index->fetch; + hsdir_index = hsdir_node->hsdir_index.fetch; /* Trigger the event. */ control_event_hs_descriptor_requested(onion_address, REND_NO_AUTH, diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c index 32278ec9f2..e85c29a24e 100644 --- a/src/or/hs_descriptor.c +++ b/src/or/hs_descriptor.c @@ -59,6 +59,8 @@ #include "ed25519_cert.h" /* Trunnel interface. */ #include "hs_descriptor.h" #include "circuitbuild.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "parsecommon.h" #include "rendcache.h" #include "hs_cache.h" diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h index 09979410e1..8195c6efbc 100644 --- a/src/or/hs_descriptor.h +++ b/src/or/hs_descriptor.h @@ -16,6 +16,7 @@ #include "container.h" #include "crypto.h" #include "crypto_ed25519.h" +#include "ed25519_cert.h" /* needed for trunnel */ #include "torcert.h" /* Trunnel */ diff --git a/src/or/hs_ident.c b/src/or/hs_ident.c index 0bce2f625b..3603e329d4 100644 --- a/src/or/hs_ident.c +++ b/src/or/hs_ident.c @@ -7,6 +7,7 @@ * subsytem. **/ +#include "crypto_util.h" #include "hs_ident.h" /* Return a newly allocated circuit identifier. The given public key is copied diff --git a/src/or/hs_ident.h b/src/or/hs_ident.h index 91ec389aa4..8f9da30c35 100644 --- a/src/or/hs_ident.h +++ b/src/or/hs_ident.h @@ -21,7 +21,6 @@ #ifndef TOR_HS_IDENT_H #define TOR_HS_IDENT_H -#include "crypto.h" #include "crypto_ed25519.h" #include "hs_common.h" diff --git a/src/or/hs_intropoint.c b/src/or/hs_intropoint.c index 9eaf572510..ee79109ca9 100644 --- a/src/or/hs_intropoint.c +++ b/src/or/hs_intropoint.c @@ -12,7 +12,6 @@ #include "config.h" #include "circuitlist.h" #include "circuituse.h" -#include "config.h" #include "relay.h" #include "rendmid.h" #include "rephist.h" diff --git a/src/or/hs_ntor.c b/src/or/hs_ntor.c index a416bc46c3..809fa83bb8 100644 --- a/src/or/hs_ntor.c +++ b/src/or/hs_ntor.c @@ -25,6 +25,7 @@ */ #include "or.h" +#include "crypto_util.h" #include "hs_ntor.h" /* String constants used by the ntor HS protocol */ diff --git a/src/or/hs_service.c b/src/or/hs_service.c index 3273705ae9..54ceb9dcad 100644 --- a/src/or/hs_service.c +++ b/src/or/hs_service.c @@ -15,6 +15,8 @@ #include "circuituse.h" #include "config.h" #include "connection.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "main.h" #include "networkstatus.h" @@ -24,14 +26,13 @@ #include "router.h" #include "routerkeys.h" #include "routerlist.h" -#include "shared_random_state.h" +#include "shared_random_client.h" #include "statefile.h" #include "hs_circuit.h" #include "hs_common.h" #include "hs_config.h" #include "hs_control.h" -#include "hs_circuit.h" #include "hs_descriptor.h" #include "hs_ident.h" #include "hs_intropoint.h" @@ -81,6 +82,7 @@ static smartlist_t *hs_service_staging_list; * reupload if needed */ static int consider_republishing_hs_descriptors = 0; +/* Static declaration. */ static void set_descriptor_revision_counter(hs_descriptor_t *hs_desc); static void move_descriptors(hs_service_t *src, hs_service_t *dst); @@ -153,6 +155,12 @@ register_service(hs_service_ht *map, hs_service_t *service) } /* Taking ownership of the object at this point. */ HT_INSERT(hs_service_ht, map, service); + + /* If we just modified the global map, we notify. */ + if (map == hs_service_map) { + hs_service_map_has_changed(); + } + return 0; } @@ -179,6 +187,11 @@ remove_service(hs_service_ht *map, hs_service_t *service) "while removing service %s", escaped(service->config.directory_path)); } + + /* If we just modified the global map, we notify. */ + if (map == hs_service_map) { + hs_service_map_has_changed(); + } } /* Set the default values for a service configuration object <b>c</b>. */ @@ -434,6 +447,10 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy, if (crypto_pk_generate_key(ip->legacy_key) < 0) { goto err; } + if (crypto_pk_get_digest(ip->legacy_key, + (char *) ip->legacy_key_digest) < 0) { + goto err; + } } if (ei == NULL) { @@ -920,6 +937,11 @@ register_all_services(void) smartlist_clear(hs_service_staging_list); service_free_all(); hs_service_map = new_service_map; + /* We've just register services into the new map and now we've replaced the + * global map with it so we have to notify that the change happened. When + * registering a service, the notify is only triggered if the destination + * map is the global map for which in here it was not. */ + hs_service_map_has_changed(); } /* Write the onion address of a given service to the given filename fname_ in @@ -2295,8 +2317,8 @@ upload_descriptor_to_hsdir(const hs_service_t *service, /* Logging so we know where it was sent. */ { int is_next_desc = (service->desc_next == desc); - const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index->store_second: - hsdir->hsdir_index->store_first; + const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index.store_second: + hsdir->hsdir_index.store_first; log_info(LD_REND, "Service %s %s descriptor of revision %" PRIu64 " initiated upload request to %s with index %s", safe_str_client(service->onion_address), @@ -2944,6 +2966,17 @@ service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list) /* Public API */ /* ========== */ +/* This is called everytime the service map (v2 or v3) changes that is if an + * element is added or removed. */ +void +hs_service_map_has_changed(void) +{ + /* If we now have services where previously we had not, we need to enable + * the HS service main loop event. If we changed to having no services, we + * need to disable the event. */ + rescan_periodic_events(get_options()); +} + /* Upload an encoded descriptor in encoded_desc of the given version. This * descriptor is for the service identity_pk and blinded_pk used to setup the * directory connection identifier. It is uploaded to the directory hsdir_rs @@ -3041,6 +3074,12 @@ hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports, goto err; } + /* Build the onion address for logging purposes but also the control port + * uses it for the HS_DESC event. */ + hs_build_address(&service->keys.identity_pk, + (uint8_t) service->config.version, + service->onion_address); + /* The only way the registration can fail is if the service public key * already exists. */ if (BUG(register_service(hs_service_map, service) < 0)) { @@ -3050,14 +3089,10 @@ hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports, goto err; } - /* Last step is to build the onion address. */ - hs_build_address(&service->keys.identity_pk, - (uint8_t) service->config.version, - service->onion_address); - *address_out = tor_strdup(service->onion_address); - log_info(LD_CONFIG, "Added ephemeral v3 onion service: %s", safe_str_client(service->onion_address)); + + *address_out = tor_strdup(service->onion_address); ret = RSAE_OKAY; goto end; diff --git a/src/or/hs_service.h b/src/or/hs_service.h index f3cd49e073..911e177702 100644 --- a/src/or/hs_service.h +++ b/src/or/hs_service.h @@ -50,6 +50,9 @@ typedef struct hs_service_intro_point_t { /* Legacy key if that intro point doesn't support v3. This should be used if * the base object legacy flag is set. */ crypto_pk_t *legacy_key; + /* Legacy key SHA1 public key digest. This should be used only if the base + * object legacy flag is set. */ + uint8_t legacy_key_digest[DIGEST_LEN]; /* Amount of INTRODUCE2 cell accepted from this intro point. */ uint64_t introduce2_count; @@ -260,6 +263,7 @@ void hs_service_lists_fnames_for_sandbox(smartlist_t *file_list, int hs_service_set_conn_addr_port(const origin_circuit_t *circ, edge_connection_t *conn); +void hs_service_map_has_changed(void); void hs_service_dir_info_changed(void); void hs_service_run_scheduled_events(time_t now); void hs_service_circuit_has_opened(origin_circuit_t *circ); diff --git a/src/or/hs_stats.c b/src/or/hs_stats.c index 3e183a5bfc..1e2a96945b 100644 --- a/src/or/hs_stats.c +++ b/src/or/hs_stats.c @@ -3,7 +3,7 @@ /** * \file hs_stats.c - * \brief Keeps stats about the activity of our hidden service. + * \brief Keeps stats about the activity of our onion service(s). **/ #include "or.h" @@ -42,14 +42,14 @@ hs_stats_get_n_introduce2_v2_cells(void) return n_introduce2_v2; } -/** Note that we attempted to launch another circuit to a rendezvous point */ +/** Note that we attempted to launch another circuit to a rendezvous point. */ void hs_stats_note_service_rendezvous_launch(void) { n_rendezvous_launches++; } -/** Return the number of rendezvous circuits we have attempted to launch */ +/** Return the number of rendezvous circuits we have attempted to launch. */ uint32_t hs_stats_get_n_rendezvous_launches(void) { diff --git a/src/or/include.am b/src/or/include.am index c1e23dd3d9..59d593a5e9 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -41,10 +41,8 @@ LIBTOR_A_SOURCES = \ src/or/consdiffmgr.c \ src/or/control.c \ src/or/cpuworker.c \ - src/or/dircollate.c \ src/or/directory.c \ src/or/dirserv.c \ - src/or/dirvote.c \ src/or/dns.c \ src/or/dnsserv.c \ src/or/dos.c \ @@ -76,8 +74,6 @@ LIBTOR_A_SOURCES = \ src/or/onion.c \ src/or/onion_fast.c \ src/or/onion_tap.c \ - src/or/shared_random.c \ - src/or/shared_random_state.c \ src/or/transports.c \ src/or/parsecommon.c \ src/or/periodic.c \ @@ -91,6 +87,7 @@ LIBTOR_A_SOURCES = \ src/or/policies.c \ src/or/reasons.c \ src/or/relay.c \ + src/or/relay_crypto.c \ src/or/rendcache.c \ src/or/rendclient.c \ src/or/rendcommon.c \ @@ -106,15 +103,43 @@ LIBTOR_A_SOURCES = \ src/or/scheduler.c \ src/or/scheduler_kist.c \ src/or/scheduler_vanilla.c \ + src/or/shared_random_client.c \ src/or/statefile.c \ src/or/status.c \ src/or/torcert.c \ src/or/tor_api.c \ + src/or/voting_schedule.c \ src/or/onion_ntor.c \ $(tor_platform_source) +# +# Modules are conditionnally compiled in tor starting here. We add the C files +# only if the modules has been enabled at configure time. We always add the +# source files of every module to libtor-testing.a so we can build the unit +# tests for everything. See the UNITTESTS_ENABLED branch below. +# +LIBTOR_TESTING_A_SOURCES = $(LIBTOR_A_SOURCES) + +# The Directory Authority module. +MODULE_DIRAUTH_SOURCES = \ + src/or/dirauth/dircollate.c \ + src/or/dirauth/dirvote.c \ + src/or/dirauth/shared_random.c \ + src/or/dirauth/shared_random_state.c +if BUILD_MODULE_DIRAUTH +LIBTOR_A_SOURCES += $(MODULE_DIRAUTH_SOURCES) +endif + src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES) -src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES) +if UNITTESTS_ENABLED + +# Add the sources of the modules that are needed for tests to work here. +LIBTOR_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES) + +src_or_libtor_testing_a_SOURCES = $(LIBTOR_TESTING_A_SOURCES) +else +src_or_libtor_testing_a_SOURCES = +endif src_or_tor_SOURCES = src/or/tor_main.c AM_CPPFLAGS += -I$(srcdir)/src/or -Isrc/or @@ -184,10 +209,8 @@ ORHEADERS = \ src/or/consdiffmgr.h \ src/or/control.h \ src/or/cpuworker.h \ - src/or/dircollate.h \ src/or/directory.h \ src/or/dirserv.h \ - src/or/dirvote.h \ src/or/dns.h \ src/or/dns_structs.h \ src/or/dnsserv.h \ @@ -224,8 +247,6 @@ ORHEADERS = \ src/or/onion_ntor.h \ src/or/onion_tap.h \ src/or/or.h \ - src/or/shared_random.h \ - src/or/shared_random_state.h \ src/or/transports.h \ src/or/parsecommon.h \ src/or/periodic.h \ @@ -238,6 +259,7 @@ ORHEADERS = \ src/or/proto_socks.h \ src/or/reasons.h \ src/or/relay.h \ + src/or/relay_crypto.h \ src/or/rendcache.h \ src/or/rendclient.h \ src/or/rendcommon.h \ @@ -252,10 +274,23 @@ ORHEADERS = \ src/or/routerset.h \ src/or/routerparse.h \ src/or/scheduler.h \ + src/or/shared_random_client.h \ src/or/statefile.h \ src/or/status.h \ src/or/torcert.h \ - src/or/tor_api_internal.h + src/or/tor_api_internal.h \ + src/or/voting_schedule.h + +# We add the headers of the modules even though they are disabled so we can +# properly compiled the entry points stub. + +# The Directory Authority module headers. +ORHEADERS += \ + src/or/dirauth/dircollate.h \ + src/or/dirauth/dirvote.h \ + src/or/dirauth/mode.h \ + src/or/dirauth/shared_random.h \ + src/or/dirauth/shared_random_state.h # This may someday want to be an installed file? noinst_HEADERS += src/or/tor_api.h diff --git a/src/or/keypin.c b/src/or/keypin.c index 1698dc184f..97e16c1f78 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -12,7 +12,7 @@ #include "orconfig.h" #include "compat.h" -#include "crypto.h" +#include "crypto_digest.h" #include "crypto_format.h" #include "di_ops.h" #include "ht.h" @@ -289,8 +289,10 @@ static int keypin_journal_fd = -1; int keypin_open_journal(const char *fname) { - /* O_SYNC ??*/ - int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY, 0600); +#ifndef O_SYNC +#define O_SYNC 0 +#endif + int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY|O_SYNC, 0600); if (fd < 0) goto err; @@ -417,10 +419,11 @@ keypin_load_journal_impl(const char *data, size_t size) ++n_entries; } - int severity = (n_corrupt_lines || n_duplicates) ? LOG_WARN : LOG_INFO; + int severity = (n_corrupt_lines || n_duplicates) ? LOG_NOTICE : LOG_INFO; tor_log(severity, LD_DIRSERV, "Loaded %d entries from keypin journal. " - "Found %d corrupt lines, %d duplicates, and %d conflicts.", + "Found %d corrupt lines (ignored), %d duplicates (harmless), " + "and %d conflicts (resolved in favor or more recent entry).", n_entries, n_corrupt_lines, n_duplicates, n_conflicts); return 0; diff --git a/src/or/main.c b/src/or/main.c index 3aa730dcec..c69a5b8b8b 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -59,6 +59,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "circuitmux_ewma.h" #include "command.h" #include "compress.h" #include "config.h" @@ -70,9 +71,9 @@ #include "control.h" #include "cpuworker.h" #include "crypto_s2k.h" +#include "crypto_rand.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "dns.h" #include "dnsserv.h" #include "dos.h" @@ -103,7 +104,6 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" -#include "shared_random.h" #include "statefile.h" #include "status.h" #include "tor_api.h" @@ -118,6 +118,10 @@ #include <event2/event.h> +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random.h" + #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -133,7 +137,7 @@ void evdns_shutdown(int); #ifdef HAVE_RUST // helper function defined in Rust to output a log message indicating if tor is // running with Rust enabled. See src/rust/tor_util -char *rust_welcome_string(void); +void rust_log_welcome_string(void); #endif /********* PROTOTYPES **********/ @@ -152,24 +156,12 @@ static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; /********* START VARIABLES **********/ -int global_read_bucket; /**< Max number of bytes I can read this second. */ -int global_write_bucket; /**< Max number of bytes I can write this second. */ - -/** Max number of relayed (bandwidth class 1) bytes I can read this second. */ -int global_relayed_read_bucket; -/** Max number of relayed (bandwidth class 1) bytes I can write this second. */ -int global_relayed_write_bucket; -/** What was the read bucket before the last second_elapsed_callback() call? - * (used to determine how many bytes we've read). */ -static int stats_prev_global_read_bucket; -/** What was the write bucket before the last second_elapsed_callback() call? - * (used to determine how many bytes we've written). */ -static int stats_prev_global_write_bucket; - -/* DOCDOC stats_prev_n_read */ -static uint64_t stats_prev_n_read = 0; -/* DOCDOC stats_prev_n_written */ -static uint64_t stats_prev_n_written = 0; + +/* Token bucket for all traffic. */ +token_bucket_rw_t global_bucket; + +/* Token bucket for relayed traffic. */ +token_bucket_rw_t global_relayed_bucket; /* XXX we might want to keep stats about global_relayed_*_bucket too. Or not.*/ /** How many bytes have we read since we started the process? */ @@ -179,7 +171,7 @@ static uint64_t stats_n_bytes_written = 0; /** What time did this process start up? */ time_t time_of_process_start = 0; /** How many seconds have we been running? */ -long stats_n_seconds_working = 0; +static long stats_n_seconds_working = 0; /** How many times have we returned from the main loop successfully? */ static uint64_t stats_n_main_loop_successes = 0; /** How many times have we received an error from the main loop? */ @@ -193,6 +185,8 @@ static uint64_t stats_n_main_loop_idle = 0; static time_t time_of_last_signewnym = 0; /** Is there a signewnym request we're currently waiting to handle? */ static int signewnym_is_pending = 0; +/** Mainloop event for the deferred signewnym call. */ +static mainloop_event_t *handle_deferred_signewnym_ev = NULL; /** How many times have we called newnym? */ static unsigned newnym_epoch = 0; @@ -410,6 +404,38 @@ connection_unlink(connection_t *conn) connection_free(conn); } +/** Event that invokes schedule_active_linked_connections_cb. */ +static mainloop_event_t *schedule_active_linked_connections_event = NULL; + +/** + * Callback: used to activate read events for all linked connections, so + * libevent knows to call their read callbacks. This callback run as a + * postloop event, so that the events _it_ activates don't happen until + * Libevent has a chance to check for other events. + */ +static void +schedule_active_linked_connections_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + + /* All active linked conns should get their read events activated, + * so that libevent knows to run their callbacks. */ + SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, + event_active(conn->read_event, EV_READ, 1)); + + /* Reactivate the event if we still have connections in the active list. + * + * A linked connection doesn't get woken up by I/O but rather artificially + * by this event callback. It has directory data spooled in it and it is + * sent incrementally by small chunks unless spool_eagerly is true. For that + * to happen, we need to induce the activation of the read event so it can + * be flushed. */ + if (smartlist_len(active_linked_connection_lst)) { + mainloop_event_activate(schedule_active_linked_connections_event); + } +} + /** Initialize the global connection list, closeable connection list, * and active connection list. */ STATIC void @@ -431,6 +457,7 @@ add_connection_to_closeable_list(connection_t *conn) tor_assert(conn->marked_for_close); assert_connection_ok(conn, time(NULL)); smartlist_add(closeable_connection_lst, conn); + mainloop_schedule_postloop_cleanup(); } /** Return 1 if conn is on the closeable list, else return 0. */ @@ -458,21 +485,37 @@ get_connection_array, (void)) return connection_array; } -/** Provides the traffic read and written over the life of the process. */ - +/** + * Return the amount of network traffic read, in bytes, over the life of this + * process. + */ MOCK_IMPL(uint64_t, get_bytes_read,(void)) { return stats_n_bytes_read; } -/* DOCDOC get_bytes_written */ +/** + * Return the amount of network traffic read, in bytes, over the life of this + * process. + */ MOCK_IMPL(uint64_t, get_bytes_written,(void)) { return stats_n_bytes_written; } +/** + * Increment the amount of network traffic read and written, over the life of + * this process. + */ +void +stats_increment_bytes_read_and_written(uint64_t r, uint64_t w) +{ + stats_n_bytes_read += r; + stats_n_bytes_written += w; +} + /** Set the event mask on <b>conn</b> to <b>events</b>. (The event * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ @@ -710,20 +753,6 @@ connection_should_read_from_linked_conn(connection_t *conn) return 0; } -/** If we called event_base_loop() and told it to never stop until it - * runs out of events, now we've changed our mind: tell it we want it to - * exit once the current round of callbacks is done, so that we can - * run external code, and then return to the main loop. */ -void -tell_event_loop_to_run_external_code(void) -{ - if (!called_loop_once) { - struct timeval tv = { 0, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &tv); - called_loop_once = 1; /* hack to avoid adding more exit events */ - } -} - /** Event to run 'shutdown did not work callback'. */ static struct event *shutdown_did_not_work_event = NULL; @@ -779,8 +808,9 @@ tor_shutdown_event_loop_and_exit(int exitcode) shutdown_did_not_work_callback, NULL); event_add(shutdown_did_not_work_event, &ten_seconds); - /* Unlike loopexit, loopbreak prevents other callbacks from running. */ - tor_event_base_loopbreak(tor_libevent_get_base()); + /* Unlike exit_loop_after_delay(), exit_loop_after_callback + * prevents other callbacks from running. */ + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); } /** Return true iff tor_shutdown_event_loop_and_exit() has been called. */ @@ -802,10 +832,7 @@ connection_start_reading_from_linked_conn(connection_t *conn) if (!conn->active_on_link) { conn->active_on_link = 1; smartlist_add(active_linked_connection_lst, conn); - /* make sure that the event_base_loop() function exits at - * the end of its run through the current connections, so we can - * activate read events for linked connections. */ - tell_event_loop_to_run_external_code(); + mainloop_event_activate(schedule_active_linked_connections_event); } else { tor_assert(smartlist_contains(active_linked_connection_lst, conn)); } @@ -1008,7 +1035,8 @@ conn_close_if_marked(int i) LOG_FN_CONN(conn, (LOG_INFO,LD_NET, "Holding conn (fd %d) open for more flushing.", (int)conn->s)); - conn->timestamp_lastwritten = now; /* reset so we can flush more */ + conn->timestamp_last_write_allowed = now; /* reset so we can flush + * more */ } else if (sz == 0) { /* Also, retval==0. If we get here, we didn't want to write anything * (because of rate-limiting) and we didn't. */ @@ -1019,20 +1047,18 @@ conn_close_if_marked(int i) * busy Libevent loops where we keep ending up here and returning * 0 until we are no longer blocked on bandwidth. */ - if (connection_is_writing(conn)) { - conn->write_blocked_on_bw = 1; - connection_stop_writing(conn); - } - if (connection_is_reading(conn)) { - /* XXXX+ We should make this code unreachable; if a connection is - * marked for close and flushing, there is no point in reading to it - * at all. Further, checking at this point is a bit of a hack: it - * would make much more sense to react in - * connection_handle_read_impl, or to just stop reading in - * mark_and_flush */ - conn->read_blocked_on_bw = 1; - connection_stop_reading(conn); + connection_consider_empty_write_buckets(conn); + /* Make sure that consider_empty_buckets really disabled the + * connection: */ + if (BUG(connection_is_writing(conn))) { + connection_write_bw_exhausted(conn, true); } + + /* The connection is being held due to write rate limit and thus will + * flush its data later. We need to stop reading because this + * connection is about to be closed once flushed. It should not + * process anything more coming in at this stage. */ + connection_stop_reading(conn); } return 0; } @@ -1059,9 +1085,8 @@ conn_close_if_marked(int i) * reason. */ static void -directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg) +directory_all_unreachable_cb(mainloop_event_t *event, void *arg) { - (void)fd; (void)event; (void)arg; @@ -1081,7 +1106,7 @@ directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg) control_event_general_error("DIR_ALL_UNREACHABLE"); } -static struct event *directory_all_unreachable_cb_event = NULL; +static mainloop_event_t *directory_all_unreachable_cb_event = NULL; /** We've just tried every dirserver we know about, and none of * them were reachable. Assume the network is down. Change state @@ -1094,16 +1119,15 @@ directory_all_unreachable(time_t now) { (void)now; - stats_n_seconds_working=0; /* reset it */ + reset_uptime(); /* reset it */ if (!directory_all_unreachable_cb_event) { directory_all_unreachable_cb_event = - tor_event_new(tor_libevent_get_base(), - -1, EV_READ, directory_all_unreachable_cb, NULL); + mainloop_event_new(directory_all_unreachable_cb, NULL); tor_assert(directory_all_unreachable_cb_event); } - event_active(directory_all_unreachable_cb_event, EV_READ, 1); + mainloop_event_activate(directory_all_unreachable_cb_event); } /** This function is called whenever we successfully pull down some new @@ -1143,7 +1167,7 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs) if (server_mode(options) && !net_is_disabled() && !from_cache && (have_completed_a_circuit() || !any_predicted_circuits(now))) - consider_testing_reachability(1, 1); + router_do_reachability_checks(1, 1); } /** Perform regular maintenance tasks for a single connection. This @@ -1159,7 +1183,7 @@ run_connection_housekeeping(int i, time_t now) channel_t *chan = NULL; int have_any_circuits; int past_keepalive = - now >= conn->timestamp_lastwritten + options->KeepalivePeriod; + now >= conn->timestamp_last_write_allowed + options->KeepalivePeriod; if (conn->outbuf && !connection_get_outbuf_len(conn) && conn->type == CONN_TYPE_OR) @@ -1174,10 +1198,10 @@ run_connection_housekeeping(int i, time_t now) * if a server or received if a client) for 5 min */ if (conn->type == CONN_TYPE_DIR && ((DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastwritten + conn->timestamp_last_write_allowed + options->TestingDirConnectionMaxStall < now) || (!DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastread + conn->timestamp_last_read_allowed + options->TestingDirConnectionMaxStall < now))) { log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)", (int)conn->s, conn->purpose); @@ -1235,7 +1259,8 @@ run_connection_housekeeping(int i, time_t now) } else if (we_are_hibernating() && ! have_any_circuits && !connection_get_outbuf_len(conn)) { - /* We're hibernating, there's no circuits, and nothing to flush.*/ + /* We're hibernating or shutting down, there's no circuits, and nothing to + * flush.*/ log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " "[Hibernating or exiting].", (int)conn->s,conn->address, conn->port); @@ -1253,13 +1278,14 @@ run_connection_housekeeping(int i, time_t now) connection_or_close_normally(TO_OR_CONN(conn), 0); } else if ( now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 && - now >= conn->timestamp_lastwritten + options->KeepalivePeriod*10) { + now >= + conn->timestamp_last_write_allowed + options->KeepalivePeriod*10) { log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL, "Expiring stuck OR connection to fd %d (%s:%d). (%d bytes to " "flush; %d seconds since last write)", (int)conn->s, conn->address, conn->port, (int)connection_get_outbuf_len(conn), - (int)(now-conn->timestamp_lastwritten)); + (int)(now-conn->timestamp_last_write_allowed)); connection_or_close_normally(TO_OR_CONN(conn), 0); } else if (past_keepalive && !connection_get_outbuf_len(conn)) { /* send a padding cell */ @@ -1296,6 +1322,16 @@ signewnym_impl(time_t now) control_event_signal(SIGNEWNYM); } +/** Callback: run a deferred signewnym. */ +static void +handle_deferred_signewnym_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + log_info(LD_CONTROL, "Honoring delayed NEWNYM request"); + signewnym_impl(time(NULL)); +} + /** Return the number of times that signewnym has been called. */ unsigned get_signewnym_epoch(void) @@ -1311,71 +1347,106 @@ static int periodic_events_initialized = 0; #undef CALLBACK #define CALLBACK(name) \ static int name ## _callback(time_t, const or_options_t *) -CALLBACK(rotate_onion_key); -CALLBACK(check_onion_keys_expiry_time); -CALLBACK(check_ed_keys); -CALLBACK(launch_descriptor_fetches); -CALLBACK(rotate_x509_certificate); CALLBACK(add_entropy); -CALLBACK(launch_reachability_tests); -CALLBACK(downrate_stability); -CALLBACK(save_stability); CALLBACK(check_authority_cert); +CALLBACK(check_canonical_channels); +CALLBACK(check_descriptor); +CALLBACK(check_dns_honesty); +CALLBACK(check_ed_keys); CALLBACK(check_expired_networkstatus); -CALLBACK(write_stats_file); -CALLBACK(record_bridge_stats); +CALLBACK(check_for_reachability_bw); +CALLBACK(check_onion_keys_expiry_time); CALLBACK(clean_caches); +CALLBACK(clean_consdiffmgr); +CALLBACK(dirvote); +CALLBACK(downrate_stability); +CALLBACK(expire_old_ciruits_serverside); +CALLBACK(fetch_networkstatus); +CALLBACK(heartbeat); +CALLBACK(hs_service); +CALLBACK(launch_descriptor_fetches); +CALLBACK(launch_reachability_tests); +CALLBACK(reachability_warnings); +CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); +CALLBACK(reset_padding_counts); CALLBACK(retry_dns); -CALLBACK(check_descriptor); -CALLBACK(check_for_reachability_bw); -CALLBACK(fetch_networkstatus); CALLBACK(retry_listeners); -CALLBACK(expire_old_ciruits_serverside); -CALLBACK(check_dns_honesty); +CALLBACK(rotate_onion_key); +CALLBACK(rotate_x509_certificate); +CALLBACK(save_stability); +CALLBACK(save_state); CALLBACK(write_bridge_ns); -CALLBACK(check_fw_helper_app); -CALLBACK(heartbeat); -CALLBACK(clean_consdiffmgr); -CALLBACK(reset_padding_counts); -CALLBACK(check_canonical_channels); -CALLBACK(hs_service); +CALLBACK(write_stats_file); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name) PERIODIC_EVENT(name) - -static periodic_event_item_t periodic_events[] = { - CALLBACK(rotate_onion_key), - CALLBACK(check_onion_keys_expiry_time), - CALLBACK(check_ed_keys), - CALLBACK(launch_descriptor_fetches), - CALLBACK(rotate_x509_certificate), - CALLBACK(add_entropy), - CALLBACK(launch_reachability_tests), - CALLBACK(downrate_stability), - CALLBACK(save_stability), - CALLBACK(check_authority_cert), - CALLBACK(check_expired_networkstatus), - CALLBACK(write_stats_file), - CALLBACK(record_bridge_stats), - CALLBACK(clean_caches), - CALLBACK(rend_cache_failure_clean), - CALLBACK(retry_dns), - CALLBACK(check_descriptor), - CALLBACK(check_for_reachability_bw), - CALLBACK(fetch_networkstatus), - CALLBACK(retry_listeners), - CALLBACK(expire_old_ciruits_serverside), - CALLBACK(check_dns_honesty), - CALLBACK(write_bridge_ns), - CALLBACK(check_fw_helper_app), - CALLBACK(heartbeat), - CALLBACK(clean_consdiffmgr), - CALLBACK(reset_padding_counts), - CALLBACK(check_canonical_channels), - CALLBACK(hs_service), +#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) + +STATIC periodic_event_item_t periodic_events[] = { + /* Everyone needs to run those. */ + CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + + /* Routers (bridge and relay) only. */ + CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + + /* Authorities (bridge and directory) only. */ + CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + + /* Directory authority only. */ + CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), + CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + + /* Relay only. */ + CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, + PERIODIC_EVENT_FLAG_NEED_NET), + + /* Hidden Service service only. */ + CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, + PERIODIC_EVENT_FLAG_NEED_NET), + + /* Bridge only. */ + CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + + /* Client only. */ + CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + + /* Bridge Authority only. */ + CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + + /* Directory server only. */ + CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + END_OF_PERIODIC_EVENTS }; #undef CALLBACK @@ -1385,9 +1456,11 @@ static periodic_event_item_t periodic_events[] = { * can access them by name. We also keep them inside periodic_events[] * so that we can implement "reset all timers" in a reasonable way. */ static periodic_event_item_t *check_descriptor_event=NULL; +static periodic_event_item_t *dirvote_event=NULL; static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; +static periodic_event_item_t *save_state_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1417,6 +1490,39 @@ find_periodic_event(const char *name) return NULL; } +/** Return a bitmask of the roles this tor instance is configured for using + * the given options. */ +STATIC int +get_my_roles(const or_options_t *options) +{ + tor_assert(options); + + int roles = 0; + int is_bridge = options->BridgeRelay; + int is_relay = server_mode(options); + int is_dirauth = authdir_mode_v3(options); + int is_bridgeauth = authdir_mode_bridge(options); + int is_hidden_service = !!hs_service_get_num_services() || + !!rend_num_services(); + int is_dirserver = dir_server_mode(options); + /* We also consider tor to have the role of a client if the ControlPort is + * set because a lot of things can be done over the control port which + * requires tor to have basic functionnalities. */ + int is_client = options_any_client_port_set(options) || + options->ControlPort_set || + options->OwningControllerFD >= 0; + + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; + if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; + if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; + if (is_dirauth) roles |= PERIODIC_EVENT_ROLE_DIRAUTH; + if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; + if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; + if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + + return roles; +} + /** Event to run initialize_periodic_events_cb */ static struct event *initialize_periodic_events_event = NULL; @@ -1431,11 +1537,10 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data) (void) fd; (void) events; (void) data; + tor_event_free(initialize_periodic_events_event); - int i; - for (i = 0; periodic_events[i].name; ++i) { - periodic_event_launch(&periodic_events[i]); - } + + rescan_periodic_events(get_options()); } /** Set up all the members of periodic_events[], and configure them all to be @@ -1446,6 +1551,7 @@ initialize_periodic_events(void) tor_assert(periodic_events_initialized == 0); periodic_events_initialized = 1; + /* Set up all periodic events. We'll launch them by roles. */ int i; for (i = 0; periodic_events[i].name; ++i) { periodic_event_setup(&periodic_events[i]); @@ -1455,9 +1561,11 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); NAMED_CALLBACK(check_dns_honesty); + NAMED_CALLBACK(save_state); struct timeval one_second = { 1, 0 }; initialize_periodic_events_event = tor_evtimer_new( @@ -1473,6 +1581,58 @@ teardown_periodic_events(void) for (i = 0; periodic_events[i].name; ++i) { periodic_event_destroy(&periodic_events[i]); } + periodic_events_initialized = 0; +} + +/** Do a pass at all our periodic events, disable those we don't need anymore + * and enable those we need now using the given options. */ +void +rescan_periodic_events(const or_options_t *options) +{ + tor_assert(options); + + /* Avoid scanning the event list if we haven't initialized it yet. This is + * particularly useful for unit tests in order to avoid initializing main + * loop events everytime. */ + if (!periodic_events_initialized) { + return; + } + + int roles = get_my_roles(options); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + + /* Handle the event flags. */ + if (net_is_disabled() && + (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) { + continue; + } + + /* Enable the event if needed. It is safe to enable an event that was + * already enabled. Same goes for disabling it. */ + if (item->roles & roles) { + log_debug(LD_GENERAL, "Launching periodic event %s", item->name); + periodic_event_enable(item); + } else { + log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); + periodic_event_disable(item); + } + } +} + +/* We just got new options globally set, see if we need to enabled or disable + * periodic events. */ +void +periodic_events_on_new_options(const or_options_t *options) +{ + /* Only if we've already initialized the events, rescan the list which will + * enable or disable events depending on our roles. This will be called at + * bootup and we don't want this function to initialize the events because + * they aren't set up at this stage. */ + if (periodic_events_initialized) { + rescan_periodic_events(options); + } } /** @@ -1483,8 +1643,9 @@ teardown_periodic_events(void) void reschedule_descriptor_update_check(void) { - tor_assert(check_descriptor_event); - periodic_event_reschedule(check_descriptor_event); + if (check_descriptor_event) { + periodic_event_reschedule(check_descriptor_event); + } } /** @@ -1501,6 +1662,35 @@ reschedule_directory_downloads(void) periodic_event_reschedule(launch_descriptor_fetches_event); } +/** Mainloop callback: clean up circuits, channels, and connections + * that are pending close. */ +static void +postloop_cleanup_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + circuit_close_all_marked(); + close_closeable_connections(); + channel_run_cleanup(); + channel_listener_run_cleanup(); +} + +/** Event to run postloop_cleanup_cb */ +static mainloop_event_t *postloop_cleanup_ev=NULL; + +/** Schedule a post-loop event to clean up marked channels, connections, and + * circuits. */ +void +mainloop_schedule_postloop_cleanup(void) +{ + if (PREDICT_UNLIKELY(postloop_cleanup_ev == NULL)) { + // (It's possible that we can get here if we decide to close a connection + // in the earliest stages of our configuration, before we create events.) + return; + } + mainloop_event_activate(postloop_cleanup_ev); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1538,17 +1728,6 @@ run_scheduled_events(time_t now) */ consider_hibernation(now); - /* 0b. If we've deferred a signewnym, make sure it gets handled - * eventually. */ - if (signewnym_is_pending && - time_of_last_signewnym + MAX_SIGNEWNYM_RATE <= now) { - log_info(LD_CONTROL, "Honoring delayed NEWNYM request"); - signewnym_impl(now); - } - - /* 0c. If we've deferred log messages for the controller, handle them now */ - flush_pending_log_callbacks(); - /* Maybe enough time elapsed for us to reconsider a circuit. */ circuit_upgrade_circuits_from_guard_wait(); @@ -1562,10 +1741,6 @@ run_scheduled_events(time_t now) accounting_run_housekeeping(now); } - if (authdir_mode_v3(options)) { - dirvote_act(options, now); - } - /* 3a. Every second, we examine pending circuits and prune the * ones which have been pending for more than a few seconds. * We do this before step 4, so it can try building more if @@ -1599,12 +1774,6 @@ run_scheduled_events(time_t now) circuit_expire_old_circs_as_needed(now); } - if (!net_is_disabled()) { - /* This is usually redundant with circuit_build_needed_circs() above, - * but it is very fast when there is no work to do. */ - connection_ap_attach_pending(0); - } - /* 5. We do housekeeping for each connection... */ channel_update_bad_for_new_circs(NULL, 0); int i; @@ -1612,32 +1781,9 @@ run_scheduled_events(time_t now) run_connection_housekeeping(i, now); } - /* 6. And remove any marked circuits... */ - circuit_close_all_marked(); - - /* 8. and blow away any connections that need to die. have to do this now, - * because if we marked a conn for close and left its socket -1, then - * we'll pass it to poll/select and bad things will happen. - */ - close_closeable_connections(); - - /* 8b. And if anything in our state is ready to get flushed to disk, we - * flush it. */ - or_state_save(now); - - /* 8c. Do channel cleanup just like for connections */ - channel_run_cleanup(); - channel_listener_run_cleanup(); - /* 11b. check pending unconfigured managed proxies */ if (!net_is_disabled() && pt_proxies_configuration_pending()) pt_configure_remaining_proxies(); - - /* 12. launch diff computations. (This is free if there are none to - * launch.) */ - if (dir_server_mode(options)) { - consdiffmgr_rescan(); - } } /* Periodic callback: rotate the onion keys after the period defined by the @@ -1847,6 +1993,40 @@ check_authority_cert_callback(time_t now, const or_options_t *options) } /** + * Scheduled callback: Run directory-authority voting functionality. + * + * The schedule is a bit complicated here, so dirvote_act() manages the + * schedule itself. + **/ +static int +dirvote_callback(time_t now, const or_options_t *options) +{ + if (!authdir_mode_v3(options)) { + tor_assert_nonfatal_unreached(); + return 3600; + } + + time_t next = dirvote_act(options, now); + if (BUG(next == TIME_MAX)) { + /* This shouldn't be returned unless we called dirvote_act() without + * being an authority. If it happens, maybe our configuration will + * fix itself in an hour or so? */ + return 3600; + } + return safe_timer_diff(now, next); +} + +/** Reschedule the directory-authority voting event. Run this whenever the + * schedule has changed. */ +void +reschedule_dirvote(const or_options_t *options) +{ + if (periodic_events_initialized && authdir_mode_v3(options)) { + periodic_event_reschedule(dirvote_event); + } +} + +/** * Periodic callback: If our consensus is too old, recalculate whether * we can actually use it. */ @@ -1869,6 +2049,34 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options) } /** + * Scheduled callback: Save the state file to disk if appropriate. + */ +static int +save_state_callback(time_t now, const or_options_t *options) +{ + (void) options; + (void) or_state_save(now); // only saves if appropriate + const time_t next_write = get_or_state()->next_write; + if (next_write == TIME_MAX) { + return 86400; + } + return safe_timer_diff(now, next_write); +} + +/** Reschedule the event for saving the state file. + * + * Run this when the state becomes dirty. */ +void +reschedule_or_state_save(void) +{ + if (save_state_event == NULL) { + /* This can happen early on during startup. */ + return; + } + periodic_event_reschedule(save_state_event); +} + +/** * Periodic callback: Write statistics to disk if appropriate. */ static int @@ -1940,14 +2148,14 @@ reset_padding_counts_callback(time_t now, const or_options_t *options) return REPHIST_CELL_PADDING_COUNTS_INTERVAL; } +static int should_init_bridge_stats = 1; + /** * Periodic callback: Write bridge statistics to disk if appropriate. */ static int record_bridge_stats_callback(time_t now, const or_options_t *options) { - static int should_init_bridge_stats = 1; - /* 1h. Check whether we should write bridge statistics to disk. */ if (should_record_bridge_info(options)) { @@ -2062,8 +2270,8 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options) if (server_mode(options) && (have_completed_a_circuit() || !any_predicted_circuits(now)) && !net_is_disabled()) { - if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - consider_testing_reachability(1, dirport_reachability_count==0); + if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + router_do_reachability_checks(1, dirport_reachability_count==0); if (++dirport_reachability_count > 5) dirport_reachability_count = 0; return 1; @@ -2141,6 +2349,56 @@ expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options) } /** + * Callback: Send warnings if Tor doesn't find its ports reachable. + */ +static int +reachability_warnings_callback(time_t now, const or_options_t *options) +{ + (void) now; + + if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime()); + } + + if (server_mode(options) && + !net_is_disabled() && + have_completed_a_circuit()) { + /* every 20 minutes, check and complain if necessary */ + const routerinfo_t *me = router_get_my_routerinfo(); + if (me && !check_whether_orport_reachable(options)) { + char *address = tor_dup_ip(me->addr); + log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " + "its ORPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->or_port); + control_event_server_status(LOG_WARN, + "REACHABILITY_FAILED ORADDRESS=%s:%d", + address, me->or_port); + tor_free(address); + } + + if (me && !check_whether_dirport_reachable(options)) { + char *address = tor_dup_ip(me->addr); + log_warn(LD_CONFIG, + "Your server (%s:%d) has not managed to confirm that its " + "DirPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->dir_port); + control_event_server_status(LOG_WARN, + "REACHABILITY_FAILED DIRADDRESS=%s:%d", + address, me->dir_port); + tor_free(address); + } + } + + return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT; +} + +static int dns_honesty_first_time = 1; + +/** * Periodic event: if we're an exit, see if our DNS server is telling us * obvious lies. */ @@ -2155,10 +2413,9 @@ check_dns_honesty_callback(time_t now, const or_options_t *options) router_my_exit_policy_is_reject_star()) return PERIODIC_EVENT_NO_UPDATE; - static int first_time = 1; - if (first_time) { + if (dns_honesty_first_time) { /* Don't launch right when we start */ - first_time = 0; + dns_honesty_first_time = 0; return crypto_rand_int_range(60, 180); } @@ -2182,32 +2439,7 @@ write_bridge_ns_callback(time_t now, const or_options_t *options) return PERIODIC_EVENT_NO_UPDATE; } -/** - * Periodic callback: poke the tor-fw-helper app if we're using one. - */ -static int -check_fw_helper_app_callback(time_t now, const or_options_t *options) -{ - if (net_is_disabled() || - ! server_mode(options) || - ! options->PortForwarding || - options->NoExec) { - return PERIODIC_EVENT_NO_UPDATE; - } - /* 11. check the port forwarding app */ - -#define PORT_FORWARDING_CHECK_INTERVAL 5 - smartlist_t *ports_to_forward = get_list_of_ports_to_forward(); - if (ports_to_forward) { - tor_check_port_forwarding(options->PortForwardingHelper, - ports_to_forward, - now); - - SMARTLIST_FOREACH(ports_to_forward, char *, cp, tor_free(cp)); - smartlist_free(ports_to_forward); - } - return PORT_FORWARDING_CHECK_INTERVAL; -} +static int heartbeat_callback_first_time = 1; /** * Periodic callback: write the heartbeat message in the logs. @@ -2218,16 +2450,14 @@ check_fw_helper_app_callback(time_t now, const or_options_t *options) static int heartbeat_callback(time_t now, const or_options_t *options) { - static int first = 1; - /* Check if heartbeat is disabled */ if (!options->HeartbeatPeriod) { return PERIODIC_EVENT_NO_UPDATE; } /* Skip the first one. */ - if (first) { - first = 0; + if (heartbeat_callback_first_time) { + heartbeat_callback_first_time = 0; return options->HeartbeatPeriod; } @@ -2246,7 +2476,7 @@ static int clean_consdiffmgr_callback(time_t now, const or_options_t *options) { (void)now; - if (server_mode(options)) { + if (dir_server_mode(options)) { consdiffmgr_cleanup(); } return CDM_CLEAN_CALLBACK_INTERVAL; @@ -2277,8 +2507,100 @@ hs_service_callback(time_t now, const or_options_t *options) /** Timer: used to invoke second_elapsed_callback() once per second. */ static periodic_timer_t *second_timer = NULL; -/** Number of libevent errors in the last second: we die if we get too many. */ -static int n_libevent_errors = 0; + +/** + * Enable or disable the per-second timer as appropriate, creating it if + * necessary. + */ +void +reschedule_per_second_timer(void) +{ + struct timeval one_second; + one_second.tv_sec = 1; + one_second.tv_usec = 0; + + if (! second_timer) { + second_timer = periodic_timer_new(tor_libevent_get_base(), + &one_second, + second_elapsed_callback, + NULL); + tor_assert(second_timer); + } + + const bool run_per_second_events = + control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + + if (run_per_second_events) { + periodic_timer_launch(second_timer, &one_second); + } else { + periodic_timer_disable(second_timer); + } +} + +/** Last time that update_current_time was called. */ +static time_t current_second = 0; +/** Last time that update_current_time updated current_second. */ +static monotime_coarse_t current_second_last_changed; + +/** + * Set the current time to "now", which should be the value returned by + * time(). Check for clock jumps and track the total number of seconds we + * have been running. + */ +void +update_current_time(time_t now) +{ + if (PREDICT_LIKELY(now == current_second)) { + /* We call this function a lot. Most frequently, the current second + * will not have changed, so we just return. */ + return; + } + + const time_t seconds_elapsed = current_second ? (now - current_second) : 0; + + /* Check the wall clock against the monotonic clock, so we can + * better tell idleness from clock jumps and/or other shenanigans. */ + monotime_coarse_t last_updated; + memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); + monotime_coarse_get(¤t_second_last_changed); + + /** How much clock jumping do we tolerate? */ +#define NUM_JUMPED_SECONDS_BEFORE_WARN 100 + + /** How much idleness do we tolerate? */ +#define NUM_IDLE_SECONDS_BEFORE_WARN 3600 + + if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { + // moving back in time is always a bad sign. + circuit_note_clock_jumped(seconds_elapsed, false); + } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { + /* Compare the monotonic clock to the result of time(). */ + const int32_t monotime_msec_passed = + monotime_coarse_diff_msec32(&last_updated, + ¤t_second_last_changed); + const int monotime_sec_passed = monotime_msec_passed / 1000; + const int discrepancy = monotime_sec_passed - (int)seconds_elapsed; + /* If the monotonic clock deviates from time(NULL), we have a couple of + * possibilities. On some systems, this means we have been suspended or + * sleeping. Everywhere, it can mean that the wall-clock time has + * been changed -- for example, with settimeofday(). + * + * On the other hand, if the monotonic time matches with the wall-clock + * time, we've probably just been idle for a while, with no events firing. + * we tolerate much more of that. + */ + const bool clock_jumped = abs(discrepancy) > 2; + + if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) { + circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped); + } + } else if (seconds_elapsed > 0) { + stats_n_seconds_working += seconds_elapsed; + } + + update_approx_time(now); + current_second = now; +} /** Libevent callback: invoked once every second. */ static void @@ -2287,83 +2609,22 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) /* XXXX This could be sensibly refactored into multiple callbacks, and we * could use Libevent's timers for this rather than checking the current * time against a bunch of timeouts every second. */ - static time_t current_second = 0; time_t now; - size_t bytes_written; - size_t bytes_read; - int seconds_elapsed; - const or_options_t *options = get_options(); (void)timer; (void)arg; - n_libevent_errors = 0; - - /* log_notice(LD_GENERAL, "Tick."); */ now = time(NULL); - update_approx_time(now); - - /* the second has rolled over. check more stuff. */ - seconds_elapsed = current_second ? (int)(now - current_second) : 0; - bytes_read = (size_t)(stats_n_bytes_read - stats_prev_n_read); - bytes_written = (size_t)(stats_n_bytes_written - stats_prev_n_written); - stats_prev_n_read = stats_n_bytes_read; - stats_prev_n_written = stats_n_bytes_written; - - control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); - control_event_stream_bandwidth_used(); - control_event_conn_bandwidth_used(); - control_event_circ_bandwidth_used(); - control_event_circuit_cell_stats(); - - if (server_mode(options) && - !net_is_disabled() && - seconds_elapsed > 0 && - have_completed_a_circuit() && - stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != - (stats_n_seconds_working+seconds_elapsed) / - TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - /* every 20 minutes, check and complain if necessary */ - const routerinfo_t *me = router_get_my_routerinfo(); - if (me && !check_whether_orport_reachable(options)) { - char *address = tor_dup_ip(me->addr); - log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " - "its ORPort is reachable. Relays do not publish descriptors " - "until their ORPort and DirPort are reachable. Please check " - "your firewalls, ports, address, /etc/hosts file, etc.", - address, me->or_port); - control_event_server_status(LOG_WARN, - "REACHABILITY_FAILED ORADDRESS=%s:%d", - address, me->or_port); - tor_free(address); - } - if (me && !check_whether_dirport_reachable(options)) { - char *address = tor_dup_ip(me->addr); - log_warn(LD_CONFIG, - "Your server (%s:%d) has not managed to confirm that its " - "DirPort is reachable. Relays do not publish descriptors " - "until their ORPort and DirPort are reachable. Please check " - "your firewalls, ports, address, /etc/hosts file, etc.", - address, me->dir_port); - control_event_server_status(LOG_WARN, - "REACHABILITY_FAILED DIRADDRESS=%s:%d", - address, me->dir_port); - tor_free(address); - } - } + /* We don't need to do this once-per-second any more: time-updating is + * only in this callback _because it is a callback_. It should be fine + * to disable this callback, and the time will still get updated. + */ + update_current_time(now); -/** If more than this many seconds have elapsed, probably the clock - * jumped: doesn't count. */ -#define NUM_JUMPED_SECONDS_BEFORE_WARN 100 - if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN || - seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { - circuit_note_clock_jumped(seconds_elapsed); - } else if (seconds_elapsed > 0) - stats_n_seconds_working += seconds_elapsed; + /* Maybe some controller events are ready to fire */ + control_per_second_events(); run_scheduled_events(now); - - current_second = now; /* remember which second it is, for next time */ } #ifdef HAVE_SYSTEMD_209 @@ -2379,70 +2640,6 @@ systemd_watchdog_callback(periodic_timer_t *timer, void *arg) } #endif /* defined(HAVE_SYSTEMD_209) */ -/** Timer: used to invoke refill_callback(). */ -static periodic_timer_t *refill_timer = NULL; - -/** Libevent callback: invoked periodically to refill token buckets - * and count r/w bytes. */ -static void -refill_callback(periodic_timer_t *timer, void *arg) -{ - static struct timeval current_millisecond; - struct timeval now; - - size_t bytes_written; - size_t bytes_read; - int milliseconds_elapsed = 0; - int seconds_rolled_over = 0; - - const or_options_t *options = get_options(); - - (void)timer; - (void)arg; - - tor_gettimeofday(&now); - - /* If this is our first time, no time has passed. */ - if (current_millisecond.tv_sec) { - long mdiff = tv_mdiff(¤t_millisecond, &now); - if (mdiff > INT_MAX) - mdiff = INT_MAX; - milliseconds_elapsed = (int)mdiff; - seconds_rolled_over = (int)(now.tv_sec - current_millisecond.tv_sec); - } - - bytes_written = stats_prev_global_write_bucket - global_write_bucket; - bytes_read = stats_prev_global_read_bucket - global_read_bucket; - - stats_n_bytes_read += bytes_read; - stats_n_bytes_written += bytes_written; - if (accounting_is_enabled(options) && milliseconds_elapsed >= 0) - accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over); - - if (milliseconds_elapsed > 0) - connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec); - - stats_prev_global_read_bucket = global_read_bucket; - stats_prev_global_write_bucket = global_write_bucket; - - current_millisecond = now; /* remember what time it is, for next time */ -} - -#ifndef _WIN32 -/** Called when a possibly ignorable libevent error occurs; ensures that we - * don't get into an infinite loop by ignoring too many errors from - * libevent. */ -static int -got_libevent_error(void) -{ - if (++n_libevent_errors > 8) { - log_err(LD_NET, "Too many libevent errors in one second; dying"); - return -1; - } - return 0; -} -#endif /* !defined(_WIN32) */ - #define UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST (6*60*60) /** Called when our IP address seems to have changed. <b>at_interface</b> @@ -2464,9 +2661,9 @@ ip_address_changed(int at_interface) } } else { if (server) { - if (stats_n_seconds_working > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST) + if (get_uptime() > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST) reset_bandwidth_test(); - stats_n_seconds_working = 0; + reset_uptime(); router_reset_reachability(); } } @@ -2587,6 +2784,20 @@ do_hup(void) return 0; } +/** Initialize some mainloop_event_t objects that we require. */ +STATIC void +initialize_mainloop_events(void) +{ + if (!schedule_active_linked_connections_event) { + schedule_active_linked_connections_event = + mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL); + } + if (!postloop_cleanup_ev) { + postloop_cleanup_ev = + mainloop_event_postloop_new(postloop_cleanup_cb, NULL); + } +} + /** Tor main loop. */ int do_main_loop(void) @@ -2600,6 +2811,8 @@ do_main_loop(void) initialize_periodic_events(); } + initialize_mainloop_events(); + /* initialize dns resolve map, spawn workers if needed */ if (dns_init() < 0) { if (get_options()->ServerDNSAllowBrokenConfig) @@ -2626,8 +2839,6 @@ do_main_loop(void) /* Set up our buckets */ connection_bucket_init(); - stats_prev_global_read_bucket = global_read_bucket; - stats_prev_global_write_bucket = global_write_bucket; /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); @@ -2691,17 +2902,7 @@ do_main_loop(void) } /* set up once-a-second callback. */ - if (! second_timer) { - struct timeval one_second; - one_second.tv_sec = 1; - one_second.tv_usec = 0; - - second_timer = periodic_timer_new(tor_libevent_get_base(), - &one_second, - second_elapsed_callback, - NULL); - tor_assert(second_timer); - } + reschedule_per_second_timer(); #ifdef HAVE_SYSTEMD_209 uint64_t watchdog_delay; @@ -2725,20 +2926,6 @@ do_main_loop(void) } #endif /* defined(HAVE_SYSTEMD_209) */ - if (!refill_timer) { - struct timeval refill_interval; - int msecs = get_options()->TokenBucketRefillInterval; - - refill_interval.tv_sec = msecs/1000; - refill_interval.tv_usec = (msecs%1000)*1000; - - refill_timer = periodic_timer_new(tor_libevent_get_base(), - &refill_interval, - refill_callback, - NULL); - tor_assert(refill_timer); - } - #ifdef HAVE_SYSTEMD { const int r = sd_notify(0, "READY=1"); @@ -2784,6 +2971,11 @@ do_main_loop(void) return run_main_loop_until_done(); } +#ifndef _WIN32 +/** Rate-limiter for EINVAL-type libevent warnings. */ +static ratelim_t libevent_error_ratelim = RATELIM_INIT(10); +#endif + /** * Run the main loop a single time. Return 0 for "exit"; -1 for "exit with * error", and 1 for "run this again." @@ -2804,17 +2996,12 @@ run_main_loop_once(void) errno = 0; #endif - /* All active linked conns should get their read events activated, - * so that libevent knows to run their callbacks. */ - SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, - event_active(conn->read_event, EV_READ, 1)); - if (get_options()->MainloopStats) { /* We always enforce that EVLOOP_ONCE is passed to event_base_loop() if we * are collecting main loop statistics. */ called_loop_once = 1; } else { - called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0; + called_loop_once = 0; } /* Make sure we know (about) what time it is. */ @@ -2824,8 +3011,8 @@ run_main_loop_once(void) * an event, or the second ends, or until we have some active linked * connections to trigger events for. Libevent will wait till one * of these happens, then run all the appropriate callbacks. */ - loop_result = event_base_loop(tor_libevent_get_base(), - called_loop_once ? EVLOOP_ONCE : 0); + loop_result = tor_libevent_run_event_loop(tor_libevent_get_base(), + called_loop_once); if (get_options()->MainloopStats) { /* Update our main loop counters. */ @@ -2854,9 +3041,12 @@ run_main_loop_once(void) return -1; #ifndef _WIN32 } else if (e == EINVAL) { - log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?"); - if (got_libevent_error()) + log_fn_ratelim(&libevent_error_ratelim, LOG_WARN, LD_NET, + "EINVAL from libevent: should you upgrade libevent?"); + if (libevent_error_ratelim.n_calls_since_last_time > 8) { + log_err(LD_NET, "Too many libevent errors, too fast: dying"); return -1; + } #endif /* !defined(_WIN32) */ } else { tor_assert_nonfatal_once(! ERRNO_IS_EINPROGRESS(e)); @@ -2870,19 +3060,6 @@ run_main_loop_once(void) if (main_loop_should_exit) return 0; - /* And here is where we put callbacks that happen "every time the event loop - * runs." They must be very fast, or else the whole Tor process will get - * slowed down. - * - * Note that this gets called once per libevent loop, which will make it - * happen once per group of events that fire, or once per second. */ - - /* If there are any pending client connections, try attaching them to - * circuits (if we can.) This will be pretty fast if nothing new is - * pending. - */ - connection_ap_attach_pending(0); - return 1; } @@ -2915,6 +3092,7 @@ signal_callback(evutil_socket_t fd, short events, void *arg) (void)fd; (void)events; + update_current_time(time(NULL)); process_signal(sig); } @@ -2977,10 +3155,20 @@ process_signal(int sig) case SIGNEWNYM: { time_t now = time(NULL); if (time_of_last_signewnym + MAX_SIGNEWNYM_RATE > now) { - signewnym_is_pending = 1; + const time_t delay_sec = + time_of_last_signewnym + MAX_SIGNEWNYM_RATE - now; + if (! signewnym_is_pending) { + signewnym_is_pending = 1; + if (!handle_deferred_signewnym_ev) { + handle_deferred_signewnym_ev = + mainloop_event_postloop_new(handle_deferred_signewnym_cb, NULL); + } + const struct timeval delay_tv = { delay_sec, 0 }; + mainloop_event_schedule(handle_deferred_signewnym_ev, &delay_tv); + } log_notice(LD_CONTROL, - "Rate limiting NEWNYM request: delaying by %d second(s)", - (int)(MAX_SIGNEWNYM_RATE+time_of_last_signewnym-now)); + "Rate limiting NEWNYM request: delaying by %d second(s)", + (int)(delay_sec)); } else { signewnym_impl(now); } @@ -3004,6 +3192,13 @@ get_uptime,(void)) return stats_n_seconds_working; } +/** Reset Tor's uptime. */ +MOCK_IMPL(void, +reset_uptime,(void)) +{ + stats_n_seconds_working = 0; +} + /** * Write current memory usage information to the log. */ @@ -3047,13 +3242,13 @@ dumpstats(int severity) i, (int)connection_get_inbuf_len(conn), (int)buf_allocation(conn->inbuf), - (int)(now - conn->timestamp_lastread)); + (int)(now - conn->timestamp_last_read_allowed)); tor_log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on outbuf " "(len %d, last written %d secs ago)",i, (int)connection_get_outbuf_len(conn), (int)buf_allocation(conn->outbuf), - (int)(now - conn->timestamp_lastwritten)); + (int)(now - conn->timestamp_last_write_allowed)); if (conn->type == CONN_TYPE_OR) { or_connection_t *or_conn = TO_OR_CONN(conn); if (or_conn->tls) { @@ -3323,14 +3518,12 @@ tor_init(int argc, char *argv[]) if (strstr(version, "alpha") || strstr(version, "beta")) log_notice(LD_GENERAL, "This version is not a stable Tor release. " "Expect more bugs than usual."); + + tor_compress_log_init_warnings(); } #ifdef HAVE_RUST - char *rust_str = rust_welcome_string(); - if (rust_str != NULL && strlen(rust_str) > 0) { - log_notice(LD_GENERAL, "%s", rust_str); - } - tor_free(rust_str); + rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ if (network_init()<0) { @@ -3488,6 +3681,9 @@ tor_free_all(int postfork) consdiffmgr_free_all(); hs_free_all(); dos_free_all(); + circuitmux_ewma_free_all(); + accounting_free_all(); + if (!postfork) { config_free_all(); or_state_free_all(); @@ -3508,9 +3704,35 @@ tor_free_all(int postfork) smartlist_free(active_linked_connection_lst); periodic_timer_free(second_timer); teardown_periodic_events(); - periodic_timer_free(refill_timer); tor_event_free(shutdown_did_not_work_event); tor_event_free(initialize_periodic_events_event); + mainloop_event_free(directory_all_unreachable_cb_event); + mainloop_event_free(schedule_active_linked_connections_event); + mainloop_event_free(postloop_cleanup_ev); + mainloop_event_free(handle_deferred_signewnym_ev); + +#ifdef HAVE_SYSTEMD_209 + periodic_timer_free(systemd_watchdog_timer); +#endif + + memset(&global_bucket, 0, sizeof(global_bucket)); + memset(&global_relayed_bucket, 0, sizeof(global_relayed_bucket)); + stats_n_bytes_read = stats_n_bytes_written = 0; + time_of_process_start = 0; + time_of_last_signewnym = 0; + signewnym_is_pending = 0; + newnym_epoch = 0; + called_loop_once = 0; + main_loop_should_exit = 0; + main_loop_exit_value = 0; + can_complete_circuits = 0; + quiet_level = 0; + should_init_bridge_stats = 1; + dns_honesty_first_time = 1; + heartbeat_callback_first_time = 1; + current_second = 0; + memset(¤t_second_last_changed, 0, + sizeof(current_second_last_changed)); if (!postfork) { release_lockfile(); diff --git a/src/or/main.h b/src/or/main.h index c49d216f4e..9dbbc6e5ee 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -28,6 +28,7 @@ int connection_is_on_closeable_list(connection_t *conn); MOCK_DECL(smartlist_t *, get_connection_array, (void)); MOCK_DECL(uint64_t,get_bytes_read,(void)); MOCK_DECL(uint64_t,get_bytes_written,(void)); +void stats_increment_bytes_read_and_written(uint64_t r, uint64_t w); /** Bitmask for events that we can turn on and off with * connection_watch_events. */ @@ -45,7 +46,6 @@ int connection_is_writing(connection_t *conn); MOCK_DECL(void,connection_stop_writing,(connection_t *conn)); MOCK_DECL(void,connection_start_writing,(connection_t *conn)); -void tell_event_loop_to_run_external_code(void); void tor_shutdown_event_loop_and_exit(int exitcode); int tor_event_loop_shutdown_is_pending(void); @@ -61,8 +61,15 @@ void dns_servers_relaunch_checks(void); void reset_all_main_loop_timers(void); void reschedule_descriptor_update_check(void); void reschedule_directory_downloads(void); +void reschedule_or_state_save(void); +void reschedule_dirvote(const or_options_t *options); +void mainloop_schedule_postloop_cleanup(void); +void rescan_periodic_events(const or_options_t *options); + +void update_current_time(time_t now); MOCK_DECL(long,get_uptime,(void)); +MOCK_DECL(void,reset_uptime,(void)); unsigned get_signewnym_epoch(void); @@ -86,21 +93,27 @@ uint64_t get_main_loop_success_count(void); uint64_t get_main_loop_error_count(void); uint64_t get_main_loop_idle_count(void); +void periodic_events_on_new_options(const or_options_t *options); +void reschedule_per_second_timer(void); + extern time_t time_of_process_start; -extern long stats_n_seconds_working; extern int quiet_level; -extern int global_read_bucket; -extern int global_write_bucket; -extern int global_relayed_read_bucket; -extern int global_relayed_write_bucket; +extern token_bucket_rw_t global_bucket; +extern token_bucket_rw_t global_relayed_bucket; #ifdef MAIN_PRIVATE STATIC void init_connection_lists(void); +STATIC void initialize_mainloop_events(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); +STATIC int get_my_roles(const or_options_t *options); #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; + +/* We need the periodic_event_item_t definition. */ +#include "periodic.h" +extern periodic_event_item_t periodic_events[]; #endif #endif /* defined(MAIN_PRIVATE) */ diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 040405555c..998eaf74e6 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -48,9 +48,10 @@ #include "connection_or.h" #include "consdiffmgr.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "dos.h" #include "entrynodes.h" #include "hibernate.h" @@ -64,10 +65,14 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" -#include "shared_random.h" #include "transports.h" #include "torcert.h" #include "channelpadding.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random.h" /** Most recently received and validated v3 "ns"-flavored consensus network * status. */ @@ -237,7 +242,7 @@ router_reload_consensus_networkstatus(void) s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); if (s) { if (networkstatus_set_current_consensus(s, flavor, - flags|NSSET_WAS_WAITING_FOR_CERTS, + flags | NSSET_WAS_WAITING_FOR_CERTS, NULL)) { log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " "from cache", flavor); @@ -365,9 +370,7 @@ networkstatus_vote_free_(networkstatus_t *ns) digestmap_free(ns->desc_digest_map, NULL); if (ns->sr_info.commits) { - SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c, - sr_commit_free(c)); - smartlist_free(ns->sr_info.commits); + dirvote_clear_commits(ns); } tor_free(ns->sr_info.previous_srv); tor_free(ns->sr_info.current_srv); @@ -391,6 +394,20 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +networkstatus_get_voter_sig_by_alg(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Check whether the signature <b>sig</b> is correctly signed with the * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on @@ -951,9 +968,12 @@ update_consensus_networkstatus_downloads(time_t now) continue; } - /* Check if we're waiting for certificates to download */ - if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) + /** Check if we're waiting for certificates to download. If we are, + * launch download for missing directory authority certificates. */ + if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) { + update_certificate_downloads(now); continue; + } /* Try the requested attempt */ log_info(LD_DIR, "Launching %s standard networkstatus consensus " @@ -1230,16 +1250,20 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out) return 0; } -/** Launch requests for networkstatus documents and authority certificates as - * appropriate. */ +/** Launch requests for networkstatus documents as appropriate. This is called + * when we retry all the connections on a SIGHUP and periodically by a Periodic + * event which checks whether we want to download any networkstatus documents. + */ void update_networkstatus_downloads(time_t now) { const or_options_t *options = get_options(); if (should_delay_dir_fetches(options, NULL)) return; + /** Launch a consensus download request, we will wait for the consensus to + * download and when it completes we will launch a certificate download + * request. */ update_consensus_networkstatus_downloads(now); - update_certificate_downloads(now); } /** Launch requests as appropriate for missing directory authority @@ -1521,7 +1545,7 @@ networkstatus_consensus_has_ipv6(const or_options_t* options) return cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS; } else { - return cons->consensus_method >= MIN_METHOD_FOR_A_LINES; + return 1; } } @@ -1667,23 +1691,6 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, #endif /* defined(TOR_UNIT_TESTS) */ /** - * Return true if any option is set in <b>options</b> to make us behave - * as a client. - * - * XXXX If we need this elsewhere at any point, we should make it nonstatic - * XXXX and move it into another file. - */ -static int -any_client_port_set(const or_options_t *options) -{ - return (options->SocksPort_set || - options->TransPort_set || - options->NATDPort_set || - options->DNSPort_set || - options->HTTPTunnelPort_set); -} - -/** * Helper for handle_missing_protocol_warning: handles either the * client case (if <b>is_client</b> is set) or the server case otherwise. */ @@ -1718,7 +1725,7 @@ handle_missing_protocol_warning(const networkstatus_t *c, const or_options_t *options) { const int is_server = server_mode(options); - const int is_client = any_client_port_set(options) || !is_server; + const int is_client = options_any_client_port_set(options) || !is_server; if (is_server) handle_missing_protocol_warning_impl(c, 0); @@ -1726,6 +1733,57 @@ handle_missing_protocol_warning(const networkstatus_t *c, handle_missing_protocol_warning_impl(c, 1); } +/** + * Check whether we received a consensus that appears to be coming + * from the future. Because we implicitly trust the directory + * authorities' idea of the current time, we produce a warning if we + * get an early consensus. + * + * If we got a consensus that is time stamped far in the past, that + * could simply have come from a stale cache. Possible ways to get a + * consensus from the future can include: + * + * - enough directory authorities have wrong clocks + * - directory authorities collude to produce misleading time stamps + * - our own clock is wrong (this is by far the most likely) + * + * We neglect highly improbable scenarios that involve actual time + * travel. + */ +STATIC void +warn_early_consensus(const networkstatus_t *c, const char *flavor, + time_t now) +{ + char tbuf[ISO_TIME_LEN+1]; + char dbuf[64]; + long delta = now - c->valid_after; + char *flavormsg = NULL; + +/** If a consensus appears more than this many seconds before it could + * possibly be a sufficiently-signed consensus, declare that our clock + * is skewed. */ +#define EARLY_CONSENSUS_NOTICE_SKEW 60 + + /* We assume that if a majority of dirauths have accurate clocks, + * the earliest that a dirauth with a skewed clock could possibly + * publish a sufficiently-signed consensus is (valid_after - + * dist_seconds). Before that time, the skewed dirauth would be + * unable to obtain enough authority signatures for the consensus to + * be valid. */ + if (now >= c->valid_after - c->dist_seconds - EARLY_CONSENSUS_NOTICE_SKEW) + return; + + format_iso_time(tbuf, c->valid_after); + format_time_interval(dbuf, sizeof(dbuf), delta); + log_warn(LD_GENERAL, "Our clock is %s behind the time published in the " + "consensus network status document (%s UTC). Tor needs an " + "accurate clock to work correctly. Please check your time and " + "date settings!", dbuf, tbuf); + tor_asprintf(&flavormsg, "%s flavor consensus", flavor); + clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS"); + tor_free(flavormsg); +} + /** Try to replace the current cached v3 networkstatus with the one in * <b>consensus</b>. If we don't have enough certificates to validate it, * store it in consensus_waiting_for_certs and launch a certificate fetch. @@ -1768,7 +1826,6 @@ networkstatus_set_current_consensus(const char *consensus, consensus_waiting_for_certs_t *waiting = NULL; time_t current_valid_after = 0; int free_consensus = 1; /* Free 'c' at the end of the function */ - int old_ewma_enabled; int checked_protocols_already = 0; if (flav < 0) { @@ -1834,17 +1891,9 @@ networkstatus_set_current_consensus(const char *consensus, current_valid_after = current_md_consensus->valid_after; } } else { - cached_dir_t *cur; - char buf[128]; - tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); - consensus_fname = get_cachedir_fname(buf); - tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); - unverified_fname = get_cachedir_fname(buf); - cur = dirserv_get_consensus(flavor); - if (cur) { - current_digests = &cur->digests; - current_valid_after = cur->published; - } + tor_assert_nonfatal_unreached(); + result = -2; + goto done; } if (current_digests && @@ -1917,6 +1966,15 @@ networkstatus_set_current_consensus(const char *consensus, } } + /* Signatures from the consensus are verified */ + if (from_cache && was_waiting_for_certs) { + /* We check if the consensus is loaded from disk cache and that it + * it is an unverified consensus. If it is unverified, rename it to + * cached-*-consensus since it has been verified. */ + log_info(LD_DIR, "Unverified consensus signatures verified."); + tor_rename(unverified_fname, consensus_fname); + } + if (!from_cache && flav == usable_consensus_flavor()) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); @@ -1985,24 +2043,16 @@ networkstatus_set_current_consensus(const char *consensus, * the first thing we need to do is recalculate the voting schedule static * object so we can use the timings in there needed by some subsystems * such as hidden service and shared random. */ - dirvote_recalculate_timing(options, now); + voting_schedule_recalculate_timing(options, now); + reschedule_dirvote(options); nodelist_set_consensus(c); /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); - /* Update ewma and adjust policy if needed; first cache the old value */ - old_ewma_enabled = cell_ewma_enabled(); /* Change the cell EWMA settings */ - cell_ewma_set_scale_factor(options, c); - /* If we just enabled ewma, set the cmux policy on all active channels */ - if (cell_ewma_enabled() && !old_ewma_enabled) { - channel_set_cmux_policy_everywhere(&ewma_policy); - } else if (!cell_ewma_enabled() && old_ewma_enabled) { - /* Turn it off everywhere */ - channel_set_cmux_policy_everywhere(NULL); - } + cmux_ewma_set_options(options, c); /* XXXX this call might be unnecessary here: can changing the * current consensus really alter our view of any OR's rate limits? */ @@ -2036,25 +2086,7 @@ networkstatus_set_current_consensus(const char *consensus, write_str_to_file(consensus_fname, consensus, 0); } -/** If a consensus appears more than this many seconds before its declared - * valid-after time, declare that our clock is skewed. */ -#define EARLY_CONSENSUS_NOTICE_SKEW 60 - - if (now < c->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) { - char tbuf[ISO_TIME_LEN+1]; - char dbuf[64]; - long delta = now - c->valid_after; - char *flavormsg = NULL; - format_iso_time(tbuf, c->valid_after); - format_time_interval(dbuf, sizeof(dbuf), delta); - log_warn(LD_GENERAL, "Our clock is %s behind the time published in the " - "consensus network status document (%s UTC). Tor needs an " - "accurate clock to work correctly. Please check your time and " - "date settings!", dbuf, tbuf); - tor_asprintf(&flavormsg, "%s flavor consensus", flavor); - clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS"); - tor_free(flavormsg); - } + warn_early_consensus(c, flavor, now); /* We got a new consesus. Reset our md fetch fail cache */ microdesc_reset_outdated_dirservers_list(); @@ -2634,6 +2666,25 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, return 0; } +/** Release all storage held in <b>s</b>. */ +void +ns_detached_signatures_free_(ns_detached_signatures_t *s) +{ + if (!s) + return; + if (s->signatures) { + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, tor_free_); + } + + tor_free(s); +} + /** Free all storage held locally in this module. */ void networkstatus_free_all(void) diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 1851a55e82..94f85c3c29 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -24,9 +24,16 @@ void routerstatus_free_(routerstatus_t *rs); void networkstatus_vote_free_(networkstatus_t *ns); #define networkstatus_vote_free(ns) \ FREE_AND_NULL(networkstatus_t, networkstatus_vote_free_, (ns)) +void ns_detached_signatures_free_(ns_detached_signatures_t *s); +#define ns_detached_signatures_free(s) \ + FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s)) networkstatus_voter_info_t *networkstatus_get_voter_by_id( networkstatus_t *vote, const char *identity); +document_signature_t *networkstatus_get_voter_sig_by_alg( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + int networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn); int networkstatus_check_document_signature(const networkstatus_t *consensus, @@ -144,6 +151,8 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs); #ifdef TOR_UNIT_TESTS STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c, const char *flavor); +STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor, + time_t now); extern networkstatus_t *current_ns_consensus; extern networkstatus_t *current_md_consensus; #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 82e0926289..032e8d669f 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -68,6 +68,8 @@ #include <string.h> +#include "dirauth/mode.h" + static void nodelist_drop_node(node_t *node, int remove_from_ht); #define node_free(val) \ FREE_AND_NULL(node_t, node_free_, (val)) @@ -76,10 +78,17 @@ static void node_free_(node_t *node); /** count_usable_descriptors counts descriptors with these flag(s) */ typedef enum { - /* All descriptors regardless of flags */ - USABLE_DESCRIPTOR_ALL = 0, - /* Only descriptors with the Exit flag */ - USABLE_DESCRIPTOR_EXIT_ONLY = 1 + /* All descriptors regardless of flags or exit policies */ + USABLE_DESCRIPTOR_ALL = 0U, + /* Only count descriptors with an exit policy that allows at least one port + */ + USABLE_DESCRIPTOR_EXIT_POLICY = 1U << 0, + /* Only count descriptors for relays that have the exit flag in the + * consensus */ + USABLE_DESCRIPTOR_EXIT_FLAG = 1U << 1, + /* Only count descriptors for relays that have the policy and the flag */ + USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG = (USABLE_DESCRIPTOR_EXIT_POLICY | + USABLE_DESCRIPTOR_EXIT_FLAG) } usable_descriptor_t; static void count_usable_descriptors(int *num_present, int *num_usable, @@ -113,6 +122,11 @@ typedef struct nodelist_t { /* Set of addresses that belong to nodes we believe in. */ address_set_t *node_addrs; + + /* The valid-after time of the last live consensus that initialized the + * nodelist. We use this to detect outdated nodelists that need to be + * rebuilt using a newer consensus. */ + time_t live_consensus_valid_after; } nodelist_t; static inline unsigned int @@ -227,7 +241,6 @@ node_get_or_create(const char *identity_digest) smartlist_add(the_nodelist->nodes, node); node->nodelist_idx = smartlist_len(the_nodelist->nodes) - 1; - node->hsdir_index = tor_malloc_zero(sizeof(hsdir_index_t)); node->country = -1; @@ -379,26 +392,26 @@ node_set_hsdir_index(node_t *node, const networkstatus_t *ns) /* Build the fetch index. */ hs_build_hsdir_index(node_identity_pk, fetch_srv, fetch_tp, - node->hsdir_index->fetch); + node->hsdir_index.fetch); /* If we are in the time segment between SRV#N and TP#N, the fetch index is the same as the first store index */ if (!hs_in_period_between_tp_and_srv(ns, now)) { - memcpy(node->hsdir_index->store_first, node->hsdir_index->fetch, - sizeof(node->hsdir_index->store_first)); + memcpy(node->hsdir_index.store_first, node->hsdir_index.fetch, + sizeof(node->hsdir_index.store_first)); } else { hs_build_hsdir_index(node_identity_pk, store_first_srv, store_first_tp, - node->hsdir_index->store_first); + node->hsdir_index.store_first); } /* If we are in the time segment between TP#N and SRV#N+1, the fetch index is the same as the second store index */ if (hs_in_period_between_tp_and_srv(ns, now)) { - memcpy(node->hsdir_index->store_second, node->hsdir_index->fetch, - sizeof(node->hsdir_index->store_second)); + memcpy(node->hsdir_index.store_second, node->hsdir_index.fetch, + sizeof(node->hsdir_index.store_second)); } else { hs_build_hsdir_index(node_identity_pk, store_second_srv, store_second_tp, - node->hsdir_index->store_second); + node->hsdir_index.store_second); } done: @@ -658,6 +671,12 @@ nodelist_set_consensus(networkstatus_t *ns) } } SMARTLIST_FOREACH_END(node); } + + /* If the consensus is live, note down the consensus valid-after that formed + * the nodelist. */ + if (networkstatus_is_live(ns, approx_time())) { + the_nodelist->live_consensus_valid_after = ns->valid_after; + } } /** Helper: return true iff a node has a usable amount of information*/ @@ -749,7 +768,6 @@ node_free_(node_t *node) if (node->md) node->md->held_by_nodes--; tor_assert(node->nodelist_idx == -1); - tor_free(node->hsdir_index); tor_free(node); } @@ -883,6 +901,25 @@ nodelist_assert_ok(void) digestmap_free(dm, NULL); } +/** Ensure that the nodelist has been created with the most recent consensus. + * If that's not the case, make it so. */ +void +nodelist_ensure_freshness(networkstatus_t *ns) +{ + tor_assert(ns); + + /* We don't even have a nodelist: this is a NOP. */ + if (!the_nodelist) { + return; + } + + if (the_nodelist->live_consensus_valid_after != ns->valid_after) { + log_info(LD_GENERAL, "Nodelist was not fresh: rebuilding. (%d / %d)", + (int) the_nodelist->live_consensus_valid_after, + (int) ns->valid_after); + nodelist_set_consensus(ns); + } +} /** Return a list of a node_t * for every node we know about. The caller * MUST NOT modify the list. (You can set and clear flags in the nodes if * you must, but you must not add or remove nodes.) */ @@ -1116,6 +1153,11 @@ node_supports_v3_rendezvous_point(const node_t *node) { tor_assert(node); + /* We can't use a v3 rendezvous point without the curve25519 onion pk. */ + if (!node_get_curve25519_onion_key(node)) { + return 0; + } + return node_get_protover_summary_flags(node)->supports_v3_rendezvous_point; } @@ -1515,9 +1557,11 @@ node_ipv6_or_preferred(const node_t *node) /* XX/teor - node->ipv6_preferred is set from * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded. */ + node_get_prim_orport(node, &ipv4_addr); if (!fascist_firewall_use_ipv6(options)) { return 0; - } else if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) { + } else if (node->ipv6_preferred || + !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) { return node_has_ipv6_orport(node); } return 0; @@ -1528,14 +1572,12 @@ node_ipv6_or_preferred(const node_t *node) if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \ tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \ (ap_out)->port = (r)->port_field; \ - return 0; \ } \ STMT_END -/** Copy the primary (IPv4) OR port (IP address and TCP port) for - * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and - * port was copied, else return non-zero.*/ -int +/** Copy the primary (IPv4) OR port (IP address and TCP port) for <b>node</b> + * into *<b>ap_out</b>. */ +void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); @@ -1552,8 +1594,6 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) RETURN_IPV4_AP(node->ri, or_port, ap_out); RETURN_IPV4_AP(node->rs, or_port, ap_out); /* Microdescriptors only have an IPv6 address */ - - return -1; } /** Copy the preferred OR port (IP address and TCP port) for @@ -1578,6 +1618,7 @@ node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); tor_assert(ap_out); + memset(ap_out, 0, sizeof(*ap_out)); /* Check ri first, because rewrite_node_address_for_bridge() updates * node->ri with the configured bridge address. @@ -1625,32 +1666,35 @@ node_ipv6_dir_preferred(const node_t *node) * so we can't use it to determine DirPort IPv6 preference. * This means that bridge clients will use IPv4 DirPorts by default. */ + node_get_prim_dirport(node, &ipv4_addr); if (!fascist_firewall_use_ipv6(options)) { return 0; - } else if (node_get_prim_dirport(node, &ipv4_addr) + } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0) || fascist_firewall_prefer_ipv6_dirport(get_options())) { return node_has_ipv6_dirport(node); } return 0; } -/** Copy the primary (IPv4) Dir port (IP address and TCP port) for - * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and - * port was copied, else return non-zero.*/ -int +/** Copy the primary (IPv4) Dir port (IP address and TCP port) for <b>node</b> + * into *<b>ap_out</b>. */ +void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); tor_assert(ap_out); + /* Clear the address, as a safety precaution if calling functions ignore the + * return value */ + tor_addr_make_null(&ap_out->addr, AF_INET); + ap_out->port = 0; + /* Check ri first, because rewrite_node_address_for_bridge() updates * node->ri with the configured bridge address. */ RETURN_IPV4_AP(node->ri, dir_port, ap_out); RETURN_IPV4_AP(node->rs, dir_port, ap_out); /* Microdescriptors only have an IPv6 address */ - - return -1; } #undef RETURN_IPV4_AP @@ -2107,8 +2151,11 @@ get_dir_info_status_string(void) * *<b>num_present</b>). * * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>. - * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes - * with the Exit flag. + * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_POLICY, only consider nodes + * present if they have an exit policy that accepts at least one port. + * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_FLAG, only consider nodes + * usable if they have the exit flag in the consensus. + * * If *<b>descs_out</b> is present, add a node_t for each usable descriptor * to it. */ @@ -2129,7 +2176,7 @@ count_usable_descriptors(int *num_present, int *num_usable, if (!node) continue; /* This would be a bug: every entry in the consensus is * supposed to have a node. */ - if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit) + if ((exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) && ! rs->is_exit) continue; if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1)) continue; @@ -2142,7 +2189,14 @@ count_usable_descriptors(int *num_present, int *num_usable, else present = NULL != router_get_by_descriptor_digest(digest); if (present) { - /* we have the descriptor listed in the consensus. */ + /* Do the policy check last, because it requires a descriptor, + * and is potentially expensive */ + if ((exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) && + node_exit_policy_rejects_all(node)) { + continue; + } + /* we have the descriptor listed in the consensus, and it + * satisfies our exit constraints (if any) */ ++*num_present; } if (descs_out) @@ -2151,10 +2205,17 @@ count_usable_descriptors(int *num_present, int *num_usable, } SMARTLIST_FOREACH_END(rs); - log_debug(LD_DIR, "%d usable, %d present (%s%s).", + log_debug(LD_DIR, "%d usable, %d present (%s%s%s%s%s).", *num_usable, *num_present, md ? "microdesc" : "desc", - exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s"); + (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ? + " exit" : "s", + (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) ? + " policies" : "" , + (exit_only == USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ? + " and" : "" , + (exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) ? + " flags" : "" ); } /** Return an estimate of which fraction of usable paths through the Tor @@ -2189,9 +2250,20 @@ compute_frac_paths_available(const networkstatus_t *consensus, count_usable_descriptors(num_present_out, num_usable_out, mid, consensus, now, NULL, USABLE_DESCRIPTOR_ALL); + log_debug(LD_NET, + "%s: %d present, %d usable", + "mid", + np, + nu); + if (options->EntryNodes) { count_usable_descriptors(&np, &nu, guards, consensus, now, options->EntryNodes, USABLE_DESCRIPTOR_ALL); + log_debug(LD_NET, + "%s: %d present, %d usable", + "guard", + np, + nu); } else { SMARTLIST_FOREACH(mid, const node_t *, node, { if (authdir) { @@ -2202,42 +2274,45 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_add(guards, (node_t*)node); } }); + log_debug(LD_NET, + "%s: %d possible", + "guard", + smartlist_len(guards)); } - /* All nodes with exit flag - * If we're in a network with TestingDirAuthVoteExit set, - * this can cause false positives on have_consensus_path, - * incorrectly setting it to CONSENSUS_PATH_EXIT. This is - * an unavoidable feature of forcing authorities to declare - * certain nodes as exits. - */ + /* All nodes with exit policy and flag */ count_usable_descriptors(&np, &nu, exits, consensus, now, - NULL, USABLE_DESCRIPTOR_EXIT_ONLY); + NULL, USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG); log_debug(LD_NET, "%s: %d present, %d usable", "exits", np, nu); - /* We need at least 1 exit present in the consensus to consider + /* We need at least 1 exit (flag and policy) in the consensus to consider * building exit paths */ /* Update our understanding of whether the consensus has exits */ consensus_path_type_t old_have_consensus_path = have_consensus_path; - have_consensus_path = ((nu > 0) ? + have_consensus_path = ((np > 0) ? CONSENSUS_PATH_EXIT : CONSENSUS_PATH_INTERNAL); - if (have_consensus_path == CONSENSUS_PATH_INTERNAL - && old_have_consensus_path != have_consensus_path) { - log_notice(LD_NET, - "The current consensus has no exit nodes. " - "Tor can only build internal paths, " - "such as paths to hidden services."); - - /* However, exit nodes can reachability self-test using this consensus, - * join the network, and appear in a later consensus. This will allow - * the network to build exit paths, such as paths for world wide web - * browsing (as distinct from hidden service web browsing). */ + if (old_have_consensus_path != have_consensus_path) { + if (have_consensus_path == CONSENSUS_PATH_INTERNAL) { + log_notice(LD_NET, + "The current consensus has no exit nodes. " + "Tor can only build internal paths, " + "such as paths to onion services."); + + /* However, exit nodes can reachability self-test using this consensus, + * join the network, and appear in a later consensus. This will allow + * the network to build exit paths, such as paths for world wide web + * browsing (as distinct from hidden service web browsing). */ + } else if (old_have_consensus_path == CONSENSUS_PATH_INTERNAL) { + log_notice(LD_NET, + "The current consensus contains exit nodes. " + "Tor can build exit and internal paths."); + } } f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD); @@ -2259,43 +2334,28 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *myexits= smartlist_new(); smartlist_t *myexits_unflagged = smartlist_new(); - /* All nodes with exit flag in ExitNodes option */ + /* All nodes with exit policy and flag in ExitNodes option */ count_usable_descriptors(&np, &nu, myexits, consensus, now, - options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY); + options->ExitNodes, + USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG); log_debug(LD_NET, "%s: %d present, %d usable", "myexits", np, nu); - /* Now compute the nodes in the ExitNodes option where which we don't know - * what their exit policy is, or we know it permits something. */ + /* Now compute the nodes in the ExitNodes option where we know their exit + * policy permits something. */ count_usable_descriptors(&np, &nu, myexits_unflagged, consensus, now, - options->ExitNodes, USABLE_DESCRIPTOR_ALL); + options->ExitNodes, + USABLE_DESCRIPTOR_EXIT_POLICY); log_debug(LD_NET, "%s: %d present, %d usable", "myexits_unflagged (initial)", np, nu); - SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { - if (node_has_preferred_descriptor(node, 0) && - node_exit_policy_rejects_all(node)) { - SMARTLIST_DEL_CURRENT(myexits_unflagged, node); - /* this node is not actually an exit */ - np--; - /* this node is unusable as an exit */ - nu--; - } - } SMARTLIST_FOREACH_END(node); - - log_debug(LD_NET, - "%s: %d present, %d usable", - "myexits_unflagged (final)", - np, - nu); - f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); f_myexit_unflagged= frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); @@ -2336,14 +2396,14 @@ compute_frac_paths_available(const networkstatus_t *consensus, tor_asprintf(status_out, "%d%% of guards bw, " "%d%% of midpoint bw, and " - "%d%% of exit bw%s = " + "%d%% of %s = " "%d%% of path bw", (int)(f_guard*100), (int)(f_mid*100), (int)(f_exit*100), (router_have_consensus_path() == CONSENSUS_PATH_EXIT ? - "" : - " (no exits in consensus)"), + "exit bw" : + "end bw (no exits in consensus)"), (int)(f_path*100)); return f_path; diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 00f12ca1e4..dbe9ad18ff 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -29,6 +29,7 @@ const node_t *node_get_by_hex_id(const char *identity_digest, node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out); node_t *nodelist_add_microdesc(microdesc_t *md); void nodelist_set_consensus(networkstatus_t *ns); +void nodelist_ensure_freshness(networkstatus_t *ns); int nodelist_probably_contains_address(const tor_addr_t *addr); void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md); @@ -79,11 +80,11 @@ int node_has_ipv6_dirport(const node_t *node); /* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */ #define node_ipv6_preferred(node) node_ipv6_or_preferred(node) int node_ipv6_or_preferred(const node_t *node); -int node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out); int node_ipv6_dir_preferred(const node_t *node); -int node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out); int node_has_curve25519_onion_key(const node_t *node); diff --git a/src/or/ntmain.c b/src/or/ntmain.c index ebbe0018bd..e9a299807a 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -24,8 +24,6 @@ #include "main.h" #include "ntmain.h" -#include <event2/event.h> - #include <windows.h> #define GENSRV_SERVICENAME "tor" #define GENSRV_DISPLAYNAME "Tor Win32 Service" @@ -245,7 +243,8 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_base_loopexit(tor_libevent_get_base(), &exit_now); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), + &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); diff --git a/src/or/onion.c b/src/or/onion.c index bd80c2f503..829be12bae 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -67,6 +67,7 @@ #include "circuitlist.h" #include "config.h" #include "cpuworker.h" +#include "crypto_util.h" #include "networkstatus.h" #include "onion.h" #include "onion_fast.h" @@ -521,6 +522,11 @@ onion_skin_create(int type, return r; } +/* This is the maximum value for keys_out_len passed to + * onion_skin_server_handshake, plus 16. We can make it bigger if needed: + * It just defines how many bytes to stack-allocate. */ +#define MAX_KEYS_TMP_LEN 128 + /** Perform the second (server-side) step of a circuit-creation handshake of * type <b>type</b>, responding to the client request in <b>onion_skin</b> * using the keys in <b>keys</b>. On success, write our response into @@ -563,20 +569,21 @@ onion_skin_server_handshake(int type, return -1; { size_t keys_tmp_len = keys_out_len + DIGEST_LEN; - uint8_t *keys_tmp = tor_malloc(keys_out_len + DIGEST_LEN); + tor_assert(keys_tmp_len <= MAX_KEYS_TMP_LEN); + uint8_t keys_tmp[MAX_KEYS_TMP_LEN]; if (onion_skin_ntor_server_handshake( onion_skin, keys->curve25519_key_map, keys->junk_keypair, keys->my_identity, reply_out, keys_tmp, keys_tmp_len)<0) { - tor_free(keys_tmp); + /* no need to memwipe here, since the output will never be used */ return -1; } + memcpy(keys_out, keys_tmp, keys_out_len); memcpy(rend_nonce_out, keys_tmp+keys_out_len, DIGEST_LEN); - memwipe(keys_tmp, 0, keys_tmp_len); - tor_free(keys_tmp); + memwipe(keys_tmp, 0, sizeof(keys_tmp)); r = NTOR_REPLY_LEN; } break; diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index de9103b1f5..9f9b2199d4 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -29,6 +29,8 @@ #include "or.h" #include "onion_fast.h" +#include "crypto_rand.h" +#include "crypto_util.h" /** Release all state held in <b>victim</b>. */ void diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index b167cb61fb..02d43cb722 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -22,6 +22,8 @@ #define ONION_NTOR_PRIVATE #include "crypto.h" +#include "crypto_digest.h" +#include "crypto_util.h" #include "onion_ntor.h" #include "torlog.h" #include "util.h" diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index c71fa236ed..44737034f4 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -29,6 +29,8 @@ #include "or.h" #include "config.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "onion_tap.h" #include "rephist.h" diff --git a/src/or/or.h b/src/or/or.h index 3c0c2ad613..98b7fc9772 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -80,6 +80,7 @@ #include "crypto_curve25519.h" #include "crypto_ed25519.h" #include "tor_queue.h" +#include "token_bucket.h" #include "util_format.h" #include "hs_circuitmap.h" @@ -727,8 +728,8 @@ typedef enum { /** Catch-all "other" reason for closing origin circuits. */ #define END_CIRC_AT_ORIGIN -1 -/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt for - * documentation of these. */ +/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt + * section 5.4 for documentation of these. */ #define END_CIRC_REASON_MIN_ 0 #define END_CIRC_REASON_NONE 0 #define END_CIRC_REASON_TORPROTOCOL 1 @@ -893,8 +894,19 @@ rend_data_v2_t *TO_REND_DATA_V2(const rend_data_t *d) struct hs_ident_edge_conn_t; struct hs_ident_dir_conn_t; struct hs_ident_circuit_t; -/* Stub because we can't include hs_common.h. */ -struct hsdir_index_t; + +/* Hidden service directory index used in a node_t which is set once we set + * the consensus. */ +typedef struct hsdir_index_t { + /* HSDir index to use when fetching a descriptor. */ + uint8_t fetch[DIGEST256_LEN]; + + /* HSDir index used by services to store their first and second + * descriptor. The first descriptor is chronologically older than the second + * one and uses older TP and SRV values. */ + uint8_t store_first[DIGEST256_LEN]; + uint8_t store_second[DIGEST256_LEN]; +} hsdir_index_t; /** Time interval for tracking replays of DH public keys received in * INTRODUCE2 cells. Used only to avoid launching multiple @@ -917,6 +929,7 @@ typedef enum { /** Initial value on both sides of a stream transmission window when the * stream is initialized. Measured in cells. */ #define STREAMWINDOW_START 500 +#define STREAMWINDOW_START_MAX 500 /** Amount to increment a stream window when we get a stream SENDME. */ #define STREAMWINDOW_INCREMENT 50 @@ -1369,10 +1382,10 @@ typedef struct connection_t { * connection. */ size_t outbuf_flushlen; /**< How much data should we try to flush from the * outbuf? */ - time_t timestamp_lastread; /**< When was the last time libevent said we could - * read? */ - time_t timestamp_lastwritten; /**< When was the last time libevent said we - * could write? */ + time_t timestamp_last_read_allowed; /**< When was the last time libevent said + * we could read? */ + time_t timestamp_last_write_allowed; /**< When was the last time libevent + * said we could write? */ time_t timestamp_created; /**< When was this connection_t created? */ @@ -1660,20 +1673,8 @@ typedef struct or_connection_t { time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/ - /* bandwidth* and *_bucket only used by ORs in OPEN state: */ - int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ - int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ - int read_bucket; /**< When this hits 0, stop receiving. Every second we - * add 'bandwidthrate' to this, capping it at - * bandwidthburst. (OPEN ORs only) */ - int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ - - /** Last emptied read token bucket in msec since midnight; only used if - * TB_EMPTY events are enabled. */ - uint32_t read_emptied_time; - /** Last emptied write token bucket in msec since midnight; only used if - * TB_EMPTY events are enabled. */ - uint32_t write_emptied_time; + token_bucket_rw_t bucket; /**< Used for rate limiting when the connection is + * in state CONN_OPEN. */ /* * Count the number of bytes flushed out on this orconn, and the number of @@ -1744,6 +1745,27 @@ typedef struct edge_connection_t { uint64_t dirreq_id; } edge_connection_t; +/** + * Struct to track a connection that we closed that the other end + * still thinks is open. Exists in origin_circuit_t.half_streams until + * we get an end cell or a resolved cell for this stream id. + */ +typedef struct half_edge_t { + /** stream_id for the half-closed connection */ + streamid_t stream_id; + + /** How many sendme's can the other end still send, based on how + * much data we had sent at the time of close */ + int sendmes_pending; + + /** How much more data can the other end still send, based on + * our deliver window */ + int data_pending; + + /** Is there a connected cell pending? */ + int connected_pending : 1; +} half_edge_t; + /** Subtype of edge_connection_t for an "entry connection" -- that is, a SOCKS * connection, a DNS request, a TransPort connection or a NATD connection */ typedef struct entry_connection_t { @@ -2351,10 +2373,10 @@ typedef struct routerstatus_t { * If it's a descriptor, we only use the first DIGEST_LEN bytes. */ char descriptor_digest[DIGEST256_LEN]; uint32_t addr; /**< IPv4 address for this router, in host order. */ - uint16_t or_port; /**< OR port for this router. */ + uint16_t or_port; /**< IPv4 OR port for this router. */ uint16_t dir_port; /**< Directory port for this router. */ tor_addr_t ipv6_addr; /**< IPv6 address for this router. */ - uint16_t ipv6_orport; /**<IPV6 OR port for this router. */ + uint16_t ipv6_orport; /**< IPv6 OR port for this router. */ unsigned int is_authority:1; /**< True iff this router is an authority. */ unsigned int is_exit:1; /**< True iff this router is a good exit. */ unsigned int is_stable:1; /**< True iff this router stays up a long time. */ @@ -2572,7 +2594,7 @@ typedef struct node_t { /* Hidden service directory index data. This is used by a service or client * in order to know what's the hs directory index for this node at the time * the consensus is set. */ - struct hsdir_index_t *hsdir_index; + struct hsdir_index_t hsdir_index; } node_t; /** Linked list of microdesc hash lines for a single router in a directory @@ -2907,11 +2929,7 @@ typedef struct { } u; } onion_handshake_state_t; -/** Holds accounting information for a single step in the layered encryption - * performed by a circuit. Used only at the client edge of a circuit. */ -typedef struct crypt_path_t { - uint32_t magic; - +typedef struct relay_crypto_t { /* crypto environments */ /** Encryption key and counter for cells heading towards the OR at this * step. */ @@ -2925,6 +2943,17 @@ typedef struct crypt_path_t { /** Digest state for cells heading away from the OR at this step. */ crypto_digest_t *b_digest; +} relay_crypto_t; + +/** Holds accounting information for a single step in the layered encryption + * performed by a circuit. Used only at the client edge of a circuit. */ +typedef struct crypt_path_t { + uint32_t magic; + + /** Cryptographic state used for encrypting and authenticating relay + * cells to and from this hop. */ + relay_crypto_t crypto; + /** Current state of the handshake as performed with the OR at this * step. */ onion_handshake_state_t handshake_state; @@ -3170,15 +3199,6 @@ typedef struct circuit_t { /** Index in smartlist of all circuits (global_circuitlist). */ int global_circuitlist_idx; - /** Next circuit in the doubly-linked ring of circuits waiting to add - * cells to n_conn. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *next_active_on_n_chan; - /** Previous circuit in the doubly-linked ring of circuits waiting to add - * cells to n_conn. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *prev_active_on_n_chan; - /** Various statistics about cells being added to or removed from this * circuit's queues; used only if CELL_STATS events are enabled and * cleared after being sent to control port. */ @@ -3262,16 +3282,40 @@ typedef struct origin_circuit_t { * associated with this circuit. */ edge_connection_t *p_streams; - /** Bytes read from any attached stream since last call to + /** Smartlist of half-closed streams (half_edge_t*) that still + * have pending activity */ + smartlist_t *half_streams; + + /** Bytes read on this circuit since last call to * control_event_circ_bandwidth_used(). Only used if we're configured * to emit CIRC_BW events. */ uint32_t n_read_circ_bw; - /** Bytes written to any attached stream since last call to + /** Bytes written to on this circuit since last call to * control_event_circ_bandwidth_used(). Only used if we're configured * to emit CIRC_BW events. */ uint32_t n_written_circ_bw; + /** Total known-valid relay cell bytes since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_delivered_read_circ_bw; + + /** Total written relay cell bytes since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_delivered_written_circ_bw; + + /** Total overhead data in all known-valid relay data cells since last + * call to control_event_circ_bandwidth_used(). Only used if we're + * configured to emit CIRC_BW events. */ + uint32_t n_overhead_read_circ_bw; + + /** Total written overhead data in all relay data cells since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_overhead_written_circ_bw; + /** Build state for this circuit. It includes the intended path * length, the chosen exit router, rendezvous information, etc. */ @@ -3458,14 +3502,6 @@ struct onion_queue_t; typedef struct or_circuit_t { circuit_t base_; - /** Next circuit in the doubly-linked ring of circuits waiting to add - * cells to p_chan. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *next_active_on_p_chan; - /** Previous circuit in the doubly-linked ring of circuits waiting to add - * cells to p_chan. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *prev_active_on_p_chan; /** Pointer to an entry on the onion queue, if this circuit is waiting for a * chance to give an onionskin to a cpuworker. Used only in onion.c */ struct onion_queue_t *onionqueue_entry; @@ -3490,21 +3526,10 @@ typedef struct or_circuit_t { /** Linked list of Exit streams associated with this circuit that are * still being resolved. */ edge_connection_t *resolving_streams; - /** The cipher used by intermediate hops for cells heading toward the - * OP. */ - crypto_cipher_t *p_crypto; - /** The cipher used by intermediate hops for cells heading away from - * the OP. */ - crypto_cipher_t *n_crypto; - - /** The integrity-checking digest used by intermediate hops, for - * cells packaged here and heading towards the OP. - */ - crypto_digest_t *p_digest; - /** The integrity-checking digest used by intermediate hops, for - * cells packaged at the OP and arriving here. - */ - crypto_digest_t *n_digest; + + /** Cryptographic state used for encrypting and authenticating relay + * cells to and from this hop. */ + relay_crypto_t crypto; /** Points to spliced circuit if purpose is REND_ESTABLISHED, and circuit * is not marked for close. */ @@ -4180,6 +4205,8 @@ typedef struct { int NumDirectoryGuards; /**< How many dir guards do we try to establish? * If 0, use value from NumEntryGuards. */ + int NumPrimaryGuards; /**< How many primary guards do we want? */ + int RephistTrackTime; /**< How many seconds do we keep rephist info? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ @@ -4223,10 +4250,6 @@ typedef struct { * testing our DNS server. */ int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the * same network zone in the same circuit. */ - int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically - * forward the DirPort and ORPort on the NAT device */ - char *PortForwardingHelper; /** < Filename or full path of the port - forwarding helper executable */ int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames * with weird characters. */ /** If true, we try resolving hostnames with weird characters. */ @@ -4352,19 +4375,19 @@ typedef struct { /** Schedule for when servers should download things in general. Only * altered on testing networks. */ - smartlist_t *TestingServerDownloadSchedule; + int TestingServerDownloadInitialDelay; /** Schedule for when clients should download things in general. Only * altered on testing networks. */ - smartlist_t *TestingClientDownloadSchedule; + int TestingClientDownloadInitialDelay; /** Schedule for when servers should download consensuses. Only altered * on testing networks. */ - smartlist_t *TestingServerConsensusDownloadSchedule; + int TestingServerConsensusDownloadInitialDelay; /** Schedule for when clients should download consensuses. Only altered * on testing networks. */ - smartlist_t *TestingClientConsensusDownloadSchedule; + int TestingClientConsensusDownloadInitialDelay; /** Schedule for when clients should download consensuses from authorities * if they are bootstrapping (that is, they don't have a usable, reasonably @@ -4374,7 +4397,7 @@ typedef struct { * This schedule is incremented by (potentially concurrent) connection * attempts, unlike other schedules, which are incremented by connection * failures. Only altered on testing networks. */ - smartlist_t *ClientBootstrapConsensusAuthorityDownloadSchedule; + int ClientBootstrapConsensusAuthorityDownloadInitialDelay; /** Schedule for when clients should download consensuses from fallback * directory mirrors if they are bootstrapping (that is, they don't have a @@ -4384,7 +4407,7 @@ typedef struct { * This schedule is incremented by (potentially concurrent) connection * attempts, unlike other schedules, which are incremented by connection * failures. Only altered on testing networks. */ - smartlist_t *ClientBootstrapConsensusFallbackDownloadSchedule; + int ClientBootstrapConsensusFallbackDownloadInitialDelay; /** Schedule for when clients should download consensuses from authorities * if they are bootstrapping (that is, they don't have a usable, reasonably @@ -4394,15 +4417,15 @@ typedef struct { * This schedule is incremented by (potentially concurrent) connection * attempts, unlike other schedules, which are incremented by connection * failures. Only altered on testing networks. */ - smartlist_t *ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + int ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay; /** Schedule for when clients should download bridge descriptors. Only * altered on testing networks. */ - smartlist_t *TestingBridgeDownloadSchedule; + int TestingBridgeDownloadInitialDelay; /** Schedule for when clients should download bridge descriptors when they * have no running bridges. Only altered on testing networks. */ - smartlist_t *TestingBridgeBootstrapDownloadSchedule; + int TestingBridgeBootstrapDownloadInitialDelay; /** When directory clients have only a few descriptors to request, they * batch them until they have more, or until this amount of time has @@ -4450,9 +4473,6 @@ typedef struct { /** Enable CELL_STATS events. Only altered on testing networks. */ int TestingEnableCellStatsEvent; - /** Enable TB_EMPTY events. Only altered on testing networks. */ - int TestingEnableTbEmptyEvent; - /** If true, and we have GeoIP data, and we're a bridge, keep a per-country * count of how many client addresses have contacted us so that we can help * the bridge authority guess which countries have blocked access to us. */ @@ -4783,15 +4803,6 @@ typedef struct { time_t LastRotatedOnionKey; } or_state_t; -/** Change the next_write time of <b>state</b> to <b>when</b>, unless the - * state is already scheduled to be written to disk earlier than <b>when</b>. - */ -static inline void or_state_mark_dirty(or_state_t *state, time_t when) -{ - if (state->next_write > when) - state->next_write = when; -} - #define MAX_SOCKS_REPLY_LEN 1024 #define MAX_SOCKS_ADDR_LEN 256 #define SOCKS_NO_AUTH 0x00 diff --git a/src/or/parsecommon.c b/src/or/parsecommon.c index e7d01a5029..cd1a0c2521 100644 --- a/src/or/parsecommon.c +++ b/src/or/parsecommon.c @@ -426,7 +426,7 @@ find_by_keyword_(smartlist_t *s, directory_keyword keyword, * NULL if no such keyword is found. */ directory_token_t * -find_opt_by_keyword(smartlist_t *s, directory_keyword keyword) +find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword) { SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t); return NULL; diff --git a/src/or/parsecommon.h b/src/or/parsecommon.h index 903d94478b..d33faf8ec7 100644 --- a/src/or/parsecommon.h +++ b/src/or/parsecommon.h @@ -314,7 +314,7 @@ directory_token_t *find_by_keyword_(smartlist_t *s, #define find_by_keyword(s, keyword) \ find_by_keyword_((s), (keyword), #keyword) -directory_token_t *find_opt_by_keyword(smartlist_t *s, +directory_token_t *find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword); smartlist_t * find_all_by_keyword(const smartlist_t *s, directory_keyword k); diff --git a/src/or/periodic.c b/src/or/periodic.c index 6896b41c86..9470376d06 100644 --- a/src/or/periodic.c +++ b/src/or/periodic.c @@ -14,10 +14,9 @@ #include "or.h" #include "compat_libevent.h" #include "config.h" +#include "main.h" #include "periodic.h" -#include <event2/event.h> - /** We disable any interval greater than this number of seconds, on the * grounds that it is probably an absolute time mistakenly passed in as a * relative time. @@ -34,24 +33,34 @@ periodic_event_set_interval(periodic_event_item_t *event, struct timeval tv; tv.tv_sec = next_interval; tv.tv_usec = 0; - event_add(event->ev, &tv); + mainloop_event_schedule(event->ev, &tv); } /** Wraps dispatches for periodic events, <b>data</b> will be a pointer to the * event that needs to be called */ static void -periodic_event_dispatch(evutil_socket_t fd, short what, void *data) +periodic_event_dispatch(mainloop_event_t *ev, void *data) { - (void)fd; - (void)what; periodic_event_item_t *event = data; + tor_assert(ev == event->ev); + + if (BUG(!periodic_event_is_enabled(event))) { + return; + } time_t now = time(NULL); + update_current_time(now); const or_options_t *options = get_options(); // log_debug(LD_GENERAL, "Dispatching %s", event->name); int r = event->fn(now, options); int next_interval = 0; + if (!periodic_event_is_enabled(event)) { + /* The event got disabled from inside its callback; no need to + * reschedule. */ + return; + } + /* update the last run time if action was taken */ if (r==0) { log_err(LD_BUG, "Invalid return value for periodic event from %s.", @@ -74,14 +83,17 @@ periodic_event_dispatch(evutil_socket_t fd, short what, void *data) // log_debug(LD_GENERAL, "Scheduling %s for %d seconds", event->name, // next_interval); struct timeval tv = { next_interval , 0 }; - event_add(event->ev, &tv); + mainloop_event_schedule(ev, &tv); } /** Schedules <b>event</b> to run as soon as possible from now. */ void periodic_event_reschedule(periodic_event_item_t *event) { - periodic_event_set_interval(event, 1); + /* Don't reschedule a disabled event. */ + if (periodic_event_is_enabled(event)) { + periodic_event_set_interval(event, 1); + } } /** Initializes the libevent backend for a periodic event. */ @@ -93,10 +105,8 @@ periodic_event_setup(periodic_event_item_t *event) tor_assert(0); } - event->ev = tor_event_new(tor_libevent_get_base(), - -1, 0, - periodic_event_dispatch, - event); + event->ev = mainloop_event_new(periodic_event_dispatch, + event); tor_assert(event->ev); } @@ -109,9 +119,15 @@ periodic_event_launch(periodic_event_item_t *event) log_err(LD_BUG, "periodic_event_launch without periodic_event_setup"); tor_assert(0); } + /* Event already enabled? This is a bug */ + if (periodic_event_is_enabled(event)) { + log_err(LD_BUG, "periodic_event_launch on an already enabled event"); + tor_assert(0); + } // Initial dispatch - periodic_event_dispatch(-1, EV_TIMEOUT, event); + event->enabled = 1; + periodic_event_dispatch(event->ev, event); } /** Release all storage associated with <b>event</b> */ @@ -120,7 +136,38 @@ periodic_event_destroy(periodic_event_item_t *event) { if (!event) return; - tor_event_free(event->ev); + mainloop_event_free(event->ev); event->last_action_time = 0; } +/** Enable the given event by setting its "enabled" flag and scheduling it to + * run immediately in the event loop. This can be called for an event that is + * already enabled. */ +void +periodic_event_enable(periodic_event_item_t *event) +{ + tor_assert(event); + /* Safely and silently ignore if this event is already enabled. */ + if (periodic_event_is_enabled(event)) { + return; + } + + tor_assert(event->ev); + event->enabled = 1; + mainloop_event_activate(event->ev); +} + +/** Disable the given event which means the event is destroyed and then the + * event's enabled flag is unset. This can be called for an event that is + * already disabled. */ +void +periodic_event_disable(periodic_event_item_t *event) +{ + tor_assert(event); + /* Safely and silently ignore if this event is already disabled. */ + if (!periodic_event_is_enabled(event)) { + return; + } + mainloop_event_cancel(event->ev); + event->enabled = 0; +} diff --git a/src/or/periodic.h b/src/or/periodic.h index 8baf3994eb..e8208b2475 100644 --- a/src/or/periodic.h +++ b/src/or/periodic.h @@ -6,6 +6,39 @@ #define PERIODIC_EVENT_NO_UPDATE (-1) +/* Tor roles for which a periodic event item is for. An event can be for + * multiple roles, they can be combined. */ +#define PERIODIC_EVENT_ROLE_CLIENT (1U << 0) +#define PERIODIC_EVENT_ROLE_RELAY (1U << 1) +#define PERIODIC_EVENT_ROLE_BRIDGE (1U << 2) +#define PERIODIC_EVENT_ROLE_DIRAUTH (1U << 3) +#define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) +#define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) +#define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) + +/* Helper macro to make it a bit less annoying to defined groups of roles that + * are often used. */ + +/* Router that is a Bridge or Relay. */ +#define PERIODIC_EVENT_ROLE_ROUTER \ + (PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_RELAY) +/* Authorities that is both bridge and directory. */ +#define PERIODIC_EVENT_ROLE_AUTHORITIES \ + (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) +/* All roles. */ +#define PERIODIC_EVENT_ROLE_ALL \ + (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) + +/* + * Event flags which can change the behavior of an event. + */ + +/* Indicate that the event needs the network meaning that if we are in + * DisableNetwork or hibernation mode, the event won't be enabled. This obey + * the net_is_disabled() check. */ +#define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled @@ -14,24 +47,42 @@ typedef int (*periodic_event_helper_t)(time_t now, const or_options_t *options); -struct event; +struct mainloop_event_t; /** A single item for the periodic-events-function table. */ typedef struct periodic_event_item_t { periodic_event_helper_t fn; /**< The function to run the event */ time_t last_action_time; /**< The last time the function did something */ - struct event *ev; /**< Libevent callback we're using to implement this */ + struct mainloop_event_t *ev; /**< Libevent callback we're using to implement + * this */ const char *name; /**< Name of the function -- for debug */ + + /* Bitmask of roles define above for which this event applies. */ + uint32_t roles; + /* Bitmask of flags which can change the behavior of the event. */ + uint32_t flags; + /* Indicate that this event has been enabled that is scheduled. */ + unsigned int enabled : 1; } periodic_event_item_t; /** events will get their interval from first execution */ -#define PERIODIC_EVENT(fn) { fn##_callback, 0, NULL, #fn } -#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL } +#define PERIODIC_EVENT(fn, r, f) { fn##_callback, 0, NULL, #fn, r, f, 0 } +#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL, 0, 0, 0 } + +/* Return true iff the given event was setup before thus is enabled to be + * scheduled. */ +static inline int +periodic_event_is_enabled(const periodic_event_item_t *item) +{ + return item->enabled; +} void periodic_event_launch(periodic_event_item_t *event); void periodic_event_setup(periodic_event_item_t *event); void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); +void periodic_event_enable(periodic_event_item_t *event); +void periodic_event_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ diff --git a/src/or/policies.c b/src/or/policies.c index f718ded326..1210ca687d 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -825,9 +825,8 @@ fascist_firewall_choose_address(const tor_addr_port_t *a, * If pref_only, only choose preferred addresses. In either case, choose * a preferred address before an address that's not preferred. * If both addresses could be chosen (they are both preferred or both allowed) - * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. - * If neither address is chosen, return 0, else return 1. */ -static int + * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. */ +static void fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, uint16_t ipv4_orport, uint16_t ipv4_dirport, @@ -845,6 +844,9 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, tor_assert(ipv6_addr); tor_assert(ap); + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + tor_addr_port_t ipv4_ap; tor_addr_copy(&ipv4_ap.addr, ipv4_addr); ipv4_ap.port = (fw_connection == FIREWALL_OR_CONNECTION @@ -865,17 +867,12 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, if (result) { tor_addr_copy(&ap->addr, &result->addr); ap->port = result->port; - return 1; - } else { - tor_addr_make_null(&ap->addr, AF_UNSPEC); - ap->port = 0; - return 0; } } /** Like fascist_firewall_choose_address_base(), but takes a host-order IPv4 * address as the first parameter. */ -static int +static void fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr, uint16_t ipv4_orport, uint16_t ipv4_dirport, @@ -889,11 +886,16 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr, { tor_addr_t ipv4_addr; tor_addr_from_ipv4h(&ipv4_addr, ipv4h_addr); - return fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport, - ipv4_dirport, ipv6_addr, - ipv6_orport, ipv6_dirport, - fw_connection, pref_only, - pref_ipv6, ap); + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + + fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport, + ipv4_dirport, ipv6_addr, + ipv6_orport, ipv6_dirport, + fw_connection, pref_only, + pref_ipv6, ap); } /* Some microdescriptor consensus methods have no IPv6 addresses in rs: they @@ -944,23 +946,25 @@ node_awaiting_ipv6(const or_options_t* options, const node_t *node) * This should only happen when there's no valid consensus, and rs doesn't * correspond to a bridge client's bridge. */ -int +void fascist_firewall_choose_address_rs(const routerstatus_t *rs, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap) { + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + if (!rs) { - return 0; + return; } - tor_assert(ap); - const or_options_t *options = get_options(); const node_t *node = node_get_by_id(rs->identity_digest); if (node && !node_awaiting_ipv6(options, node)) { - return fascist_firewall_choose_address_node(node, fw_connection, pref_only, - ap); + fascist_firewall_choose_address_node(node, fw_connection, pref_only, ap); } else { /* There's no node-specific IPv6 preference, so use the generic IPv6 * preference instead. */ @@ -970,33 +974,31 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs, /* Assume IPv4 and IPv6 DirPorts are the same. * Assume the IPv6 OR and Dir addresses are the same. */ - return fascist_firewall_choose_address_ipv4h(rs->addr, - rs->or_port, - rs->dir_port, - &rs->ipv6_addr, - rs->ipv6_orport, - rs->dir_port, - fw_connection, - pref_only, - pref_ipv6, - ap); + fascist_firewall_choose_address_ipv4h(rs->addr, rs->or_port, rs->dir_port, + &rs->ipv6_addr, rs->ipv6_orport, + rs->dir_port, fw_connection, + pref_only, pref_ipv6, ap); } } /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and * looks up the node's IPv6 preference rather than taking an argument * for pref_ipv6. */ -int +void fascist_firewall_choose_address_node(const node_t *node, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t *ap) { + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + if (!node) { - return 0; + return; } node_assert_ok(node); - /* Calling fascist_firewall_choose_address_node() when the node is missing * IPv6 information breaks IPv6-only clients. * If the node is a hard-coded fallback directory or authority, call @@ -1006,7 +1008,7 @@ fascist_firewall_choose_address_node(const node_t *node, * descriptor (routerinfo), or is one of our configured bridges before * calling this function. */ if (BUG(node_awaiting_ipv6(get_options(), node))) { - return 0; + return; } const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION @@ -1024,27 +1026,27 @@ fascist_firewall_choose_address_node(const node_t *node, node_get_pref_ipv6_dirport(node, &ipv6_dir_ap); /* Assume the IPv6 OR and Dir addresses are the same. */ - return fascist_firewall_choose_address_base(&ipv4_or_ap.addr, - ipv4_or_ap.port, - ipv4_dir_ap.port, - &ipv6_or_ap.addr, - ipv6_or_ap.port, - ipv6_dir_ap.port, - fw_connection, - pref_only, - pref_ipv6_node, - ap); + fascist_firewall_choose_address_base(&ipv4_or_ap.addr, ipv4_or_ap.port, + ipv4_dir_ap.port, &ipv6_or_ap.addr, + ipv6_or_ap.port, ipv6_dir_ap.port, + fw_connection, pref_only, + pref_ipv6_node, ap); } /** Like fascist_firewall_choose_address_rs(), but takes <b>ds</b>. */ -int +void fascist_firewall_choose_address_dir_server(const dir_server_t *ds, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t *ap) { + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + if (!ds) { - return 0; + return; } /* A dir_server_t always has a fake_status. As long as it has the same @@ -1052,8 +1054,8 @@ fascist_firewall_choose_address_dir_server(const dir_server_t *ds, * (See #17867.) * This function relies on fascist_firewall_choose_address_rs looking up the * node if it can, because that will get the latest info for the relay. */ - return fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection, - pref_only, ap); + fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection, + pref_only, ap); } /** Return 1 if <b>addr</b> is permitted to connect to our dir port, @@ -2997,11 +2999,12 @@ getinfo_helper_policies(control_connection_t *conn, smartlist_free(private_policy_strings); } else if (!strcmp(question, "exit-policy/reject-private/relay")) { const or_options_t *options = get_options(); - const routerinfo_t *me = router_get_my_routerinfo(); + int err = 0; + const routerinfo_t *me = router_get_my_routerinfo_with_err(&err); if (!me) { - *errmsg = "router_get_my_routerinfo returned NULL"; - return -1; + *errmsg = routerinfo_err_to_string(err); + return routerinfo_err_is_transient(err) ? -1 : 0; } if (!options->ExitPolicyRejectPrivate && @@ -3036,11 +3039,17 @@ getinfo_helper_policies(control_connection_t *conn, SMARTLIST_FOREACH(configured_addresses, tor_addr_t *, a, tor_free(a)); smartlist_free(configured_addresses); } else if (!strcmpstart(question, "exit-policy/")) { - const routerinfo_t *me = router_get_my_routerinfo(); - int include_ipv4 = 0; int include_ipv6 = 0; + int err = 0; + const routerinfo_t *me = router_get_my_routerinfo_with_err(&err); + + if (!me) { + *errmsg = routerinfo_err_to_string(err); + return routerinfo_err_is_transient(err) ? -1 : 0; + } + if (!strcmp(question, "exit-policy/ipv4")) { include_ipv4 = 1; } else if (!strcmp(question, "exit-policy/ipv6")) { @@ -3051,13 +3060,10 @@ getinfo_helper_policies(control_connection_t *conn, return 0; /* No such key. */ } - if (!me) { - *errmsg = "router_get_my_routerinfo returned NULL"; - return -1; - } - - *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6); + *answer = router_dump_exit_policy_to_string(me,include_ipv4, + include_ipv6); } + return 0; } diff --git a/src/or/policies.h b/src/or/policies.h index 35220a812f..4879acdd8d 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -55,13 +55,13 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds, firewall_connection_t fw_connection, int pref_only); -int fascist_firewall_choose_address_rs(const routerstatus_t *rs, - firewall_connection_t fw_connection, - int pref_only, tor_addr_port_t* ap); -int fascist_firewall_choose_address_node(const node_t *node, - firewall_connection_t fw_connection, - int pref_only, tor_addr_port_t* ap); -int fascist_firewall_choose_address_dir_server(const dir_server_t *ds, +void fascist_firewall_choose_address_rs(const routerstatus_t *rs, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); +void fascist_firewall_choose_address_node(const node_t *node, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); +void fascist_firewall_choose_address_dir_server(const dir_server_t *ds, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap); diff --git a/src/or/proto_socks.c b/src/or/proto_socks.c index 8700fe1269..57a7d1cd64 100644 --- a/src/or/proto_socks.c +++ b/src/or/proto_socks.c @@ -9,6 +9,7 @@ #include "buffers.h" #include "control.h" #include "config.h" +#include "crypto_util.h" #include "ext_orport.h" #include "proto_socks.h" #include "reasons.h" diff --git a/src/or/protover.c b/src/or/protover.c index a63c2eb02d..c973660980 100644 --- a/src/or/protover.c +++ b/src/or/protover.c @@ -781,7 +781,7 @@ protover_all_supported(const char *s, char **missing_out) versions->high = i; } /* If the last one to be unsupported is one less than the current - * one, we're in a continous range, so set the high field. */ + * one, we're in a continuous range, so set the high field. */ if ((versions->high && versions->high == i - 1) || /* Similarly, if the last high wasn't set and we're currently * one higher than the low, add current index as the highest diff --git a/src/or/protover.h b/src/or/protover.h index b94ebab15b..c46a13de66 100644 --- a/src/or/protover.h +++ b/src/or/protover.h @@ -10,7 +10,7 @@ #define TOR_PROTOVER_H #include "container.h" -#include <stdbool.h> + /** The first version of Tor that included "proto" entries in its * descriptors. Authorities should use this to decide whether to * guess proto lines. */ diff --git a/src/or/relay.c b/src/or/relay.c index 4c1a8ed96d..497afe756a 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -61,6 +61,8 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "geoip.h" #include "hs_cache.h" #include "main.h" @@ -70,6 +72,7 @@ #include "policies.h" #include "reasons.h" #include "relay.h" +#include "relay_crypto.h" #include "rendcache.h" #include "rendcommon.h" #include "router.h" @@ -82,9 +85,6 @@ static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint); static void circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint); static void circuit_resume_edge_reading(circuit_t *circ, @@ -122,77 +122,6 @@ uint64_t stats_n_circ_max_cell_reached = 0; /** Used to tell which stream to read from first on a circuit. */ static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT; -/** Update digest from the payload of cell. Assign integrity part to - * cell. - */ -static void -relay_set_digest(crypto_digest_t *digest, cell_t *cell) -{ - char integrity[4]; - relay_header_t rh; - - crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE); - crypto_digest_get_digest(digest, integrity, 4); -// log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.", -// integrity[0], integrity[1], integrity[2], integrity[3]); - relay_header_unpack(&rh, cell->payload); - memcpy(rh.integrity, integrity, 4); - relay_header_pack(cell->payload, &rh); -} - -/** Does the digest for this circuit indicate that this cell is for us? - * - * Update digest from the payload of cell (with the integrity part set - * to 0). If the integrity part is valid, return 1, else restore digest - * and cell to their original state and return 0. - */ -static int -relay_digest_matches(crypto_digest_t *digest, cell_t *cell) -{ - uint32_t received_integrity, calculated_integrity; - relay_header_t rh; - crypto_digest_t *backup_digest=NULL; - - backup_digest = crypto_digest_dup(digest); - - relay_header_unpack(&rh, cell->payload); - memcpy(&received_integrity, rh.integrity, 4); - memset(rh.integrity, 0, 4); - relay_header_pack(cell->payload, &rh); - -// log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.", -// received_integrity[0], received_integrity[1], -// received_integrity[2], received_integrity[3]); - - crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE); - crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4); - - if (calculated_integrity != received_integrity) { -// log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing."); -// (%d vs %d).", received_integrity, calculated_integrity); - /* restore digest to its old form */ - crypto_digest_assign(digest, backup_digest); - /* restore the relay header */ - memcpy(rh.integrity, &received_integrity, 4); - relay_header_pack(cell->payload, &rh); - crypto_digest_free(backup_digest); - return 0; - } - crypto_digest_free(backup_digest); - return 1; -} - -/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b> - * (in place). - * - * Note that we use the same operation for encrypting and for decrypting. - */ -static void -relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) -{ - crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); -} - /** * Update channel usage state based on the type of relay cell and * circuit properties. @@ -297,7 +226,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, if (circ->marked_for_close) return 0; - if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) { + if (relay_decrypt_cell(circ, cell, cell_direction, &layer_hint, &recognized) + < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "relay crypt failed. Dropping connection."); return -END_CIRC_REASON_INTERNAL; @@ -309,7 +239,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, edge_connection_t *conn = NULL; if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) { - pathbias_check_probe_response(circ, cell); + if (pathbias_check_probe_response(circ, cell) == -1) { + pathbias_count_valid_cells(circ, cell); + } /* We need to drop this cell no matter what to avoid code that expects * a certain purpose (such as the hidserv code). */ @@ -402,87 +334,6 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; } -/** Do the appropriate en/decryptions for <b>cell</b> arriving on - * <b>circ</b> in direction <b>cell_direction</b>. - * - * If cell_direction == CELL_DIRECTION_IN: - * - If we're at the origin (we're the OP), for hops 1..N, - * decrypt cell. If recognized, stop. - * - Else (we're not the OP), encrypt one hop. Cell is not recognized. - * - * If cell_direction == CELL_DIRECTION_OUT: - * - decrypt one hop. Check if recognized. - * - * If cell is recognized, set *recognized to 1, and set - * *layer_hint to the hop that recognized it. - * - * Return -1 to indicate that we should mark the circuit for close, - * else return 0. - */ -int -relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, - crypt_path_t **layer_hint, char *recognized) -{ - relay_header_t rh; - - tor_assert(circ); - tor_assert(cell); - tor_assert(recognized); - tor_assert(cell_direction == CELL_DIRECTION_IN || - cell_direction == CELL_DIRECTION_OUT); - - if (cell_direction == CELL_DIRECTION_IN) { - if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit. - * We'll want to do layered decrypts. */ - crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath; - thishop = cpath; - if (thishop->state != CPATH_STATE_OPEN) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Relay cell before first created cell? Closing."); - return -1; - } - do { /* Remember: cpath is in forward order, that is, first hop first. */ - tor_assert(thishop); - - /* decrypt one layer */ - relay_crypt_one_payload(thishop->b_crypto, cell->payload); - - relay_header_unpack(&rh, cell->payload); - if (rh.recognized == 0) { - /* it's possibly recognized. have to check digest to be sure. */ - if (relay_digest_matches(thishop->b_digest, cell)) { - *recognized = 1; - *layer_hint = thishop; - return 0; - } - } - - thishop = thishop->next; - } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN); - log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Incoming cell at client not recognized. Closing."); - return -1; - } else { - /* We're in the middle. Encrypt one layer. */ - relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->p_crypto, cell->payload); - } - } else /* cell_direction == CELL_DIRECTION_OUT */ { - /* We're in the middle. Decrypt one layer. */ - - relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->n_crypto, cell->payload); - - relay_header_unpack(&rh, cell->payload); - if (rh.recognized == 0) { - /* it's possibly recognized. have to check digest to be sure. */ - if (relay_digest_matches(TO_OR_CIRCUIT(circ)->n_digest, cell)) { - *recognized = 1; - return 0; - } - } - } - return 0; -} - /** Package a relay cell from an edge: * - Encrypt it to the right layer * - Append it to the appropriate cell_queue on <b>circ</b>. @@ -501,7 +352,6 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, } if (cell_direction == CELL_DIRECTION_OUT) { - crypt_path_t *thishop; /* counter for repeated crypts */ chan = circ->n_chan; if (!chan) { log_warn(LD_BUG,"outgoing relay cell sent from %s:%d has n_chan==NULL." @@ -524,20 +374,14 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, return 0; /* just drop it */ } - relay_set_digest(layer_hint->f_digest, cell); + relay_encrypt_cell_outbound(cell, TO_ORIGIN_CIRCUIT(circ), layer_hint); - thishop = layer_hint; - /* moving from farthest to nearest hop */ - do { - tor_assert(thishop); - log_debug(LD_OR,"encrypting a layer of the relay cell."); - relay_crypt_one_payload(thishop->f_crypto, cell->payload); - - thishop = thishop->prev; - } while (thishop != TO_ORIGIN_CIRCUIT(circ)->cpath->prev); + /* Update circ written totals for control port */ + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_written_circ_bw = tor_add_u32_nowrap(ocirc->n_written_circ_bw, + CELL_PAYLOAD_SIZE); } else { /* incoming cell */ - or_circuit_t *or_circ; if (CIRCUIT_IS_ORIGIN(circ)) { /* We should never package an _incoming_ cell from the circuit * origin; that means we messed up somewhere. */ @@ -545,11 +389,9 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, assert_circuit_ok(circ); return 0; /* just drop it */ } - or_circ = TO_OR_CIRCUIT(circ); + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + relay_encrypt_cell_inbound(cell, or_circ); chan = or_circ->p_chan; - relay_set_digest(or_circ->p_digest, cell); - /* encrypt one layer */ - relay_crypt_one_payload(or_circ->p_crypto, cell->payload); } ++stats_n_relay_cells_relayed; @@ -770,6 +612,10 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, tor_free(commands); smartlist_free(commands_list); } + + /* Let's assume we're well-behaved: Anything that we decide to send is + * valid, delivered data. */ + circuit_sent_valid_data(origin_circ, rh.length); } if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer, @@ -899,6 +745,9 @@ connection_ap_process_end_not_open( } } + /* This end cell is now valid. */ + circuit_read_valid_data(circ, rh->length); + if (rh->length == 0) { reason = END_STREAM_REASON_MISC; } @@ -1118,7 +967,12 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr) * header has already been parsed into <b>rh</b>. On success, set * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to * the ttl of that address, in seconds, and return 0. On failure, return - * -1. */ + * -1. + * + * Note that the resulting address can be UNSPEC if the connected cell had no + * address (as for a stream to an union service or a tunneled directory + * connection), and that the ttl can be absent (in which case <b>ttl_out</b> + * is set to -1). */ STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out) @@ -1389,6 +1243,12 @@ connection_edge_process_resolved_cell(edge_connection_t *conn, } } + /* This is valid data at this point. Count it */ + if (conn->on_circuit && CIRCUIT_IS_ORIGIN(conn->on_circuit)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(conn->on_circuit), + rh->length); + } + connection_ap_handshake_socks_got_resolved_cell(entry_conn, errcode, resolved_addresses); @@ -1449,7 +1309,7 @@ connection_edge_process_relay_cell_not_open( "after %d seconds.", (unsigned)circ->n_circ_id, rh->stream_id, - (int)(time(NULL) - conn->base_.timestamp_lastread)); + (int)(time(NULL) - conn->base_.timestamp_last_read_allowed)); if (connected_cell_parse(rh, cell, &addr, &ttl) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a badly formatted connected cell. Closing."); @@ -1458,6 +1318,9 @@ connection_edge_process_relay_cell_not_open( return 0; } if (tor_addr_family(&addr) != AF_UNSPEC) { + /* The family is not UNSPEC: so we were given an address in the + * connected cell. (This is normal, except for BEGINDIR and onion + * service streams.) */ const sa_family_t family = tor_addr_family(&addr); if (tor_addr_is_null(&addr) || (get_options()->ClientDNSRejectInternalAddresses && @@ -1524,6 +1387,9 @@ connection_edge_process_relay_cell_not_open( entry_conn->pending_optimistic_data = NULL; } + /* This is valid data at this point. Count it */ + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + /* handle anything that might have queued */ if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -1556,7 +1422,7 @@ connection_edge_process_relay_cell_not_open( * * Return -reason if you want to warn and tear down the circuit, else 0. */ -static int +STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, edge_connection_t *conn, crypt_path_t *layer_hint) @@ -1656,7 +1522,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, circ->dirreq_id = ++next_id; TO_OR_CIRCUIT(circ)->p_chan->dirreq_id = circ->dirreq_id; } - return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1682,6 +1547,17 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "stream_id. Dropping."); return 0; } else if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (connection_half_edge_is_valid_data(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(ocirc, rh.length); + log_info(domain, + "data cell on circ %u valid on half-closed " + "stream id %d", ocirc->global_identifier, rh.stream_id); + } + } + log_info(domain,"data cell dropped, unknown stream (streamid %d).", rh.stream_id); return 0; @@ -1692,6 +1568,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "(relay data) conn deliver_window below 0. Killing."); return -END_CIRC_REASON_TORPROTOCOL; } + /* Total all valid application bytes delivered */ + if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } stats_n_data_bytes_received += rh.length; connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE), @@ -1719,6 +1599,20 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, reason = rh.length > 0 ? get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (connection_half_edge_is_valid_end(ocirc->half_streams, + rh.stream_id)) { + + circuit_read_valid_data(ocirc, rh.length); + log_info(domain, + "end cell (%s) on circ %u valid on half-closed " + "stream id %d", + stream_end_reason_to_string(reason), + ocirc->global_identifier, rh.stream_id); + return 0; + } + } log_info(domain,"end cell (%s) dropped, unknown stream.", stream_end_reason_to_string(reason)); return 0; @@ -1744,6 +1638,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* only mark it if not already marked. it's possible to * get the 'end' right around when the client hangs up on us. */ connection_mark_and_flush(TO_CONN(conn)); + + /* Total all valid application bytes delivered */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } } return 0; case RELAY_COMMAND_EXTEND: @@ -1809,6 +1708,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_info(domain,"circuit_send_next_onion_skin() failed."); return reason; } + /* Total all valid bytes delivered. */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } return 0; case RELAY_COMMAND_TRUNCATE: if (layer_hint) { @@ -1845,7 +1748,14 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "'truncated' unsupported at non-origin. Dropping."); return 0; } - circuit_truncated(TO_ORIGIN_CIRCUIT(circ), layer_hint, + + /* Count the truncated as valid, for completeness. The + * circuit is being torn down anyway, though. */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), + rh.length); + } + circuit_truncated(TO_ORIGIN_CIRCUIT(circ), get_uint8(cell->payload + RELAY_HEADER_SIZE)); return 0; case RELAY_COMMAND_CONNECTED: @@ -1854,6 +1764,19 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "'connected' unsupported while open. Closing circ."); return -END_CIRC_REASON_TORPROTOCOL; } + + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (connection_half_edge_is_valid_connected(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(ocirc, rh.length); + log_info(domain, + "connected cell on circ %u valid on half-closed " + "stream id %d", ocirc->global_identifier, rh.stream_id); + return 0; + } + } + log_info(domain, "'connected' received on circid %u for streamid %d, " "no conn attached anymore. Ignoring.", @@ -1874,6 +1797,15 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.", layer_hint->package_window); circuit_resume_edge_reading(circ, layer_hint); + + /* We count circuit-level sendme's as valid delivered data because + * they are rate limited. + */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), + rh.length); + } + } else { if (circ->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { @@ -1893,10 +1825,42 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return 0; } if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (connection_half_edge_is_valid_sendme(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(ocirc, rh.length); + log_info(domain, + "sendme cell on circ %u valid on half-closed " + "stream id %d", ocirc->global_identifier, rh.stream_id); + } + } + log_info(domain,"sendme cell dropped, unknown stream (streamid %d).", rh.stream_id); return 0; } + + /* Don't allow the other endpoint to request more than our maximim + * (ie initial) stream SENDME window worth of data. Well-behaved + * stock clients will not request more than this max (as per the check + * in the while loop of connection_edge_consider_sending_sendme()). + */ + if (conn->package_window + STREAMWINDOW_INCREMENT > + STREAMWINDOW_START_MAX) { + static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&stream_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unexpected stream sendme cell. Closing circ (window %d).", + conn->package_window); + return -END_CIRC_REASON_TORPROTOCOL; + } + + /* At this point, the stream sendme is valid */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), + rh.length); + } + conn->package_window += STREAMWINDOW_INCREMENT; log_debug(domain,"stream-level sendme, packagewindow now %d.", conn->package_window); @@ -1935,6 +1899,19 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "'resolved' unsupported while open. Closing circ."); return -END_CIRC_REASON_TORPROTOCOL; } + + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (connection_half_edge_is_valid_resolved(ocirc->half_streams, + rh.stream_id)) { + circuit_read_valid_data(ocirc, rh.length); + log_info(domain, + "resolved cell on circ %u valid on half-closed " + "stream id %d", ocirc->global_identifier, rh.stream_id); + return 0; + } + } + log_info(domain, "'resolved' received, no conn attached anymore. Ignoring."); return 0; @@ -2398,13 +2375,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) } } -#ifdef ACTIVE_CIRCUITS_PARANOIA -#define assert_cmux_ok_paranoid(chan) \ - assert_circuit_mux_okay(chan) -#else -#define assert_cmux_ok_paranoid(chan) -#endif /* defined(ACTIVE_CIRCUITS_PARANOIA) */ - /** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; @@ -2616,6 +2586,7 @@ cell_queues_check_size(void) { time_t now = time(NULL); size_t alloc = cell_queues_get_total_allocation(); + alloc += half_streams_get_total_allocation(); alloc += buf_get_total_allocation(); alloc += tor_compress_get_total_allocation(); const size_t rend_cache_total = rend_cache_get_total_allocation(); @@ -2692,16 +2663,12 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, } tor_assert(circuitmux_attached_circuit_direction(cmux, circ) == direction); - assert_cmux_ok_paranoid(chan); - /* Update the number of cells we have for the circuit mux */ if (direction == CELL_DIRECTION_OUT) { circuitmux_set_num_cells(cmux, circ, circ->n_chan_cells.n); } else { circuitmux_set_num_cells(cmux, circ, or_circ->p_chan_cells.n); } - - assert_cmux_ok_paranoid(chan); } /** Remove all circuits from the cmux on <b>chan</b>. @@ -2846,7 +2813,6 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max)) } /* If it returns NULL, no cells left to send */ if (!circ) break; - assert_cmux_ok_paranoid(chan); if (circ->n_chan == chan) { queue = &circ->n_chan_cells; @@ -2950,8 +2916,6 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max)) } /* Okay, we're done sending now */ - assert_cmux_ok_paranoid(chan); - return n_flushed; } @@ -3012,7 +2976,7 @@ relay_consensus_has_changed(const networkstatus_t *ns) /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b> * transmitting in <b>direction</b>. * - * The given <b>cell</b> is copied over the circuit queue so the caller must + * The given <b>cell</b> is copied onto the circuit queue so the caller must * cleanup the memory. * * This function is part of the fast path. */ @@ -3167,17 +3131,6 @@ circuit_clear_cell_queue(circuit_t *circ, channel_t *chan) update_circuit_on_cmux(circ, direction); } -/** Fail with an assert if the circuit mux on chan is corrupt - */ -void -assert_circuit_mux_okay(channel_t *chan) -{ - tor_assert(chan); - tor_assert(chan->cmux); - - circuitmux_assert_okay(chan->cmux); -} - /** Return 1 if we shouldn't restart reading on this circuit, even if * we get a SENDME. Else return 0. */ @@ -3190,4 +3143,3 @@ circuit_queue_streams_are_blocked(circuit_t *circ) return circ->streams_blocked_on_p_chan; } } - diff --git a/src/or/relay.h b/src/or/relay.h index e96639170c..ce0969b46c 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -78,7 +78,6 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue, void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); MOCK_DECL(int, channel_flush_from_first_active_circuit, (channel_t *chan, int max)); -void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); #define update_circuit_on_cmux(circ, direction) \ @@ -92,9 +91,6 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); void stream_choice_seed_weak_rng(void); -int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, - crypt_path_t **layer_hint, char *recognized); - circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); #ifdef RELAY_PRIVATE @@ -118,6 +114,10 @@ STATIC packed_cell_t *packed_cell_new(void); STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); STATIC destroy_cell_t *destroy_cell_queue_pop(destroy_cell_queue_t *queue); STATIC int cell_queues_check_size(void); +STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint); + #endif /* defined(RELAY_PRIVATE) */ #endif /* !defined(TOR_RELAY_H) */ diff --git a/src/or/relay_crypto.c b/src/or/relay_crypto.c new file mode 100644 index 0000000000..530c8e5828 --- /dev/null +++ b/src/or/relay_crypto.c @@ -0,0 +1,327 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "config.h" +#include "crypto_util.h" +#include "hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN +#include "relay.h" +#include "relay_crypto.h" + +/** Update digest from the payload of cell. Assign integrity part to + * cell. + */ +static void +relay_set_digest(crypto_digest_t *digest, cell_t *cell) +{ + char integrity[4]; + relay_header_t rh; + + crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE); + crypto_digest_get_digest(digest, integrity, 4); +// log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.", +// integrity[0], integrity[1], integrity[2], integrity[3]); + relay_header_unpack(&rh, cell->payload); + memcpy(rh.integrity, integrity, 4); + relay_header_pack(cell->payload, &rh); +} + +/** Does the digest for this circuit indicate that this cell is for us? + * + * Update digest from the payload of cell (with the integrity part set + * to 0). If the integrity part is valid, return 1, else restore digest + * and cell to their original state and return 0. + */ +static int +relay_digest_matches(crypto_digest_t *digest, cell_t *cell) +{ + uint32_t received_integrity, calculated_integrity; + relay_header_t rh; + crypto_digest_checkpoint_t backup_digest; + + crypto_digest_checkpoint(&backup_digest, digest); + + relay_header_unpack(&rh, cell->payload); + memcpy(&received_integrity, rh.integrity, 4); + memset(rh.integrity, 0, 4); + relay_header_pack(cell->payload, &rh); + +// log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.", +// received_integrity[0], received_integrity[1], +// received_integrity[2], received_integrity[3]); + + crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE); + crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4); + + int rv = 1; + + if (calculated_integrity != received_integrity) { +// log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing."); +// (%d vs %d).", received_integrity, calculated_integrity); + /* restore digest to its old form */ + crypto_digest_restore(digest, &backup_digest); + /* restore the relay header */ + memcpy(rh.integrity, &received_integrity, 4); + relay_header_pack(cell->payload, &rh); + rv = 0; + } + + memwipe(&backup_digest, 0, sizeof(backup_digest)); + return rv; +} + +/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b> + * (in place). + * + * Note that we use the same operation for encrypting and for decrypting. + */ +static void +relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) +{ + crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); +} + +/** Do the appropriate en/decryptions for <b>cell</b> arriving on + * <b>circ</b> in direction <b>cell_direction</b>. + * + * If cell_direction == CELL_DIRECTION_IN: + * - If we're at the origin (we're the OP), for hops 1..N, + * decrypt cell. If recognized, stop. + * - Else (we're not the OP), encrypt one hop. Cell is not recognized. + * + * If cell_direction == CELL_DIRECTION_OUT: + * - decrypt one hop. Check if recognized. + * + * If cell is recognized, set *recognized to 1, and set + * *layer_hint to the hop that recognized it. + * + * Return -1 to indicate that we should mark the circuit for close, + * else return 0. + */ +int +relay_decrypt_cell(circuit_t *circ, cell_t *cell, + cell_direction_t cell_direction, + crypt_path_t **layer_hint, char *recognized) +{ + relay_header_t rh; + + tor_assert(circ); + tor_assert(cell); + tor_assert(recognized); + tor_assert(cell_direction == CELL_DIRECTION_IN || + cell_direction == CELL_DIRECTION_OUT); + + if (cell_direction == CELL_DIRECTION_IN) { + if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit. + * We'll want to do layered decrypts. */ + crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath; + thishop = cpath; + if (thishop->state != CPATH_STATE_OPEN) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Relay cell before first created cell? Closing."); + return -1; + } + do { /* Remember: cpath is in forward order, that is, first hop first. */ + tor_assert(thishop); + + /* decrypt one layer */ + relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload); + + relay_header_unpack(&rh, cell->payload); + if (rh.recognized == 0) { + /* it's possibly recognized. have to check digest to be sure. */ + if (relay_digest_matches(thishop->crypto.b_digest, cell)) { + *recognized = 1; + *layer_hint = thishop; + return 0; + } + } + + thishop = thishop->next; + } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN); + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Incoming cell at client not recognized. Closing."); + return -1; + } else { + relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto; + /* We're in the middle. Encrypt one layer. */ + relay_crypt_one_payload(crypto->b_crypto, cell->payload); + } + } else /* cell_direction == CELL_DIRECTION_OUT */ { + /* We're in the middle. Decrypt one layer. */ + relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto; + + relay_crypt_one_payload(crypto->f_crypto, cell->payload); + + relay_header_unpack(&rh, cell->payload); + if (rh.recognized == 0) { + /* it's possibly recognized. have to check digest to be sure. */ + if (relay_digest_matches(crypto->f_digest, cell)) { + *recognized = 1; + return 0; + } + } + } + return 0; +} + +/** + * Encrypt a cell <b>cell</b> that we are creating, and sending outbound on + * <b>circ</b> until the hop corresponding to <b>layer_hint</b>. + * + * The integrity field and recognized field of <b>cell</b>'s relay headers + * must be set to zero. + */ +void +relay_encrypt_cell_outbound(cell_t *cell, + origin_circuit_t *circ, + crypt_path_t *layer_hint) +{ + crypt_path_t *thishop; /* counter for repeated crypts */ + relay_set_digest(layer_hint->crypto.f_digest, cell); + + thishop = layer_hint; + /* moving from farthest to nearest hop */ + do { + tor_assert(thishop); + log_debug(LD_OR,"encrypting a layer of the relay cell."); + relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload); + + thishop = thishop->prev; + } while (thishop != circ->cpath->prev); +} + +/** + * Encrypt a cell <b>cell</b> that we are creating, and sending on + * <b>circuit</b> to the origin. + * + * The integrity field and recognized field of <b>cell</b>'s relay headers + * must be set to zero. + */ +void +relay_encrypt_cell_inbound(cell_t *cell, + or_circuit_t *or_circ) +{ + relay_set_digest(or_circ->crypto.b_digest, cell); + /* encrypt one layer */ + relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload); +} + +/** + * Release all storage held inside <b>crypto</b>, but do not free + * <b>crypto</b> itself: it lives inside another object. + */ +void +relay_crypto_clear(relay_crypto_t *crypto) +{ + if (BUG(!crypto)) + return; + crypto_cipher_free(crypto->f_crypto); + crypto_cipher_free(crypto->b_crypto); + crypto_digest_free(crypto->f_digest); + crypto_digest_free(crypto->b_digest); +} + +/** Initialize <b>crypto</b> from the key material in key_data. + * + * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden + * service circuits and <b>key_data</b> must be at least + * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length. + * + * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN + * bytes, which are used as follows: + * - 20 to initialize f_digest + * - 20 to initialize b_digest + * - 16 to key f_crypto + * - 16 to key b_crypto + * + * (If 'reverse' is true, then f_XX and b_XX are swapped.) + * + * Return 0 if init was successful, else -1 if it failed. + */ +int +relay_crypto_init(relay_crypto_t *crypto, + const char *key_data, size_t key_data_len, + int reverse, int is_hs_v3) +{ + crypto_digest_t *tmp_digest; + crypto_cipher_t *tmp_crypto; + size_t digest_len = 0; + size_t cipher_key_len = 0; + + tor_assert(crypto); + tor_assert(key_data); + tor_assert(!(crypto->f_crypto || crypto->b_crypto || + crypto->f_digest || crypto->b_digest)); + + /* Basic key size validation */ + if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) { + goto err; + } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) { + goto err; + } + + /* If we are using this crypto for next gen onion services use SHA3-256, + otherwise use good ol' SHA1 */ + if (is_hs_v3) { + digest_len = DIGEST256_LEN; + cipher_key_len = CIPHER256_KEY_LEN; + crypto->f_digest = crypto_digest256_new(DIGEST_SHA3_256); + crypto->b_digest = crypto_digest256_new(DIGEST_SHA3_256); + } else { + digest_len = DIGEST_LEN; + cipher_key_len = CIPHER_KEY_LEN; + crypto->f_digest = crypto_digest_new(); + crypto->b_digest = crypto_digest_new(); + } + + tor_assert(digest_len != 0); + tor_assert(cipher_key_len != 0); + const int cipher_key_bits = (int) cipher_key_len * 8; + + crypto_digest_add_bytes(crypto->f_digest, key_data, digest_len); + crypto_digest_add_bytes(crypto->b_digest, key_data+digest_len, digest_len); + + crypto->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len), + cipher_key_bits); + if (!crypto->f_crypto) { + log_warn(LD_BUG,"Forward cipher initialization failed."); + goto err; + } + + crypto->b_crypto = crypto_cipher_new_with_bits( + key_data+(2*digest_len)+cipher_key_len, + cipher_key_bits); + if (!crypto->b_crypto) { + log_warn(LD_BUG,"Backward cipher initialization failed."); + goto err; + } + + if (reverse) { + tmp_digest = crypto->f_digest; + crypto->f_digest = crypto->b_digest; + crypto->b_digest = tmp_digest; + tmp_crypto = crypto->f_crypto; + crypto->f_crypto = crypto->b_crypto; + crypto->b_crypto = tmp_crypto; + } + + return 0; + err: + relay_crypto_clear(crypto); + return -1; +} + +/** Assert that <b>crypto</b> is valid and set. */ +void +relay_crypto_assert_ok(const relay_crypto_t *crypto) +{ + tor_assert(crypto->f_crypto); + tor_assert(crypto->b_crypto); + tor_assert(crypto->f_digest); + tor_assert(crypto->b_digest); +} + diff --git a/src/or/relay_crypto.h b/src/or/relay_crypto.h new file mode 100644 index 0000000000..66ae02cee9 --- /dev/null +++ b/src/or/relay_crypto.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file relay.h + * \brief Header file for relay.c. + **/ + +#ifndef TOR_RELAY_CRYPTO_H +#define TOR_RELAY_CRYPTO_H + +int relay_crypto_init(relay_crypto_t *crypto, + const char *key_data, size_t key_data_len, + int reverse, int is_hs_v3); + +int relay_decrypt_cell(circuit_t *circ, cell_t *cell, + cell_direction_t cell_direction, + crypt_path_t **layer_hint, char *recognized); +void relay_encrypt_cell_outbound(cell_t *cell, origin_circuit_t *or_circ, + crypt_path_t *layer_hint); +void relay_encrypt_cell_inbound(cell_t *cell, or_circuit_t *or_circ); + +void relay_crypto_clear(relay_crypto_t *crypto); + +void relay_crypto_assert_ok(const relay_crypto_t *crypto); + +#endif /* !defined(TOR_RELAY_CRYPTO_H) */ + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index e61207f2ee..74118f8f37 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -15,10 +15,13 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" -#include "hs_common.h" #include "hs_circuit.h" #include "hs_client.h" +#include "hs_common.h" #include "main.h" #include "networkstatus.h" #include "nodelist.h" @@ -29,7 +32,6 @@ #include "router.h" #include "routerlist.h" #include "routerset.h" -#include "control.h" static extend_info_t *rend_client_get_random_intro_impl( const rend_cache_entry_t *rend_query, @@ -924,8 +926,8 @@ rend_client_desc_trynow(const char *query) /* restart their timeout values, so they get a fair shake at * connecting to the hidden service. */ base_conn->timestamp_created = now; - base_conn->timestamp_lastread = now; - base_conn->timestamp_lastwritten = now; + base_conn->timestamp_last_read_allowed = now; + base_conn->timestamp_last_write_allowed = now; connection_ap_mark_as_pending_circuit(conn); } else { /* 404, or fetch didn't get that far */ diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 230da4be5c..f3fa2f64d1 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -12,20 +12,23 @@ #include "or.h" #include "circuitbuild.h" +#include "circuituse.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" +#include "hs_client.h" #include "hs_common.h" +#include "hs_intropoint.h" +#include "networkstatus.h" #include "rendclient.h" #include "rendcommon.h" #include "rendmid.h" -#include "hs_intropoint.h" -#include "hs_client.h" #include "rendservice.h" #include "rephist.h" #include "router.h" #include "routerlist.h" #include "routerparse.h" -#include "networkstatus.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -807,6 +810,11 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, tor_fragile_assert(); } + if (r == 0 && origin_circ) { + /* This was a valid cell. Count it as delivered + overhead. */ + circuit_read_valid_data(origin_circ, length); + } + if (r == -2) log_info(LD_PROTOCOL, "Dropping cell (type %d) for wrong circuit type.", command); diff --git a/src/or/rendservice.c b/src/or/rendservice.c index ac86c143d1..92c323b10d 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -16,6 +16,8 @@ #include "circuituse.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "hs_common.h" #include "hs_config.h" @@ -348,6 +350,13 @@ rend_add_service(smartlist_t *service_list, rend_service_t *service) /* The service passed all the checks */ tor_assert(s_list); smartlist_add(s_list, service); + + /* Notify that our global service list has changed only if this new service + * went into our global list. If not, when we move service from the staging + * list to the new list, a notify is triggered. */ + if (s_list == rend_service_list) { + hs_service_map_has_changed(); + } return 0; } @@ -609,6 +618,8 @@ rend_service_prune_list_impl_(void) circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED); } smartlist_free(surviving_services); + /* Notify that our global service list has changed. */ + hs_service_map_has_changed(); } /* Try to prune our main service list using the temporary one that we just @@ -959,6 +970,8 @@ rend_service_del_ephemeral(const char *service_id) } } SMARTLIST_FOREACH_END(circ); smartlist_remove(rend_service_list, s); + /* Notify that we just removed a service from our global list. */ + hs_service_map_has_changed(); rend_service_free(s); log_debug(LD_CONFIG, "Removed ephemeral Onion Service: %s", service_id); diff --git a/src/or/rephist.c b/src/or/rephist.c index 67e4d3150f..f86736e219 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -78,6 +78,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "crypto_rand.h" #include "networkstatus.h" #include "nodelist.h" #include "rephist.h" @@ -85,9 +86,8 @@ #include "routerlist.h" #include "ht.h" #include "channelpadding.h" - -#include "channelpadding.h" #include "connection_or.h" +#include "statefile.h" static void bw_arrays_init(void); static void predicted_ports_alloc(void); diff --git a/src/or/router.c b/src/or/router.c index edaa040dd7..03959e907d 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -13,6 +13,8 @@ #include "config.h" #include "connection.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "crypto_curve25519.h" #include "directory.h" #include "dirserv.h" @@ -35,6 +37,8 @@ #include "transports.h" #include "routerset.h" +#include "dirauth/mode.h" + /** * \file router.c * \brief Miscellaneous relay functionality, including RSA key maintenance, @@ -103,6 +107,64 @@ static authority_cert_t *legacy_key_certificate = NULL; * used by tor-gencert to sign new signing keys and make new key * certificates. */ +const char *format_node_description(char *buf, + const char *id_digest, + int is_named, + const char *nickname, + const tor_addr_t *addr, + uint32_t addr32h); + +/** Return a readonly string with human readable description + * of <b>err</b>. + */ +const char * +routerinfo_err_to_string(int err) +{ + switch (err) { + case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR: + return "No known exit address yet"; + case TOR_ROUTERINFO_ERROR_CANNOT_PARSE: + return "Cannot parse descriptor"; + case TOR_ROUTERINFO_ERROR_NOT_A_SERVER: + return "Not running in server mode"; + case TOR_ROUTERINFO_ERROR_DIGEST_FAILED: + return "Key digest failed"; + case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE: + return "Cannot generate descriptor"; + case TOR_ROUTERINFO_ERROR_DESC_REBUILDING: + return "Descriptor still rebuilding - not ready yet"; + } + + log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err); + tor_assert_unreached(); + + return "Unknown error"; +} + +/** Return true if we expect given error to be transient. + * Return false otherwise. + */ +int +routerinfo_err_is_transient(int err) +{ + switch (err) { + case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR: + return 1; + case TOR_ROUTERINFO_ERROR_CANNOT_PARSE: + return 1; + case TOR_ROUTERINFO_ERROR_NOT_A_SERVER: + return 0; + case TOR_ROUTERINFO_ERROR_DIGEST_FAILED: + return 0; // XXX: bug? + case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE: + return 1; + case TOR_ROUTERINFO_ERROR_DESC_REBUILDING: + return 1; + } + + return 0; +} + /** Replace the current onion key with <b>k</b>. Does not affect * lastonionkey; to update lastonionkey correctly, call rotate_onion_key(). */ @@ -1234,7 +1296,8 @@ check_whether_dirport_reachable(const or_options_t *options) /* XXX Should this be increased? */ #define MIN_BW_TO_ADVERTISE_DIRSERVER 51200 -/** Return true iff we have enough configured bandwidth to cache directory +/** Return true iff we have enough configured bandwidth to advertise or + * automatically provide directory services from cache directory * information. */ static int router_has_bandwidth_to_be_dirserver(const or_options_t *options) @@ -1257,7 +1320,7 @@ router_has_bandwidth_to_be_dirserver(const or_options_t *options) * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests. */ static int -router_should_be_directory_server(const or_options_t *options, int dir_port) +router_should_be_dirserver(const or_options_t *options, int dir_port) { static int advertising=1; /* start out assuming we will advertise */ int new_choice=1; @@ -1362,7 +1425,7 @@ decide_to_advertise_dir_impl(const or_options_t *options, /* Part two: consider config options that could make us choose to * publish or not publish that the user might find surprising. */ - return router_should_be_directory_server(options, dir_port); + return router_should_be_dirserver(options, dir_port); } /** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to @@ -1370,7 +1433,7 @@ decide_to_advertise_dir_impl(const or_options_t *options, * DirPort we want to advertise. */ static int -decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) +router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port) { /* supports_tunnelled_dir_requests is not relevant, pass 0 */ return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0; @@ -1380,7 +1443,7 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) * advertise the fact that we support begindir requests, else return 1. */ static int -decide_to_advertise_begindir(const or_options_t *options, +router_should_advertise_begindir(const or_options_t *options, int supports_tunnelled_dir_requests) { /* dir_port is not relevant, pass 0 */ @@ -1413,26 +1476,17 @@ extend_info_from_router(const routerinfo_t *r) &ap.addr, ap.port); } -/** Some time has passed, or we just got new directory information. - * See if we currently believe our ORPort or DirPort to be - * unreachable. If so, launch a new test for it. - * - * For ORPort, we simply try making a circuit that ends at ourselves. - * Success is noticed in onionskin_answer(). - * - * For DirPort, we make a connection via Tor to our DirPort and ask - * for our own server descriptor. - * Success is noticed in connection_dir_client_reached_eof(). +/**See if we currently believe our ORPort or DirPort to be + * unreachable. If so, return 1 else return 0. */ -void -consider_testing_reachability(int test_or, int test_dir) +static int +router_should_check_reachability(int test_or, int test_dir) { const routerinfo_t *me = router_get_my_routerinfo(); const or_options_t *options = get_options(); - int orport_reachable = check_whether_orport_reachable(options); - tor_addr_t addr; + if (!me) - return; + return 0; if (routerset_contains_router(options->ExcludeNodes, me, -1) && options->StrictNodes) { @@ -1447,43 +1501,66 @@ consider_testing_reachability(int test_or, int test_dir) "We cannot learn whether we are usable, and will not " "be able to advertise ourself."); } - return; + return 0; } + return 1; +} + +/** Some time has passed, or we just got new directory information. + * See if we currently believe our ORPort or DirPort to be + * unreachable. If so, launch a new test for it. + * + * For ORPort, we simply try making a circuit that ends at ourselves. + * Success is noticed in onionskin_answer(). + * + * For DirPort, we make a connection via Tor to our DirPort and ask + * for our own server descriptor. + * Success is noticed in connection_dir_client_reached_eof(). + */ +void +router_do_reachability_checks(int test_or, int test_dir) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); + int orport_reachable = check_whether_orport_reachable(options); + tor_addr_t addr; + + if (router_should_check_reachability(test_or, test_dir)) { + if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { + extend_info_t *ei = extend_info_from_router(me); + /* XXX IPv6 self testing */ + log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", + !orport_reachable ? "reachability" : "bandwidth", + fmt_addr32(me->addr), me->or_port); + circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, + CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); + extend_info_free(ei); + } - if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { - extend_info_t *ei = extend_info_from_router(me); /* XXX IPv6 self testing */ - log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", - !orport_reachable ? "reachability" : "bandwidth", - fmt_addr32(me->addr), me->or_port); - circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, - CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); - extend_info_free(ei); - } - - /* XXX IPv6 self testing */ - tor_addr_from_ipv4h(&addr, me->addr); - if (test_dir && !check_whether_dirport_reachable(options) && - !connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, me->dir_port, - DIR_PURPOSE_FETCH_SERVERDESC)) { - tor_addr_port_t my_orport, my_dirport; - memcpy(&my_orport.addr, &addr, sizeof(addr)); - memcpy(&my_dirport.addr, &addr, sizeof(addr)); - my_orport.port = me->or_port; - my_dirport.port = me->dir_port; - /* ask myself, via tor, for my server descriptor. */ - directory_request_t *req = - directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); - directory_request_set_or_addr_port(req, &my_orport); - directory_request_set_dir_addr_port(req, &my_dirport); - directory_request_set_directory_id_digest(req, + tor_addr_from_ipv4h(&addr, me->addr); + if (test_dir && !check_whether_dirport_reachable(options) && + !connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, me->dir_port, + DIR_PURPOSE_FETCH_SERVERDESC)) { + tor_addr_port_t my_orport, my_dirport; + memcpy(&my_orport.addr, &addr, sizeof(addr)); + memcpy(&my_dirport.addr, &addr, sizeof(addr)); + my_orport.port = me->or_port; + my_dirport.port = me->dir_port; + /* ask myself, via tor, for my server descriptor. */ + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &my_orport); + directory_request_set_dir_addr_port(req, &my_dirport); + directory_request_set_directory_id_digest(req, me->cache_info.identity_digest); - // ask via an anon circuit, connecting to our dirport. - directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); - directory_request_set_resource(req, "authority.z"); - directory_initiate_request(req); - directory_request_free(req); + // ask via an anon circuit, connecting to our dirport. + directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); + directory_request_set_resource(req, "authority.z"); + directory_initiate_request(req); + directory_request_free(req); + } } } @@ -1528,7 +1605,7 @@ router_dirport_found_reachable(void) && check_whether_orport_reachable(options) ? " Publishing server descriptor." : ""); can_reach_dir_port = 1; - if (decide_to_advertise_dirport(options, me->dir_port)) { + if (router_should_advertise_dirport(options, me->dir_port)) { mark_my_descriptor_dirty("DirPort found reachable"); /* This is a significant enough change to upload immediately, * at least in a test network */ @@ -1573,30 +1650,31 @@ router_perform_bandwidth_test(int num_circs, time_t now) } } -/** Return true iff our network is in some sense disabled: either we're - * hibernating, entering hibernation, or the network is turned off with - * DisableNetwork. */ +/** Return true iff our network is in some sense disabled or shutting down: + * either we're hibernating, entering hibernation, or the network is turned + * off with DisableNetwork. */ int net_is_disabled(void) { return get_options()->DisableNetwork || we_are_hibernating(); } -/** Return true iff we believe ourselves to be an authoritative - * directory server. - */ +/** Return true iff our network is in some sense "completely disabled" either + * we're fully hibernating or the network is turned off with + * DisableNetwork. */ int -authdir_mode(const or_options_t *options) +net_is_completely_disabled(void) { - return options->AuthoritativeDir != 0; + return get_options()->DisableNetwork || we_are_fully_hibernating(); } -/** Return true iff we believe ourselves to be a v3 authoritative + +/** Return true iff we believe ourselves to be an authoritative * directory server. */ int -authdir_mode_v3(const or_options_t *options) +authdir_mode(const or_options_t *options) { - return authdir_mode(options) && options->V3AuthoritativeDir != 0; + return options->AuthoritativeDir != 0; } /** Return true iff we are an authoritative directory server that is * authoritative about receiving and serving descriptors of type @@ -1999,10 +2077,43 @@ router_is_me(const routerinfo_t *router) MOCK_IMPL(const routerinfo_t *, router_get_my_routerinfo,(void)) { - if (!server_mode(get_options())) + return router_get_my_routerinfo_with_err(NULL); +} + +/** Return routerinfo of this OR. Rebuild it from + * scratch if needed. Set <b>*err</b> to 0 on success or to + * appropriate TOR_ROUTERINFO_ERROR_* value on failure. + */ +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo_with_err,(int *err)) +{ + if (!server_mode(get_options())) { + if (err) + *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER; + return NULL; - if (router_rebuild_descriptor(0)) + } + + if (!desc_clean_since) { + int rebuild_err = router_rebuild_descriptor(0); + if (rebuild_err < 0) { + if (err) + *err = rebuild_err; + + return NULL; + } + } + + if (!desc_routerinfo) { + if (err) + *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING; + return NULL; + } + + if (err) + *err = 0; + return desc_routerinfo; } @@ -2179,7 +2290,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (router_pick_published_address(options, &addr, 0) < 0) { log_warn(LD_CONFIG, "Don't know my address while generating descriptor"); - return -1; + return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; } /* Log a message if the address in the descriptor doesn't match the ORPort @@ -2235,7 +2346,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (crypto_pk_get_digest(ri->identity_pkey, ri->cache_info.identity_digest)<0) { routerinfo_free(ri); - return -1; + return TOR_ROUTERINFO_ERROR_DIGEST_FAILED; } ri->cache_info.signing_key_cert = tor_cert_dup(get_master_signing_key_cert()); @@ -2251,6 +2362,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) /* and compute ri->bandwidthburst similarly */ ri->bandwidthburst = get_effective_bwburst(options); + /* Report bandwidth, unless we're hibernating or shutting down */ ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); if (dns_seems_to_be_broken() || has_dns_init_failed()) { @@ -2368,7 +2480,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) log_warn(LD_BUG, "Couldn't generate router descriptor."); routerinfo_free(ri); extrainfo_free(ei); - return -1; + return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE; } ri->cache_info.signed_descriptor_len = strlen(ri->cache_info.signed_descriptor_body); @@ -2411,6 +2523,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) int router_rebuild_descriptor(int force) { + int err = 0; routerinfo_t *ri; extrainfo_t *ei; uint32_t addr; @@ -2425,13 +2538,14 @@ router_rebuild_descriptor(int force) * learn that it's time to try again when ip_address_changed() * marks it dirty. */ desc_clean_since = time(NULL); - return -1; + return TOR_ROUTERINFO_ERROR_DESC_REBUILDING; } log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : ""); - if (router_build_fresh_descriptor(&ri, &ei) < 0) { - return -1; + err = router_build_fresh_descriptor(&ri, &ei); + if (err < 0) { + return err; } routerinfo_free(desc_routerinfo); @@ -2544,6 +2658,9 @@ check_descriptor_bandwidth_changed(time_t now) return; prev = my_ri->bandwidthcapacity; + + /* Consider ourselves to have zero bandwidth if we're hibernating or + * shutting down. */ cur = hibernating ? 0 : rep_hist_bandwidth_assess(); if ((prev != cur && (!prev || !cur)) || cur > (prev * BANDWIDTH_CHANGE_FACTOR) || @@ -2947,14 +3064,14 @@ router_dump_router_to_string(routerinfo_t *router, router->nickname, address, router->or_port, - decide_to_advertise_dirport(options, router->dir_port), + router_should_advertise_dirport(options, router->dir_port), ed_cert_line ? ed_cert_line : "", extra_or_address ? extra_or_address : "", router->platform, proto_line, published, fingerprint, - stats_n_seconds_working, + get_uptime(), (int) router->bandwidthrate, (int) router->bandwidthburst, (int) router->bandwidthcapacity, @@ -3021,7 +3138,7 @@ router_dump_router_to_string(routerinfo_t *router, tor_free(p6); } - if (decide_to_advertise_begindir(options, + if (router_should_advertise_begindir(options, router->supports_tunnelled_dir_requests)) { smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); } @@ -3470,6 +3587,15 @@ is_legal_hexdigest(const char *s) strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN); } +/** + * Longest allowed output of format_node_description, plus 1 character for + * NUL. This allows space for: + * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" + * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" + * plus a terminating NUL. + */ +#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN) + /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to * hold a human-readable description of a node with identity digest * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>, @@ -3515,15 +3641,16 @@ format_node_description(char *buf, return buf; } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>ri</b>. - * +/** Return a human-readable description of the routerinfo_t <b>ri</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -router_get_description(char *buf, const routerinfo_t *ri) +router_describe(const routerinfo_t *ri) { + static char buf[NODE_DESC_BUF_LEN]; + if (!ri) return "<null>"; return format_node_description(buf, @@ -3534,14 +3661,15 @@ router_get_description(char *buf, const routerinfo_t *ri) ri->addr); } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>node</b>. +/** Return a human-readable description of the node_t <b>node</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -node_get_description(char *buf, const node_t *node) +node_describe(const node_t *node) { + static char buf[NODE_DESC_BUF_LEN]; const char *nickname = NULL; uint32_t addr32h = 0; int is_named = 0; @@ -3566,14 +3694,16 @@ node_get_description(char *buf, const node_t *node) addr32h); } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>rs</b>. +/** Return a human-readable description of the routerstatus_t <b>rs</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -routerstatus_get_description(char *buf, const routerstatus_t *rs) +routerstatus_describe(const routerstatus_t *rs) { + static char buf[NODE_DESC_BUF_LEN]; + if (!rs) return "<null>"; return format_node_description(buf, @@ -3584,14 +3714,16 @@ routerstatus_get_description(char *buf, const routerstatus_t *rs) rs->addr); } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>ei</b>. +/** Return a human-readable description of the extend_info_t <b>ei</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -extend_info_get_description(char *buf, const extend_info_t *ei) +extend_info_describe(const extend_info_t *ei) { + static char buf[NODE_DESC_BUF_LEN]; + if (!ei) return "<null>"; return format_node_description(buf, @@ -3602,54 +3734,6 @@ extend_info_get_description(char *buf, const extend_info_t *ei) 0); } -/** Return a human-readable description of the routerinfo_t <b>ri</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -router_describe(const routerinfo_t *ri) -{ - static char buf[NODE_DESC_BUF_LEN]; - return router_get_description(buf, ri); -} - -/** Return a human-readable description of the node_t <b>node</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -node_describe(const node_t *node) -{ - static char buf[NODE_DESC_BUF_LEN]; - return node_get_description(buf, node); -} - -/** Return a human-readable description of the routerstatus_t <b>rs</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -routerstatus_describe(const routerstatus_t *rs) -{ - static char buf[NODE_DESC_BUF_LEN]; - return routerstatus_get_description(buf, rs); -} - -/** Return a human-readable description of the extend_info_t <b>ei</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -extend_info_describe(const extend_info_t *ei) -{ - static char buf[NODE_DESC_BUF_LEN]; - return extend_info_get_description(buf, ei); -} - /** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the * verbose representation of the identity of <b>router</b>. The format is: * A dollar sign. diff --git a/src/or/router.h b/src/or/router.h index 696e983662..752f2f2dbe 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -14,6 +14,13 @@ #include "testsupport.h" +#define TOR_ROUTERINFO_ERROR_NO_EXT_ADDR (-1) +#define TOR_ROUTERINFO_ERROR_CANNOT_PARSE (-2) +#define TOR_ROUTERINFO_ERROR_NOT_A_SERVER (-3) +#define TOR_ROUTERINFO_ERROR_DIGEST_FAILED (-4) +#define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5) +#define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6) + crypto_pk_t *get_onion_key(void); time_t get_onion_key_set_at(void); void set_server_identity_key(crypto_pk_t *k); @@ -47,15 +54,15 @@ int init_keys_client(void); int check_whether_orport_reachable(const or_options_t *options); int check_whether_dirport_reachable(const or_options_t *options); int dir_server_mode(const or_options_t *options); -void consider_testing_reachability(int test_or, int test_dir); +void router_do_reachability_checks(int test_or, int test_dir); void router_orport_found_reachable(void); void router_dirport_found_reachable(void); void router_perform_bandwidth_test(int num_circs, time_t now); int net_is_disabled(void); +int net_is_completely_disabled(void); int authdir_mode(const or_options_t *options); -int authdir_mode_v3(const or_options_t *options); int authdir_mode_handles_descs(const or_options_t *options, int purpose); int authdir_mode_publishes_statuses(const or_options_t *options); int authdir_mode_tests_reachability(const or_options_t *options); @@ -86,6 +93,7 @@ void router_new_address_suggestion(const char *suggestion, int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port); MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void)); MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); +MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo_with_err,(int *err)); extrainfo_t *router_get_my_extrainfo(void); const char *router_get_my_descriptor(void); const char *router_get_descriptor_gen_reason(void); @@ -123,29 +131,14 @@ int is_legal_nickname(const char *s); int is_legal_nickname_or_hexdigest(const char *s); int is_legal_hexdigest(const char *s); -/** - * Longest allowed output of format_node_description, plus 1 character for - * NUL. This allows space for: - * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" - * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" - * plus a terminating NUL. - */ -#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN) -const char *format_node_description(char *buf, - const char *id_digest, - int is_named, - const char *nickname, - const tor_addr_t *addr, - uint32_t addr32h); -const char *router_get_description(char *buf, const routerinfo_t *ri); -const char *node_get_description(char *buf, const node_t *node); -const char *routerstatus_get_description(char *buf, const routerstatus_t *rs); -const char *extend_info_get_description(char *buf, const extend_info_t *ei); const char *router_describe(const routerinfo_t *ri); const char *node_describe(const node_t *node); const char *routerstatus_describe(const routerstatus_t *ri); const char *extend_info_describe(const extend_info_t *ei); +const char *routerinfo_err_to_string(int err); +int routerinfo_err_is_transient(int err); + void router_get_verbose_nickname(char *buf, const routerinfo_t *router); void router_reset_warnings(void); void router_reset_reachability(void); diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c index 1933aaf4b6..43460da8cc 100644 --- a/src/or/routerkeys.c +++ b/src/or/routerkeys.c @@ -16,6 +16,7 @@ #include "or.h" #include "config.h" +#include "crypto_util.h" #include "router.h" #include "crypto_pwbox.h" #include "routerkeys.h" diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 203895c867..2f27af7f06 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -99,9 +99,9 @@ #include "config.h" #include "connection.h" #include "control.h" +#include "crypto_rand.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "entrynodes.h" #include "fp_pair.h" #include "geoip.h" @@ -122,6 +122,9 @@ #include "sandbox.h" #include "torcert.h" +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" + // #define DEBUG_ROUTERLIST /****************************************************************************/ @@ -2761,9 +2764,8 @@ frac_nodes_with_descriptors(const smartlist_t *sl, if (node_has_any_descriptor(node)) n_with_descs++; }); - tor_free(bandwidths); - return ((double)n_with_descs) / (double)smartlist_len(sl); + return ((double)n_with_descs) / smartlist_len(sl); } present = 0.0; diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 6f82859e61..e82ecec5b7 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -56,30 +56,34 @@ #define ROUTERPARSE_PRIVATE #include "or.h" -#include "config.h" #include "circuitstats.h" +#include "config.h" +#include "crypto_util.h" +#include "dirauth/shared_random.h" #include "dirserv.h" -#include "dirvote.h" +#include "entrynodes.h" +#include "memarea.h" +#include "microdesc.h" +#include "networkstatus.h" #include "parsecommon.h" #include "policies.h" #include "protover.h" #include "rendcommon.h" -#include "router.h" -#include "routerlist.h" -#include "memarea.h" -#include "microdesc.h" -#include "networkstatus.h" #include "rephist.h" +#include "router.h" #include "routerkeys.h" +#include "routerlist.h" #include "routerparse.h" -#include "entrynodes.h" -#include "torcert.h" #include "sandbox.h" -#include "shared_random.h" +#include "shared_random_client.h" +#include "torcert.h" +#include "voting_schedule.h" #undef log #include <math.h> +#include "dirauth/dirvote.h" + /****************************************************************************/ /** List of tokens recognized in router descriptors */ @@ -2743,8 +2747,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, /* These are implied true by having been included in a consensus made * with a given method */ rs->is_flagged_running = 1; /* Starting with consensus method 4. */ - if (consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES) - rs->is_valid = 1; + rs->is_valid = 1; /* Starting with consensus method 24. */ } { const char *protocols = NULL, *version = NULL; @@ -3283,60 +3286,6 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) return valid; } -/** Parse and extract all SR commits from <b>tokens</b> and place them in - * <b>ns</b>. */ -static void -extract_shared_random_commits(networkstatus_t *ns, smartlist_t *tokens) -{ - smartlist_t *chunks = NULL; - - tor_assert(ns); - tor_assert(tokens); - /* Commits are only present in a vote. */ - tor_assert(ns->type == NS_TYPE_VOTE); - - ns->sr_info.commits = smartlist_new(); - - smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT); - /* It's normal that a vote might contain no commits even if it participates - * in the SR protocol. Don't treat it as an error. */ - if (commits == NULL) { - goto end; - } - - /* Parse the commit. We do NO validation of number of arguments or ordering - * for forward compatibility, it's the parse commit job to inform us if it's - * supported or not. */ - chunks = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) { - /* Extract all arguments and put them in the chunks list. */ - for (int i = 0; i < tok->n_args; i++) { - smartlist_add(chunks, tok->args[i]); - } - sr_commit_t *commit = sr_parse_commit(chunks); - smartlist_clear(chunks); - if (commit == NULL) { - /* Get voter identity so we can warn that this dirauth vote contains - * commit we can't parse. */ - networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); - tor_assert(voter); - log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.", - escaped(tok->object_body), - hex_str(voter->identity_digest, - sizeof(voter->identity_digest))); - /* Commitment couldn't be parsed. Continue onto the next commit because - * this one could be unsupported for instance. */ - continue; - } - /* Add newly created commit object to the vote. */ - smartlist_add(ns->sr_info.commits, commit); - } SMARTLIST_FOREACH_END(tok); - - end: - smartlist_free(chunks); - smartlist_free(commits); -} - /** Check if a shared random value of type <b>srv_type</b> is in * <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return * -1 on failure, 0 on success. The resulting srv is allocated on the heap and @@ -3774,13 +3723,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* If this is a vote document, check if information about the shared randomness protocol is included, and extract it. */ if (ns->type == NS_TYPE_VOTE) { - /* Does this authority participates in the SR protocol? */ - tok = find_opt_by_keyword(tokens, K_SR_FLAG); - if (tok) { - ns->sr_info.participate = 1; - /* Get the SR commitments and reveals from the vote. */ - extract_shared_random_commits(ns, tokens); - } + dirvote_parse_sr_commits(ns, tokens); } /* For both a vote and consensus, extract the shared random values. */ if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) { @@ -3970,7 +3913,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } - if (voter_get_sig_by_algorithm(v, sig->alg)) { + if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) { /* We already parsed a vote with this algorithm from this voter. Use the first one. */ log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " diff --git a/src/or/scheduler.c b/src/or/scheduler.c index 382b3e3ca9..da894294bf 100644 --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@ -13,8 +13,6 @@ #define TOR_CHANNEL_INTERNAL_ #include "channeltls.h" -#include <event2/event.h> - /** * \file scheduler.c * \brief Channel scheduling system: decides which channels should send and @@ -169,7 +167,7 @@ STATIC smartlist_t *channels_pending = NULL; * This event runs the scheduler from its callback, and is manually * activated whenever a channel enters open for writes/cells to send. */ -STATIC struct event *run_sched_ev = NULL; +STATIC struct mainloop_event_t *run_sched_ev = NULL; static int have_logged_kist_suddenly_disabled = 0; @@ -203,10 +201,9 @@ get_scheduler_type_string(scheduler_types_t type) * if any scheduling work was created during the event loop. */ static void -scheduler_evt_callback(evutil_socket_t fd, short events, void *arg) +scheduler_evt_callback(mainloop_event_t *event, void *arg) { - (void) fd; - (void) events; + (void) event; (void) arg; log_debug(LD_SCHED, "Scheduler event callback called"); @@ -487,10 +484,7 @@ scheduler_free_all(void) log_debug(LD_SCHED, "Shutting down scheduler"); if (run_sched_ev) { - if (event_del(run_sched_ev) < 0) { - log_warn(LD_BUG, "Problem deleting run_sched_ev"); - } - tor_event_free(run_sched_ev); + mainloop_event_free(run_sched_ev); run_sched_ev = NULL; } @@ -589,7 +583,7 @@ scheduler_ev_add(const struct timeval *next_run) { tor_assert(run_sched_ev); tor_assert(next_run); - if (BUG(event_add(run_sched_ev, next_run) < 0)) { + if (BUG(mainloop_event_schedule(run_sched_ev, next_run) < 0)) { log_warn(LD_SCHED, "Adding to libevent failed. Next run time was set to: " "%ld.%06ld", next_run->tv_sec, (long)next_run->tv_usec); return; @@ -598,10 +592,10 @@ scheduler_ev_add(const struct timeval *next_run) /** Make the scheduler event active with the given flags. */ void -scheduler_ev_active(int flags) +scheduler_ev_active(void) { tor_assert(run_sched_ev); - event_active(run_sched_ev, flags, 1); + mainloop_event_activate(run_sched_ev); } /* @@ -618,11 +612,10 @@ scheduler_init(void) IF_BUG_ONCE(!!run_sched_ev) { log_warn(LD_SCHED, "We should not already have a libevent scheduler event." "I'll clean the old one up, but this is odd."); - tor_event_free(run_sched_ev); + mainloop_event_free(run_sched_ev); run_sched_ev = NULL; } - run_sched_ev = tor_event_new(tor_libevent_get_base(), -1, - 0, scheduler_evt_callback, NULL); + run_sched_ev = mainloop_event_new(scheduler_evt_callback, NULL); channels_pending = smartlist_new(); set_scheduler(); diff --git a/src/or/scheduler.h b/src/or/scheduler.h index aeba9e2b75..08b02e286f 100644 --- a/src/or/scheduler.h +++ b/src/or/scheduler.h @@ -155,12 +155,12 @@ void scheduler_bug_occurred(const channel_t *chan); smartlist_t *get_channels_pending(void); MOCK_DECL(int, scheduler_compare_channels, (const void *c1_v, const void *c2_v)); -void scheduler_ev_active(int flags); +void scheduler_ev_active(void); void scheduler_ev_add(const struct timeval *next_run); #ifdef TOR_UNIT_TESTS extern smartlist_t *channels_pending; -extern struct event *run_sched_ev; +extern struct mainloop_event_t *run_sched_ev; extern const scheduler_t *the_scheduler; void scheduler_touch_channel(channel_t *chan); #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c index 6d6490077d..c6e9b72c48 100644 --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@ -3,8 +3,6 @@ #define SCHEDULER_KIST_PRIVATE -#include <event2/event.h> - #include "or.h" #include "buffers.h" #include "config.h" @@ -553,7 +551,7 @@ kist_scheduler_schedule(void) /* Re-adding an event reschedules it. It does not duplicate it. */ scheduler_ev_add(&next_run); } else { - scheduler_ev_active(EV_TIMEOUT); + scheduler_ev_active(); } } diff --git a/src/or/scheduler_vanilla.c b/src/or/scheduler_vanilla.c index 7a83b9da18..b674d8256c 100644 --- a/src/or/scheduler_vanilla.c +++ b/src/or/scheduler_vanilla.c @@ -1,8 +1,6 @@ /* Copyright (c) 2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#include <event2/event.h> - #include "or.h" #include "config.h" #define TOR_CHANNEL_INTERNAL_ @@ -42,7 +40,7 @@ vanilla_scheduler_schedule(void) } /* Activate our event so it can process channels. */ - scheduler_ev_active(EV_TIMEOUT); + scheduler_ev_active(); } static void diff --git a/src/or/shared_random_client.c b/src/or/shared_random_client.c new file mode 100644 index 0000000000..3aef83cef4 --- /dev/null +++ b/src/or/shared_random_client.c @@ -0,0 +1,259 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shared_random_client.c + * \brief This file contains functions that are from the shared random + * subsystem but used by many part of tor. The full feature is built + * as part of the dirauth module. + **/ + +#define SHARED_RANDOM_CLIENT_PRIVATE +#include "shared_random_client.h" + +#include "config.h" +#include "voting_schedule.h" +#include "networkstatus.h" +#include "util.h" +#include "util_format.h" + +/* Convert a given srv object to a string for the control port. This doesn't + * fail and the srv object MUST be valid. */ +static char * +srv_to_control_string(const sr_srv_t *srv) +{ + char *srv_str; + char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1]; + tor_assert(srv); + + sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv); + tor_asprintf(&srv_str, "%s", srv_hash_encoded); + return srv_str; +} + +/* Return the voting interval of the tor vote subsystem. */ +int +get_voting_interval(void) +{ + int interval; + networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL)); + + if (consensus) { + interval = (int)(consensus->fresh_until - consensus->valid_after); + } else { + /* Same for both a testing and real network. We voluntarily ignore the + * InitialVotingInterval since it complexifies things and it doesn't + * affect the SR protocol. */ + interval = get_options()->V3AuthVotingInterval; + } + tor_assert(interval > 0); + return interval; +} + +/* Given the time <b>now</b>, return the start time of the current round of + * the SR protocol. For example, if it's 23:47:08, the current round thus + * started at 23:47:00 for a voting interval of 10 seconds. */ +time_t +get_start_time_of_current_round(void) +{ + const or_options_t *options = get_options(); + int voting_interval = get_voting_interval(); + /* First, get the start time of the next round */ + time_t next_start = voting_schedule_get_next_valid_after_time(); + /* Now roll back next_start by a voting interval to find the start time of + the current round. */ + time_t curr_start = voting_schedule_get_start_of_next_interval( + next_start - voting_interval - 1, + voting_interval, + options->TestingV3AuthVotingStartOffset); + return curr_start; +} + +/* + * Public API + */ + +/* Encode the given shared random value and put it in dst. Destination + * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */ +void +sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv) +{ + int ret; + /* Extra byte for the NULL terminated char. */ + char buf[SR_SRV_VALUE_BASE64_LEN + 1]; + + tor_assert(dst); + tor_assert(srv); + tor_assert(dst_len >= sizeof(buf)); + + ret = base64_encode(buf, sizeof(buf), (const char *) srv->value, + sizeof(srv->value), 0); + /* Always expect the full length without the NULL byte. */ + tor_assert(ret == (sizeof(buf) - 1)); + tor_assert(ret <= (int) dst_len); + strlcpy(dst, buf, dst_len); +} + +/* Return the current SRV string representation for the control port. Return a + * newly allocated string on success containing the value else "" if not found + * or if we don't have a valid consensus yet. */ +char * +sr_get_current_for_control(void) +{ + char *srv_str; + const networkstatus_t *c = networkstatus_get_latest_consensus(); + if (c && c->sr_info.current_srv) { + srv_str = srv_to_control_string(c->sr_info.current_srv); + } else { + srv_str = tor_strdup(""); + } + return srv_str; +} + +/* Return the previous SRV string representation for the control port. Return + * a newly allocated string on success containing the value else "" if not + * found or if we don't have a valid consensus yet. */ +char * +sr_get_previous_for_control(void) +{ + char *srv_str; + const networkstatus_t *c = networkstatus_get_latest_consensus(); + if (c && c->sr_info.previous_srv) { + srv_str = srv_to_control_string(c->sr_info.previous_srv); + } else { + srv_str = tor_strdup(""); + } + return srv_str; +} + +/* Return current shared random value from the latest consensus. Caller can + * NOT keep a reference to the returned pointer. Return NULL if none. */ +const sr_srv_t * +sr_get_current(const networkstatus_t *ns) +{ + const networkstatus_t *consensus; + + /* Use provided ns else get a live one */ + if (ns) { + consensus = ns; + } else { + consensus = networkstatus_get_live_consensus(approx_time()); + } + /* Ideally we would never be asked for an SRV without a live consensus. Make + * sure this assumption is correct. */ + tor_assert_nonfatal(consensus); + + if (consensus) { + return consensus->sr_info.current_srv; + } + return NULL; +} + +/* Return previous shared random value from the latest consensus. Caller can + * NOT keep a reference to the returned pointer. Return NULL if none. */ +const sr_srv_t * +sr_get_previous(const networkstatus_t *ns) +{ + const networkstatus_t *consensus; + + /* Use provided ns else get a live one */ + if (ns) { + consensus = ns; + } else { + consensus = networkstatus_get_live_consensus(approx_time()); + } + /* Ideally we would never be asked for an SRV without a live consensus. Make + * sure this assumption is correct. */ + tor_assert_nonfatal(consensus); + + if (consensus) { + return consensus->sr_info.previous_srv; + } + return NULL; +} + +/* Parse a list of arguments from a SRV value either from a vote, consensus + * or from our disk state and return a newly allocated srv object. NULL is + * returned on error. + * + * The arguments' order: + * num_reveals, value + */ +sr_srv_t * +sr_parse_srv(const smartlist_t *args) +{ + char *value; + int ok, ret; + uint64_t num_reveals; + sr_srv_t *srv = NULL; + + tor_assert(args); + + if (smartlist_len(args) < 2) { + goto end; + } + + /* First argument is the number of reveal values */ + num_reveals = tor_parse_uint64(smartlist_get(args, 0), + 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + goto end; + } + /* Second and last argument is the shared random value it self. */ + value = smartlist_get(args, 1); + if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) { + goto end; + } + + srv = tor_malloc_zero(sizeof(*srv)); + srv->num_reveals = num_reveals; + /* We subtract one byte from the srclen because the function ignores the + * '=' character in the given buffer. This is broken but it's a documented + * behavior of the implementation. */ + ret = base64_decode((char *) srv->value, sizeof(srv->value), value, + SR_SRV_VALUE_BASE64_LEN - 1); + if (ret != sizeof(srv->value)) { + tor_free(srv); + srv = NULL; + goto end; + } + end: + return srv; +} + +/** Return the start time of the current SR protocol run. For example, if the + * time is 23/06/2017 23:47:08 and a full SR protocol run is 24 hours, this + * function should return 23/06/2017 00:00:00. */ +time_t +sr_state_get_start_time_of_current_protocol_run(time_t now) +{ + int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; + int voting_interval = get_voting_interval(); + /* Find the time the current round started. */ + time_t beginning_of_current_round = get_start_time_of_current_round(); + + /* Get current SR protocol round */ + int current_round = (now / voting_interval) % total_rounds; + + /* Get start time by subtracting the time elapsed from the beginning of the + protocol run */ + time_t time_elapsed_since_start_of_run = current_round * voting_interval; + return beginning_of_current_round - time_elapsed_since_start_of_run; +} + +/** Return the time (in seconds) it takes to complete a full SR protocol phase + * (e.g. the commit phase). */ +unsigned int +sr_state_get_phase_duration(void) +{ + return SHARED_RANDOM_N_ROUNDS * get_voting_interval(); +} + +/** Return the time (in seconds) it takes to complete a full SR protocol run */ +unsigned int +sr_state_get_protocol_run_duration(void) +{ + int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; + return total_protocol_rounds * get_voting_interval(); +} + diff --git a/src/or/shared_random_client.h b/src/or/shared_random_client.h new file mode 100644 index 0000000000..89c608d45f --- /dev/null +++ b/src/or/shared_random_client.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shared_random_client.h + * \brief Header file for shared_random_client.c. + **/ + +#ifndef TOR_SHARED_RANDOM_CLIENT_H +#define TOR_SHARED_RANDOM_CLIENT_H + +/* Dirauth module. */ +#include "dirauth/shared_random.h" + +/* Helper functions. */ +void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv); +int get_voting_interval(void); + +/* Control port functions. */ +char *sr_get_current_for_control(void); +char *sr_get_previous_for_control(void); + +/* SRV functions. */ +const sr_srv_t *sr_get_current(const networkstatus_t *ns); +const sr_srv_t *sr_get_previous(const networkstatus_t *ns); +sr_srv_t *sr_parse_srv(const smartlist_t *args); + +/* + * Shared Random State API + */ + +/* Each protocol phase has 12 rounds */ +#define SHARED_RANDOM_N_ROUNDS 12 +/* Number of phase we have in a protocol. */ +#define SHARED_RANDOM_N_PHASES 2 + +time_t sr_state_get_start_time_of_current_protocol_run(time_t now); +unsigned int sr_state_get_phase_duration(void); +unsigned int sr_state_get_protocol_run_duration(void); +time_t get_start_time_of_current_round(void); + +#ifdef TOR_UNIT_TESTS + +#endif /* TOR_UNIT_TESTS */ + +#endif /* TOR_SHARED_RANDOM_CLIENT_H */ + diff --git a/src/or/statefile.c b/src/or/statefile.c index cc114f0a2b..c81ea44e06 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -37,6 +37,7 @@ #include "control.h" #include "entrynodes.h" #include "hibernate.h" +#include "main.h" #include "rephist.h" #include "router.h" #include "sandbox.h" @@ -680,6 +681,18 @@ save_transport_to_state(const char *transport, tor_free(transport_addrport); } +/** Change the next_write time of <b>state</b> to <b>when</b>, unless the + * state is already scheduled to be written to disk earlier than <b>when</b>. + */ +void +or_state_mark_dirty(or_state_t *state, time_t when) +{ + if (state->next_write > when) { + state->next_write = when; + reschedule_or_state_save(); + } +} + STATIC void or_state_free_(or_state_t *state) { diff --git a/src/or/statefile.h b/src/or/statefile.h index b4cc4d1dc6..5aa2ca9320 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -17,6 +17,7 @@ char *get_stored_bindaddr_for_server_transport(const char *transport); int or_state_load(void); int or_state_loaded(void); void or_state_free_all(void); +void or_state_mark_dirty(or_state_t *state, time_t when); #ifdef STATEFILE_PRIVATE STATIC config_line_t *get_transport_in_state_by_name(const char *transport); diff --git a/src/or/status.c b/src/or/status.c index 4f7be164b1..4b8033d114 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -25,7 +25,6 @@ #include "main.h" #include "rephist.h" #include "hibernate.h" -#include "rephist.h" #include "statefile.h" #include "hs_stats.h" #include "hs_service.h" @@ -88,19 +87,19 @@ bytes_to_usage(uint64_t bytes) return bw_string; } -/** Log some usage info about our hidden service */ +/** Log some usage info about our onion service(s). */ static void log_onion_service_stats(void) { unsigned int num_services = hs_service_get_num_services(); - /* If there are no active hidden services, no need to print logs */ + /* If there are no active onion services, no need to print logs */ if (num_services == 0) { return; } log_notice(LD_HEARTBEAT, - "Our hidden service%s received %u v2 and %u v3 INTRODUCE2 cells " + "Our onion service%s received %u v2 and %u v3 INTRODUCE2 cells " "and attempted to launch %d rendezvous circuits.", num_services == 1 ? "" : "s", hs_stats_get_n_introduce2_v2_cells(), diff --git a/src/or/torcert.c b/src/or/torcert.c index 51935ddf72..1c5afd965a 100644 --- a/src/or/torcert.c +++ b/src/or/torcert.c @@ -27,7 +27,7 @@ #include "or.h" #include "config.h" -#include "crypto.h" +#include "crypto_util.h" #include "torcert.h" #include "ed25519_cert.h" #include "torlog.h" diff --git a/src/or/transports.c b/src/or/transports.c index b08dcd1613..614fc81da8 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -135,7 +135,7 @@ static smartlist_t *transport_list = NULL; /** Returns a transport_t struct for a transport proxy supporting the protocol <b>name</b> listening at <b>addr</b>:<b>port</b> using SOCKS version <b>socks_ver</b>. */ -static transport_t * +STATIC transport_t * transport_new(const tor_addr_t *addr, uint16_t port, const char *name, int socks_ver, const char *extra_info_args) @@ -222,8 +222,8 @@ transport_copy(const transport_t *transport) /** Returns the transport in our transport list that has the name <b>name</b>. * Else returns NULL. */ -transport_t * -transport_get_by_name(const char *name) +MOCK_IMPL(transport_t *, +transport_get_by_name,(const char *name)) { tor_assert(name); @@ -1025,48 +1025,71 @@ parse_method_error(const char *line, int is_server) line+strlen(error)+1); } -/** Parses an SMETHOD <b>line</b> and if well-formed it registers the - * new transport in <b>mp</b>. */ -STATIC int -parse_smethod_line(const char *line, managed_proxy_t *mp) +/** A helper for parse_{c,s}method_line(), bootstraps its + * functionalities. If <b>is_smethod</b> is true then the + * the line to parse is a SMETHOD line otherwise it is a + * CMETHOD line*/ +static int +parse_method_line_helper(const char *line, + managed_proxy_t *mp, + int is_smethod) { + int item_index = 0; int r; - smartlist_t *items = NULL; - char *method_name=NULL; + char *transport_name=NULL; char *args_string=NULL; char *addrport=NULL; - tor_addr_t tor_addr; + int socks_ver=PROXY_NONE; char *address=NULL; uint16_t port = 0; + const char *method_str = is_smethod ? PROTO_SMETHOD : PROTO_CMETHOD; + const int min_args_count = is_smethod ? 3 : 4; + + tor_addr_t tor_addr; transport_t *transport=NULL; + smartlist_t *items= smartlist_new(); - items = smartlist_new(); smartlist_split_string(items, line, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - if (smartlist_len(items) < 3) { - log_warn(LD_CONFIG, "Server managed proxy sent us a SMETHOD line " - "with too few arguments."); + if (smartlist_len(items) < min_args_count) { + log_warn(LD_CONFIG, "Managed proxy sent us a %s line " + "with too few arguments.", method_str); goto err; } - /* Example of legit SMETHOD line: - SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ - - tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD)); + tor_assert(!strcmp(smartlist_get(items, item_index),method_str)); + ++item_index; - method_name = smartlist_get(items,1); - if (!string_is_C_identifier(method_name)) { + transport_name = smartlist_get(items,item_index); + ++item_index; + if (!string_is_C_identifier(transport_name)) { log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", - method_name); + transport_name); goto err; } - addrport = smartlist_get(items, 2); + /** Check for the proxy method sent to us in CMETHOD line. */ + if (!is_smethod) { + const char *socks_ver_str = smartlist_get(items,item_index); + ++item_index; + + if (!strcmp(socks_ver_str,"socks4")) { + socks_ver = PROXY_SOCKS4; + } else if (!strcmp(socks_ver_str,"socks5")) { + socks_ver = PROXY_SOCKS5; + } else { + log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol " + "we don't recognize. (%s)", socks_ver_str); + goto err; + } + } + + addrport = smartlist_get(items, item_index); + ++item_index; if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); + log_warn(LD_CONFIG, "Error parsing transport address '%s'", addrport); goto err; } @@ -1081,10 +1104,11 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } - if (smartlist_len(items) > 3) { + /** Check for options in the SMETHOD line. */ + if (is_smethod && smartlist_len(items) > min_args_count) { /* Seems like there are also some [options] in the SMETHOD line. Let's see if we can parse them. */ - char *options_string = smartlist_get(items, 3); + char *options_string = smartlist_get(items, item_index); log_debug(LD_CONFIG, "Got options_string: %s", options_string); if (!strcmpstart(options_string, "ARGS:")) { args_string = options_string+strlen("ARGS:"); @@ -1092,15 +1116,20 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) } } - transport = transport_new(&tor_addr, port, method_name, - PROXY_NONE, args_string); + transport = transport_new(&tor_addr, port, transport_name, + socks_ver, args_string); smartlist_add(mp->transports, transport); - /* For now, notify the user so that they know where the server - transport is listening. */ - log_info(LD_CONFIG, "Server transport %s at %s:%d.", - method_name, address, (int)port); + /** Logs info about line parsing success for client or server */ + if (is_smethod) { + log_info(LD_CONFIG, "Server transport %s at %s:%d.", + transport_name, address, (int)port); + } else { + log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. " + "Attached to managed proxy.", + transport_name, address, (int)port, socks_ver); + } r=0; goto done; @@ -1115,93 +1144,24 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) return r; } +/** Parses an SMETHOD <b>line</b> and if well-formed it registers the + * new transport in <b>mp</b>. */ +STATIC int +parse_smethod_line(const char *line, managed_proxy_t *mp) +{ + /* Example of legit SMETHOD line: + SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ + return parse_method_line_helper(line, mp, 1); +} + /** Parses a CMETHOD <b>line</b>, and if well-formed it registers * the new transport in <b>mp</b>. */ STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp) { - int r; - smartlist_t *items = NULL; - - char *method_name=NULL; - - char *socks_ver_str=NULL; - int socks_ver=PROXY_NONE; - - char *addrport=NULL; - tor_addr_t tor_addr; - char *address=NULL; - uint16_t port = 0; - - transport_t *transport=NULL; - - items = smartlist_new(); - smartlist_split_string(items, line, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - if (smartlist_len(items) < 4) { - log_warn(LD_CONFIG, "Client managed proxy sent us a CMETHOD line " - "with too few arguments."); - goto err; - } - - tor_assert(!strcmp(smartlist_get(items,0),PROTO_CMETHOD)); - - method_name = smartlist_get(items,1); - if (!string_is_C_identifier(method_name)) { - log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", - method_name); - goto err; - } - - socks_ver_str = smartlist_get(items,2); - - if (!strcmp(socks_ver_str,"socks4")) { - socks_ver = PROXY_SOCKS4; - } else if (!strcmp(socks_ver_str,"socks5")) { - socks_ver = PROXY_SOCKS5; - } else { - log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol " - "we don't recognize. (%s)", socks_ver_str); - goto err; - } - - addrport = smartlist_get(items, 3); - if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); - goto err; - } - - if (!port) { - log_warn(LD_CONFIG, - "Transport address '%s' has no port.", addrport); - goto err; - } - - if (tor_addr_parse(&tor_addr, address) < 0) { - log_warn(LD_CONFIG, "Error parsing transport address '%s'", address); - goto err; - } - - transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL); - - smartlist_add(mp->transports, transport); - - log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. " - "Attached to managed proxy.", - method_name, address, (int)port, socks_ver); - - r=0; - goto done; - - err: - r = -1; - - done: - SMARTLIST_FOREACH(items, char*, s, tor_free(s)); - smartlist_free(items); - tor_free(address); - return r; + /* Example of legit CMETHOD line: + CMETHOD obfs2 socks5 127.0.0.1:35713 */ + return parse_method_line_helper(line, mp, 0); } /** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */ diff --git a/src/or/transports.h b/src/or/transports.h index 1b2786472c..022b926a03 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -38,7 +38,7 @@ MOCK_DECL(int, transport_add_from_config, void transport_free_(transport_t *transport); #define transport_free(tr) FREE_AND_NULL(transport_t, transport_free_, (tr)) -transport_t *transport_get_by_name(const char *name); +MOCK_DECL(transport_t*, transport_get_by_name, (const char *name)); MOCK_DECL(void, pt_kickstart_proxy, (const smartlist_t *transport_list, char **proxy_argv, @@ -113,6 +113,9 @@ typedef struct { smartlist_t *transports; } managed_proxy_t; +STATIC transport_t *transport_new(const tor_addr_t *addr, uint16_t port, + const char *name, int socks_ver, + const char *extra_info_args); STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp); STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp); diff --git a/src/or/voting_schedule.c b/src/or/voting_schedule.c new file mode 100644 index 0000000000..d230a6dbcd --- /dev/null +++ b/src/or/voting_schedule.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file voting_schedule.c + * \brief This file contains functions that are from the directory authority + * subsystem related to voting specifically but used by many part of + * tor. The full feature is built as part of the dirauth module. + **/ + +#define VOTING_SCHEDULE_PRIVATE +#include "voting_schedule.h" + +#include "or.h" +#include "config.h" +#include "networkstatus.h" + +/* ===== + * Vote scheduling + * ===== */ + +/** Return the start of the next interval of size <b>interval</b> (in + * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always + * starts a fresh interval, and if the last interval of a day would be + * truncated to less than half its size, it is rolled into the + * previous interval. */ +time_t +voting_schedule_get_start_of_next_interval(time_t now, int interval, + int offset) +{ + struct tm tm; + time_t midnight_today=0; + time_t midnight_tomorrow; + time_t next; + + tor_gmtime_r(&now, &tm); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + + if (tor_timegm(&tm, &midnight_today) < 0) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight."); + // LCOV_EXCL_STOP + } + midnight_tomorrow = midnight_today + (24*60*60); + + next = midnight_today + ((now-midnight_today)/interval + 1)*interval; + + /* Intervals never cross midnight. */ + if (next > midnight_tomorrow) + next = midnight_tomorrow; + + /* If the interval would only last half as long as it's supposed to, then + * skip over to the next day. */ + if (next + interval/2 > midnight_tomorrow) + next = midnight_tomorrow; + + next += offset; + if (next - interval > now) + next -= interval; + + return next; +} + +/* Populate and return a new voting_schedule_t that can be used to schedule + * voting. The object is allocated on the heap and it's the responsibility of + * the caller to free it. Can't fail. */ +static voting_schedule_t * +get_voting_schedule(const or_options_t *options, time_t now, int severity) +{ + int interval, vote_delay, dist_delay; + time_t start; + time_t end; + networkstatus_t *consensus; + voting_schedule_t *new_voting_schedule; + + new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t)); + + consensus = networkstatus_get_live_consensus(now); + + if (consensus) { + interval = (int)( consensus->fresh_until - consensus->valid_after ); + vote_delay = consensus->vote_seconds; + dist_delay = consensus->dist_seconds; + + /* Note down the consensus valid after, so that we detect outdated voting + * schedules in case of skewed clocks etc. */ + new_voting_schedule->live_consensus_valid_after = consensus->valid_after; + } else { + interval = options->TestingV3AuthInitialVotingInterval; + vote_delay = options->TestingV3AuthInitialVoteDelay; + dist_delay = options->TestingV3AuthInitialDistDelay; + } + + tor_assert(interval > 0); + + if (vote_delay + dist_delay > interval/2) + vote_delay = dist_delay = interval / 4; + + start = new_voting_schedule->interval_starts = + voting_schedule_get_start_of_next_interval(now,interval, + options->TestingV3AuthVotingStartOffset); + end = voting_schedule_get_start_of_next_interval(start+1, interval, + options->TestingV3AuthVotingStartOffset); + + tor_assert(end > start); + + new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2); + new_voting_schedule->voting_ends = start - dist_delay; + new_voting_schedule->fetch_missing_votes = + start - dist_delay - (vote_delay/2); + new_voting_schedule->voting_starts = start - dist_delay - vote_delay; + + { + char tbuf[ISO_TIME_LEN+1]; + format_iso_time(tbuf, new_voting_schedule->interval_starts); + tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: " + "consensus_set=%d, interval=%d", + tbuf, consensus?1:0, interval); + } + + return new_voting_schedule; +} + +#define voting_schedule_free(s) \ + FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s)) + +/** Frees a voting_schedule_t. This should be used instead of the generic + * tor_free. */ +static void +voting_schedule_free_(voting_schedule_t *voting_schedule_to_free) +{ + if (!voting_schedule_to_free) + return; + tor_free(voting_schedule_to_free); +} + +voting_schedule_t voting_schedule; + +/* Using the time <b>now</b>, return the next voting valid-after time. */ +time_t +voting_schedule_get_next_valid_after_time(void) +{ + time_t now = approx_time(); + bool need_to_recalculate_voting_schedule = false; + + /* This is a safe guard in order to make sure that the voting schedule + * static object is at least initialized. Using this function with a zeroed + * voting schedule can lead to bugs. */ + if (tor_mem_is_zero((const char *) &voting_schedule, + sizeof(voting_schedule))) { + need_to_recalculate_voting_schedule = true; + goto done; /* no need for next check if we have to recalculate anyway */ + } + + /* Also make sure we are not using an outdated voting schedule. If we have a + * newer consensus, make sure we recalculate the voting schedule. */ + const networkstatus_t *ns = networkstatus_get_live_consensus(now); + if (ns && ns->valid_after != voting_schedule.live_consensus_valid_after) { + log_info(LD_DIR, "Voting schedule is outdated: recalculating (%d/%d)", + (int) ns->valid_after, + (int) voting_schedule.live_consensus_valid_after); + need_to_recalculate_voting_schedule = true; + } + + done: + if (need_to_recalculate_voting_schedule) { + voting_schedule_recalculate_timing(get_options(), now); + voting_schedule.created_on_demand = 1; + } + + return voting_schedule.interval_starts; +} + +/** Set voting_schedule to hold the timing for the next vote we should be + * doing. All type of tor do that because HS subsystem needs the timing as + * well to function properly. */ +void +voting_schedule_recalculate_timing(const or_options_t *options, time_t now) +{ + voting_schedule_t *new_voting_schedule; + + /* get the new voting schedule */ + new_voting_schedule = get_voting_schedule(options, now, LOG_INFO); + tor_assert(new_voting_schedule); + + /* Fill in the global static struct now */ + memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule)); + voting_schedule_free(new_voting_schedule); +} + diff --git a/src/or/voting_schedule.h b/src/or/voting_schedule.h new file mode 100644 index 0000000000..087701408e --- /dev/null +++ b/src/or/voting_schedule.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file voting_schedule.h + * \brief Header file for voting_schedule.c. + **/ + +#ifndef TOR_VOTING_SCHEDULE_H +#define TOR_VOTING_SCHEDULE_H + +#include "or.h" + +/** Scheduling information for a voting interval. */ +typedef struct { + /** When do we generate and distribute our vote for this interval? */ + time_t voting_starts; + /** When do we send an HTTP request for any votes that we haven't + * been posted yet?*/ + time_t fetch_missing_votes; + /** When do we give up on getting more votes and generate a consensus? */ + time_t voting_ends; + /** When do we send an HTTP request for any signatures we're expecting to + * see on the consensus? */ + time_t fetch_missing_signatures; + /** When do we publish the consensus? */ + time_t interval_starts; + + /* True iff we have generated and distributed our vote. */ + int have_voted; + /* True iff we've requested missing votes. */ + int have_fetched_missing_votes; + /* True iff we have built a consensus and sent the signatures around. */ + int have_built_consensus; + /* True iff we've fetched missing signatures. */ + int have_fetched_missing_signatures; + /* True iff we have published our consensus. */ + int have_published_consensus; + + /* True iff this voting schedule was set on demand meaning not through the + * normal vote operation of a dirauth or when a consensus is set. This only + * applies to a directory authority that needs to recalculate the voting + * timings only for the first vote even though this object was initilized + * prior to voting. */ + int created_on_demand; + + /** The valid-after time of the last live consensus that filled this voting + * schedule. It's used to detect outdated voting schedules. */ + time_t live_consensus_valid_after; +} voting_schedule_t; + +/* Public API. */ + +extern voting_schedule_t voting_schedule; + +void voting_schedule_recalculate_timing(const or_options_t *options, + time_t now); + +time_t voting_schedule_get_start_of_next_interval(time_t now, + int interval, + int offset); +time_t voting_schedule_get_next_valid_after_time(void); + +#endif /* TOR_VOTING_SCHEDULE_H */ + diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in index 301e7fdbe7..6eddc75459 100644 --- a/src/rust/.cargo/config.in +++ b/src/rust/.cargo/config.in @@ -6,3 +6,7 @@ @RUST_DL@ [source.vendored-sources] @RUST_DL@ directory = '@TOR_RUST_DEPENDENCIES@' + +[build] +@RUST_WARN@ rustflags = [ "-D", "warnings" ] +@RUST_TARGET_PROP@ diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 4f918c0221..1d2a7359aa 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1,8 +1,39 @@ [[package]] +name = "crypto" +version = "0.0.1" +dependencies = [ + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "external 0.0.1", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smartlist 0.0.1", + "tor_allocate 0.0.1", + "tor_log 0.1.0", +] + +[[package]] +name = "digest" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "external" version = "0.0.1" dependencies = [ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "smartlist 0.0.1", +] + +[[package]] +name = "generic-array" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -18,10 +49,24 @@ dependencies = [ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "smartlist 0.0.1", "tor_allocate 0.0.1", + "tor_log 0.1.0", "tor_util 0.0.1", ] [[package]] +name = "rand" +version = "0.5.0-pre.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.0-pre.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "smartlist" version = "0.0.1" dependencies = [ @@ -36,6 +81,14 @@ dependencies = [ ] [[package]] +name = "tor_log" +version = "0.1.0" +dependencies = [ + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "tor_allocate 0.0.1", +] + +[[package]] name = "tor_rust" version = "0.1.0" dependencies = [ @@ -49,7 +102,18 @@ version = "0.0.1" dependencies = [ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "tor_allocate 0.0.1", + "tor_log 0.1.0", ] +[[package]] +name = "typenum" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [metadata] +"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" +"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" "checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff" +"checksum rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3795e4701d9628a63a84d0289e66279883b40df165fca7caed7b87122447032a" +"checksum rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7255ffbdb188d5be1a69b6f9f3cf187de4207430b9e79ed5b76458a6b20de9a" +"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd" diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 953c9b96b7..c3e44d2a79 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -1,5 +1,14 @@ [workspace] -members = ["tor_util", "protover", "smartlist", "external", "tor_allocate", "tor_rust"] +members = [ + "crypto", + "external", + "protover", + "smartlist", + "tor_allocate", + "tor_log", + "tor_rust", + "tor_util", +] [profile.release] debug = true diff --git a/src/rust/build.rs b/src/rust/build.rs new file mode 100644 index 0000000000..b943aa5535 --- /dev/null +++ b/src/rust/build.rs @@ -0,0 +1,179 @@ +//! Build script for Rust modules in Tor. +//! +//! We need to use this because some of our Rust tests need to use some +//! of our C modules, which need to link some external libraries. +//! +//! This script works by looking at a "config.rust" file generated by our +//! configure script, and then building a set of options for cargo to pass to +//! the compiler. + +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::prelude::*; +use std::io; +use std::path::PathBuf; + +/// Wrapper around a key-value map. +struct Config( + HashMap<String,String> +); + +/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR +/// location provided by cargo and recursing up the directory tree. Note that +/// we need to look in the OUT_DIR, since autoconf will place generated files +/// in the build directory. +fn find_cfg() -> io::Result<String> { + let mut path = PathBuf::from(env::var("OUT_DIR").unwrap()); + loop { + path.push("config.rust"); + if path.exists() { + return Ok(path.to_str().unwrap().to_owned()); + } + path.pop(); // remove config.rust + if ! path.pop() { // can't remove last part of directory + return Err(io::Error::new(io::ErrorKind::NotFound, + "No config.rust")); + } + } +} + +impl Config { + /// Find the config.rust file and try to parse it. + /// + /// The file format is a series of lines of the form KEY=VAL, with + /// any blank lines and lines starting with # ignored. + fn load() -> io::Result<Config> { + let path = find_cfg()?; + let f = File::open(&path)?; + let reader = io::BufReader::new(f); + let mut map = HashMap::new(); + for line in reader.lines() { + let s = line?; + if s.trim().starts_with("#") || s.trim() == "" { + continue; + } + let idx = match s.find("=") { + None => { + return Err(io::Error::new(io::ErrorKind::InvalidData, + "missing =")); + }, + Some(x) => x + }; + let (var,eq_val) = s.split_at(idx); + let val = &eq_val[1..]; + map.insert(var.to_owned(), val.to_owned()); + } + Ok(Config(map)) + } + + /// Return a reference to the value whose key is 'key'. + /// + /// Panics if 'key' is not found in the configuration. + fn get(&self, key : &str) -> &str { + self.0.get(key).unwrap() + } + + /// Add a dependency on a static C library that is part of Tor, by name. + fn component(&self, s : &str) { + println!("cargo:rustc-link-lib=static={}", s); + } + + /// Add a dependency on a native library that is not part of Tor, by name. + fn dependency(&self, s : &str) { + println!("cargo:rustc-link-lib={}", s); + } + + /// Add a link path, relative to Tor's build directory. + fn link_relpath(&self, s : &str) { + let builddir = self.get("BUILDDIR"); + println!("cargo:rustc-link-search=native={}/{}", builddir, s); + } + + /// Add an absolute link path. + fn link_path(&self, s : &str) { + println!("cargo:rustc-link-search=native={}", s); + } + + /// Parse the CFLAGS in s, looking for -l and -L items, and adding + /// rust configuration as appropriate. + fn from_cflags(&self, s : &str) { + let mut next_is_lib = false; + let mut next_is_path = false; + for ent in self.get(s).split_whitespace() { + if next_is_lib { + self.dependency(ent); + next_is_lib = false; + } else if next_is_path { + self.link_path(ent); + next_is_path = false; + } else if ent == "-l" { + next_is_lib = true; + } else if ent == "-L" { + next_is_path = true; + } else if ent.starts_with("-L") { + self.link_path(&ent[2..]); + } else if ent.starts_with("-l") { + self.dependency(&ent[2..]); + } + } + } +} + +pub fn main() { + let cfg = Config::load().unwrap(); + let package = env::var("CARGO_PKG_NAME").unwrap(); + + match package.as_ref() { + "crypto" => { + // Right now, I'm having a separate configuration for each Rust + // package, since I'm hoping we can trim them down. Once we have a + // second Rust package that needs to use this build script, let's + // extract some of this stuff into a module. + // + // This is a ridiculous amount of code to be pulling in just + // to test our crypto library: modularity would be our + // friend here. + cfg.from_cflags("TOR_LDFLAGS_zlib"); + cfg.from_cflags("TOR_LDFLAGS_openssl"); + cfg.from_cflags("TOR_LDFLAGS_libevent"); + + cfg.link_relpath("src/common"); + cfg.link_relpath("src/ext/keccak-tiny"); + cfg.link_relpath("src/ext/keccak-tiny"); + cfg.link_relpath("src/ext/ed25519/ref10"); + cfg.link_relpath("src/ext/ed25519/donna"); + cfg.link_relpath("src/trunnel"); + + // Note that we can't pull in "libtor-testing", or else we + // will have dependencies on all the other rust packages that + // tor uses. We must be careful with factoring and dependencies + // moving forward! + cfg.component("or-crypto-testing"); + cfg.component("or-ctime-testing"); + cfg.component("or-testing"); + cfg.component("or-event-testing"); + cfg.component("or-ctime-testing"); + cfg.component("curve25519_donna"); + cfg.component("keccak-tiny"); + cfg.component("ed25519_ref10"); + cfg.component("ed25519_donna"); + cfg.component("or-trunnel-testing"); + + cfg.from_cflags("TOR_ZLIB_LIBS"); + cfg.from_cflags("TOR_LIB_MATH"); + cfg.from_cflags("TOR_OPENSSL_LIBS"); + cfg.from_cflags("TOR_LIBEVENT_LIBS"); + cfg.from_cflags("TOR_LIB_WS32"); + cfg.from_cflags("TOR_LIB_GDI"); + cfg.from_cflags("TOR_LIB_USERENV"); + cfg.from_cflags("CURVE25519_LIBS"); + cfg.from_cflags("TOR_LZMA_LIBS"); + cfg.from_cflags("TOR_ZSTD_LIBS"); + cfg.from_cflags("LIBS"); + }, + _ => { + panic!("No configuration in build.rs for package {}", package); + } + } +} diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml new file mode 100644 index 0000000000..869e0d6256 --- /dev/null +++ b/src/rust/crypto/Cargo.toml @@ -0,0 +1,28 @@ +[package] +authors = ["The Tor Project", + "Isis Lovecruft <isis@torproject.org>"] +name = "crypto" +version = "0.0.1" +publish = false +build = "../build.rs" + +[lib] +name = "crypto" +path = "lib.rs" +crate_type = ["rlib", "staticlib"] + +[dependencies] +libc = "=0.2.39" +digest = "=0.7.2" +rand_core = { version = "=0.2.0-pre.0", default-features = false } + +external = { path = "../external" } +smartlist = { path = "../smartlist" } +tor_allocate = { path = "../tor_allocate" } +tor_log = { path = "../tor_log" } + +[dev-dependencies] +rand = { version = "=0.5.0-pre.2", default-features = false } +rand_core = { version = "=0.2.0-pre.0", default-features = false } + +[features] diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs new file mode 100644 index 0000000000..a2463b89eb --- /dev/null +++ b/src/rust/crypto/digests/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Hash Digests and eXtendible Output Functions (XOFs) + +pub mod sha2; diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs new file mode 100644 index 0000000000..03e0843dc0 --- /dev/null +++ b/src/rust/crypto/digests/sha2.rs @@ -0,0 +1,222 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Hash Digests and eXtendible Output Functions (XOFs) + +pub use digest::Digest; + +use digest::BlockInput; +use digest::FixedOutput; +use digest::Input; +use digest::generic_array::GenericArray; +use digest::generic_array::typenum::U32; +use digest::generic_array::typenum::U64; + +use external::crypto_digest::CryptoDigest; +use external::crypto_digest::DigestAlgorithm; +use external::crypto_digest::get_256_bit_digest; +use external::crypto_digest::get_512_bit_digest; + +pub use external::crypto_digest::DIGEST256_LEN; +pub use external::crypto_digest::DIGEST512_LEN; + +/// The block size for both SHA-256 and SHA-512 digests is 512 bits/64 bytes. +/// +/// Unfortunately, we have to use the generic_array crate currently to express +/// this at compile time. Later, in the future, when Rust implements const +/// generics, we'll be able to remove this dependency (actually, it will get +/// removed from the digest crate, which is currently `pub use`ing it). +type BlockSize = U64; + +/// A SHA2-256 digest. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_dup` +#[derive(Clone)] +pub struct Sha256 { + engine: CryptoDigest, +} + +/// Construct a new, default instance of a `Sha256` hash digest function. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha256, Digest}; +/// +/// let mut hasher: Sha256 = Sha256::default(); +/// ``` +/// +/// # Returns +/// +/// A new `Sha256` digest. +impl Default for Sha256 { + fn default() -> Sha256 { + Sha256{ engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_256)) } + } +} + +impl BlockInput for Sha256 { + type BlockSize = BlockSize; +} + +/// Input `msg` into the digest. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha256, Digest}; +/// +/// let mut hasher: Sha256 = Sha256::default(); +/// +/// hasher.input(b"foo"); +/// hasher.input(b"bar"); +/// ``` +impl Input for Sha256 { + fn process(&mut self, msg: &[u8]) { + self.engine.add_bytes(&msg); + } +} + +/// Retrieve the output hash from everything which has been fed into this +/// `Sha256` digest thus far. +/// +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest in external::crypto_digest. +impl FixedOutput for Sha256 { + type OutputSize = U32; + + fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { + let buffer: [u8; DIGEST256_LEN] = get_256_bit_digest(self.engine); + + GenericArray::from(buffer) + } +} + +/// A SHA2-512 digest. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_dup` +#[derive(Clone)] +pub struct Sha512 { + engine: CryptoDigest, +} + +/// Construct a new, default instance of a `Sha512` hash digest function. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha512, Digest}; +/// +/// let mut hasher: Sha512 = Sha512::default(); +/// ``` +/// +/// # Returns +/// +/// A new `Sha512` digest. +impl Default for Sha512 { + fn default() -> Sha512 { + Sha512{ engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_512)) } + } +} + +impl BlockInput for Sha512 { + type BlockSize = BlockSize; +} + +/// Input `msg` into the digest. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha512, Digest}; +/// +/// let mut hasher: Sha512 = Sha512::default(); +/// +/// hasher.input(b"foo"); +/// hasher.input(b"bar"); +/// ``` +impl Input for Sha512 { + fn process(&mut self, msg: &[u8]) { + self.engine.add_bytes(&msg); + } +} + +/// Retrieve the output hash from everything which has been fed into this +/// `Sha512` digest thus far. +/// +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest in external::crypto_digest. +impl FixedOutput for Sha512 { + type OutputSize = U64; + + fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { + let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine); + + GenericArray::clone_from_slice(&buffer) + } +} + +#[cfg(test)] +mod test { + use digest::Digest; + + use super::*; + + #[test] + fn sha256_default() { + let _: Sha256 = Sha256::default(); + } + + #[test] + fn sha256_digest() { + let mut h: Sha256 = Sha256::new(); + let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; + let expected = [151, 223, 53, 136, 181, 163, 242, 75, 171, 195, + 133, 27, 55, 47, 11, 167, 26, 157, 205, 222, 212, + 59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157]; + + h.input(b"foo"); + h.input(b"bar"); + h.input(b"baz"); + + result.copy_from_slice(h.fixed_result().as_slice()); + + println!("{:?}", &result[..]); + + assert_eq!(result, expected); + } + + #[test] + fn sha512_default() { + let _: Sha512 = Sha512::default(); + } + + #[test] + fn sha512_digest() { + let mut h: Sha512 = Sha512::new(); + let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; + + let expected = [203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167, + 153, 217, 233, 8, 190, 69, 231, 103, 245, 209, 71, 212, 116, + 73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20, 122, + 221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38, + 179, 237, 61, 126, 246, 240, 103, 202, 153, 24, 90]; + + h.input(b"foo"); + h.input(b"bar"); + h.input(b"baz"); + + result.copy_from_slice(h.fixed_result().as_slice()); + + println!("{:?}", &result[..]); + + assert_eq!(&result[..], &expected[..]); + } +} diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs new file mode 100644 index 0000000000..f72a859dd7 --- /dev/null +++ b/src/rust/crypto/lib.rs @@ -0,0 +1,45 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Common cryptographic functions and utilities. +//! +//! # Hash Digests and eXtendable Output Functions (XOFs) +//! +//! The `digests` module contains submodules for specific hash digests +//! and extendable output functions. +//! +//! ```rust,no_run +//! use crypto::digests::sha2::*; +//! +//! let mut hasher: Sha256 = Sha256::default(); +//! let mut result: [u8; 32] = [0u8; 32]; +//! +//! hasher.input(b"foo"); +//! hasher.input(b"bar"); +//! hasher.input(b"baz"); +//! +//! result.copy_from_slice(hasher.result().as_slice()); +//! +//! assert!(result == [b'X'; DIGEST256_LEN]); +//! ``` + +#[deny(missing_docs)] + +// External crates from cargo or TOR_RUST_DEPENDENCIES. +extern crate digest; +extern crate libc; +extern crate rand_core; + +// External dependencies for tests. +#[cfg(test)] +extern crate rand as rand_crate; + +// Our local crates. +extern crate external; +#[cfg(not(test))] +#[macro_use] +extern crate tor_log; + +pub mod digests; // Unfortunately named "digests" plural to avoid name conflict with the digest crate +pub mod rand; diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs new file mode 100644 index 0000000000..82d02a70bb --- /dev/null +++ b/src/rust/crypto/rand/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +// Internal dependencies +pub mod rng; diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs new file mode 100644 index 0000000000..07a0a7bdc7 --- /dev/null +++ b/src/rust/crypto/rand/rng.rs @@ -0,0 +1,140 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Wrappers for Tor's random number generators to provide implementations of +//! `rand_core` traits. + +// This is the real implementation, in use in production, which calls into our C +// wrappers in /src/common/crypto_rand.c, which call into OpenSSL, system +// libraries, and make syscalls. +#[cfg(not(test))] +mod internal { + use std::u64; + + use rand_core::CryptoRng; + use rand_core::Error; + use rand_core::RngCore; + use rand_core::impls::next_u32_via_fill; + use rand_core::impls::next_u64_via_fill; + + use external::c_tor_crypto_rand; + use external::c_tor_crypto_strongest_rand; + use external::c_tor_crypto_seed_rng; + + use tor_log::LogDomain; + use tor_log::LogSeverity; + + /// Largest strong entropy request permitted. + // + // C_RUST_COUPLED: `MAX_STRONGEST_RAND_SIZE` /src/common/crypto_rand.c + const MAX_STRONGEST_RAND_SIZE: usize = 256; + + /// A wrapper around OpenSSL's RNG. + pub struct TorRng { + // This private, zero-length field forces the struct to be treated the + // same as its opaque C couterpart. + _unused: [u8; 0], + } + + /// Mark `TorRng` as being suitable for cryptographic purposes. + impl CryptoRng for TorRng {} + + impl TorRng { + // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c + #[allow(dead_code)] + pub fn new() -> Self { + if !c_tor_crypto_seed_rng() { + tor_log_msg!(LogSeverity::Warn, LogDomain::General, + "TorRng::from_seed()", + "The RNG could not be seeded!"); + } + // XXX also log success at info level —isis + TorRng{ _unused: [0u8; 0] } + } + } + + impl RngCore for TorRng { + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u32(&mut self) -> u32 { + next_u32_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u64(&mut self) -> u64 { + next_u64_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn fill_bytes(&mut self, dest: &mut [u8]) { + c_tor_crypto_rand(dest); + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { + Ok(self.fill_bytes(dest)) + } + } + + /// A CSPRNG which hashes together randomness from OpenSSL's RNG and entropy + /// obtained from the operating system. + pub struct TorStrongestRng { + // This private, zero-length field forces the struct to be treated the + // same as its opaque C couterpart. + _unused: [u8; 0], + } + + /// Mark `TorRng` as being suitable for cryptographic purposes. + impl CryptoRng for TorStrongestRng {} + + impl TorStrongestRng { + // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c + #[allow(dead_code)] + pub fn new() -> Self { + if !c_tor_crypto_seed_rng() { + tor_log_msg!(LogSeverity::Warn, LogDomain::General, + "TorStrongestRng::from_seed()", + "The RNG could not be seeded!"); + } + // XXX also log success at info level —isis + TorStrongestRng{ _unused: [0u8; 0] } + } + } + + impl RngCore for TorStrongestRng { + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u32(&mut self) -> u32 { + next_u32_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u64(&mut self) -> u64 { + next_u64_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn fill_bytes(&mut self, dest: &mut [u8]) { + debug_assert!(dest.len() <= MAX_STRONGEST_RAND_SIZE); + + c_tor_crypto_strongest_rand(dest); + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { + Ok(self.fill_bytes(dest)) + } + } +} + +// For testing, we expose a pure-Rust implementation. +#[cfg(test)] +mod internal { + // It doesn't matter if we pretend ChaCha is a CSPRNG in tests. + pub use rand_crate::ChaChaRng as TorRng; + pub use rand_crate::ChaChaRng as TorStrongestRng; +} + +// Finally, expose the public functionality of whichever appropriate internal +// module. +pub use self::internal::*; + diff --git a/src/rust/external/Cargo.toml b/src/rust/external/Cargo.toml index b5957b1079..60ec03be40 100644 --- a/src/rust/external/Cargo.toml +++ b/src/rust/external/Cargo.toml @@ -6,6 +6,9 @@ name = "external" [dependencies] libc = "=0.2.39" +[dependencies.smartlist] +path = "../smartlist" + [lib] name = "external" path = "lib.rs" diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs new file mode 100644 index 0000000000..3e8801f203 --- /dev/null +++ b/src/rust/external/crypto_digest.rs @@ -0,0 +1,406 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Bindings to external digest and XOF functions which live within +//! src/common/crypto_digest.[ch]. +//! +//! We wrap our C implementations in src/common/crypto_digest.[ch] with more +//! Rusty types and interfaces in src/rust/crypto/digest/. + +use std::process::abort; + +use libc::c_char; +use libc::c_int; +use libc::size_t; +use libc::uint8_t; + +use smartlist::Stringlist; + +/// Length of the output of our message digest. +pub const DIGEST_LEN: usize = 20; + +/// Length of the output of our second (improved) message digests. (For now +/// this is just sha256, but it could be any other 256-bit digest.) +pub const DIGEST256_LEN: usize = 32; + +/// Length of the output of our 64-bit optimized message digests (SHA512). +pub const DIGEST512_LEN: usize = 64; + +/// Length of a sha1 message digest when encoded in base32 with trailing = signs +/// removed. +pub const BASE32_DIGEST_LEN: usize = 32; + +/// Length of a sha1 message digest when encoded in base64 with trailing = signs +/// removed. +pub const BASE64_DIGEST_LEN: usize = 27; + +/// Length of a sha256 message digest when encoded in base64 with trailing = +/// signs removed. +pub const BASE64_DIGEST256_LEN: usize = 43; + +/// Length of a sha512 message digest when encoded in base64 with trailing = +/// signs removed. +pub const BASE64_DIGEST512_LEN: usize = 86; + +/// Length of hex encoding of SHA1 digest, not including final NUL. +pub const HEX_DIGEST_LEN: usize = 40; + +/// Length of hex encoding of SHA256 digest, not including final NUL. +pub const HEX_DIGEST256_LEN: usize = 64; + +/// Length of hex encoding of SHA512 digest, not including final NUL. +pub const HEX_DIGEST512_LEN: usize = 128; + +/// Our C code uses an enum to declare the digest algorithm types which we know +/// about. However, because enums are implementation-defined in C, we can +/// neither work with them directly nor translate them into Rust enums. +/// Instead, we represent them as a u8 (under the assumption that we'll never +/// support more than 256 hash functions). +#[allow(non_camel_case_types)] +type digest_algorithm_t = u8; + +const DIGEST_SHA1: digest_algorithm_t = 0; +const DIGEST_SHA256: digest_algorithm_t = 1; +const DIGEST_SHA512: digest_algorithm_t = 2; +const DIGEST_SHA3_256: digest_algorithm_t = 3; +const DIGEST_SHA3_512: digest_algorithm_t = 4; + +/// The number of hash digests we produce for a `common_digests_t`. +/// +/// We can't access these from Rust, because their definitions in C require +/// introspecting the `digest_algorithm_t` typedef, which is an enum, so we have +/// to redefine them here. +const N_COMMON_DIGEST_ALGORITHMS: usize = DIGEST_SHA256 as usize + 1; + +/// A digest function. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[allow(non_camel_case_types)] +struct crypto_digest_t { + // This private, zero-length field forces the struct to be treated the same + // as its opaque C couterpart. + _unused: [u8; 0], +} + +/// An eXtendible Output Function (XOF). +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[allow(non_camel_case_types)] +struct crypto_xof_t { + // This private, zero-length field forces the struct to be treated the same + // as its opaque C couterpart. + _unused: [u8; 0], +} + +/// A set of all the digests we commonly compute, taken on a single +/// string. Any digests that are shorter than 512 bits are right-padded +/// with 0 bits. +/// +/// Note that this representation wastes 44 bytes for the SHA1 case, so +/// don't use it for anything where we need to allocate a whole bunch at +/// once. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[allow(non_camel_case_types)] +struct common_digests_t { + pub d: [[c_char; N_COMMON_DIGEST_ALGORITHMS]; DIGEST256_LEN], +} + +/// A `smartlist_t` is just an alias for the `#[repr(C)]` type `Stringlist`, to +/// make it more clear that we're working with a smartlist which is owned by C. +#[allow(non_camel_case_types)] +// BINDGEN_GENERATED: This type isn't actually bindgen generated, but the code +// below it which uses it is. As such, this comes up as "dead code" as well. +#[allow(dead_code)] +type smartlist_t = Stringlist; + +/// All of the external functions from `src/common/crypto_digest.h`. +/// +/// These are kept private because they should be wrapped with Rust to make their usage safer. +// +// BINDGEN_GENERATED: These definitions were generated with bindgen and cleaned +// up manually. As such, there are more bindings than are likely necessary or +// which are in use. +#[allow(dead_code)] +extern "C" { + fn crypto_digest(digest: *mut c_char, m: *const c_char, len: size_t) -> c_int; + fn crypto_digest256(digest: *mut c_char, m: *const c_char, len: size_t, + algorithm: digest_algorithm_t) -> c_int; + fn crypto_digest512(digest: *mut c_char, m: *const c_char, len: size_t, + algorithm: digest_algorithm_t) -> c_int; + fn crypto_common_digests(ds_out: *mut common_digests_t, m: *const c_char, len: size_t) -> c_int; + fn crypto_digest_smartlist_prefix(digest_out: *mut c_char, len_out: size_t, prepend: *const c_char, + lst: *const smartlist_t, append: *const c_char, alg: digest_algorithm_t); + fn crypto_digest_smartlist(digest_out: *mut c_char, len_out: size_t, + lst: *const smartlist_t, append: *const c_char, alg: digest_algorithm_t); + fn crypto_digest_algorithm_get_name(alg: digest_algorithm_t) -> *const c_char; + fn crypto_digest_algorithm_get_length(alg: digest_algorithm_t) -> size_t; + fn crypto_digest_algorithm_parse_name(name: *const c_char) -> c_int; + fn crypto_digest_new() -> *mut crypto_digest_t; + fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; + fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; + fn crypto_digest_free_(digest: *mut crypto_digest_t); + fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t); + fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t); + fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t; + fn crypto_digest_assign(into: *mut crypto_digest_t, from: *const crypto_digest_t); + fn crypto_hmac_sha256(hmac_out: *mut c_char, key: *const c_char, key_len: size_t, + msg: *const c_char, msg_len: size_t); + fn crypto_mac_sha3_256(mac_out: *mut uint8_t, len_out: size_t, + key: *const uint8_t, key_len: size_t, + msg: *const uint8_t, msg_len: size_t); + fn crypto_xof_new() -> *mut crypto_xof_t; + fn crypto_xof_add_bytes(xof: *mut crypto_xof_t, data: *const uint8_t, len: size_t); + fn crypto_xof_squeeze_bytes(xof: *mut crypto_xof_t, out: *mut uint8_t, len: size_t); + fn crypto_xof_free(xof: *mut crypto_xof_t); +} + +/// A wrapper around a `digest_algorithm_t`. +pub enum DigestAlgorithm { + SHA2_256, + SHA2_512, + SHA3_256, + SHA3_512, +} + +impl From<DigestAlgorithm> for digest_algorithm_t { + fn from(digest: DigestAlgorithm) -> digest_algorithm_t { + match digest { + DigestAlgorithm::SHA2_256 => DIGEST_SHA256, + DigestAlgorithm::SHA2_512 => DIGEST_SHA512, + DigestAlgorithm::SHA3_256 => DIGEST_SHA3_256, + DigestAlgorithm::SHA3_512 => DIGEST_SHA3_512, + } + } +} + +/// A wrapper around a mutable pointer to a `crypto_digest_t`. +pub struct CryptoDigest(*mut crypto_digest_t); + +/// Explicitly copy the state of a `CryptoDigest` hash digest context. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_dup` +impl Clone for CryptoDigest { + fn clone(&self) -> CryptoDigest { + let digest: *mut crypto_digest_t; + + unsafe { + digest = crypto_digest_dup(self.0 as *const crypto_digest_t); + } + + // See the note in the implementation of CryptoDigest for the + // reasoning for `abort()` here. + if digest.is_null() { + abort(); + } + + CryptoDigest(digest) + } +} + +impl CryptoDigest { + /// A wrapper to call one of the C functions `crypto_digest_new`, + /// `crypto_digest256_new`, or `crypto_digest512_new`. + /// + /// # Warnings + /// + /// This function will `abort()` the entire process in an "abnormal" fashion, + /// i.e. not unwinding this or any other thread's stack, running any + /// destructors, or calling any panic/exit hooks) if `tor_malloc()` (called in + /// `crypto_digest256_new()`) is unable to allocate memory. + /// + /// # Returns + /// + /// A new `CryptoDigest`, which is a wrapper around a opaque representation + /// of a `crypto_digest_t`. The underlying `crypto_digest_t` _MUST_ only + /// ever be handled via a raw pointer, and never introspected. + /// + /// # C_RUST_COUPLED + /// + /// * `crypto_digest_new` + /// * `crypto_digest256_new` + /// * `crypto_digest512_new` + /// * `tor_malloc` (called by `crypto_digest256_new`, but we make + /// assumptions about its behvaiour and return values here) + pub fn new(algorithm: Option<DigestAlgorithm>) -> CryptoDigest { + let digest: *mut crypto_digest_t; + + if algorithm.is_none() { + unsafe { + digest = crypto_digest_new(); + } + } else { + let algo: digest_algorithm_t = algorithm.unwrap().into(); // can't fail because it's Some + + unsafe { + // XXX This is a pretty awkward API to use from Rust... + digest = match algo { + DIGEST_SHA1 => crypto_digest_new(), + DIGEST_SHA256 => crypto_digest256_new(DIGEST_SHA256), + DIGEST_SHA3_256 => crypto_digest256_new(DIGEST_SHA3_256), + DIGEST_SHA512 => crypto_digest512_new(DIGEST_SHA512), + DIGEST_SHA3_512 => crypto_digest512_new(DIGEST_SHA3_512), + _ => abort(), + } + } + } + + // In our C code, `crypto_digest*_new()` allocates memory with + // `tor_malloc()`. In `tor_malloc()`, if the underlying malloc + // implementation fails to allocate the requested memory and returns a + // NULL pointer, we call `exit(1)`. In the case that this `exit(1)` is + // called within a worker, be that a process or a thread, the inline + // comments within `tor_malloc()` mention "that's ok, since the parent + // will run out of memory soon anyway". However, if it takes long + // enough for the worker to die, and it manages to return a NULL pointer + // to our Rust code, our Rust is now in an irreparably broken state and + // may exhibit undefined behaviour. An even worse scenario, if/when we + // have parent/child processes/threads controlled by Rust, would be that + // the UB contagion in Rust manages to spread to other children before + // the entire process (hopefully terminates). + // + // However, following the assumptions made in `tor_malloc()` that + // calling `exit(1)` in a child is okay because the parent will + // eventually run into the same errors, and also to stymie any UB + // contagion in the meantime, we call abort!() here to terminate the + // entire program immediately. + if digest.is_null() { + abort(); + } + + CryptoDigest(digest) + } + + /// A wrapper to call the C function `crypto_digest_add_bytes`. + /// + /// # Inputs + /// + /// * `bytes`: a byte slice of bytes to be added into this digest. + /// + /// # C_RUST_COUPLED + /// + /// * `crypto_digest_add_bytes` + pub fn add_bytes(&self, bytes: &[u8]) { + unsafe { + crypto_digest_add_bytes(self.0 as *mut crypto_digest_t, + bytes.as_ptr() as *const c_char, + bytes.len() as size_t) + } + } +} + +impl Drop for CryptoDigest { + fn drop(&mut self) { + unsafe { + crypto_digest_free_(self.0 as *mut crypto_digest_t); + } + } +} + +/// Get the 256-bit digest output of a `crypto_digest_t`. +/// +/// # Inputs +/// +/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA256` or a +/// `DIGEST_SHA3_256`. +/// +/// # Warning +/// +/// Calling this function with a `CryptoDigest` which is neither SHA2-256 or +/// SHA3-256 is a programming error. Since we cannot introspect the opaque +/// struct from Rust, however, there is no way for us to check that the correct +/// one is being passed in. That is up to you, dear programmer. If you mess +/// up, you will get a incorrectly-sized hash digest in return, and it will be +/// your fault. Don't do that. +/// +/// # Returns +/// +/// A 256-bit hash digest, as a `[u8; 32]`. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_get_digest` +/// * `DIGEST256_LEN` +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest w.r.t. output array size. +pub fn get_256_bit_digest(digest: CryptoDigest) -> [u8; DIGEST256_LEN] { + let mut buffer: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; + + unsafe { + crypto_digest_get_digest(digest.0, + buffer.as_mut_ptr() as *mut c_char, + DIGEST256_LEN as size_t); + + if buffer.as_ptr().is_null() { + abort(); + } + } + buffer +} + +/// Get the 512-bit digest output of a `crypto_digest_t`. +/// +/// # Inputs +/// +/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA512` or a +/// `DIGEST_SHA3_512`. +/// +/// # Warning +/// +/// Calling this function with a `CryptoDigest` which is neither SHA2-512 or +/// SHA3-512 is a programming error. Since we cannot introspect the opaque +/// struct from Rust, however, there is no way for us to check that the correct +/// one is being passed in. That is up to you, dear programmer. If you mess +/// up, you will get a incorrectly-sized hash digest in return, and it will be +/// your fault. Don't do that. +/// +/// # Returns +/// +/// A 512-bit hash digest, as a `[u8; 64]`. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_get_digest` +/// * `DIGEST512_LEN` +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest w.r.t. output array size. +pub fn get_512_bit_digest(digest: CryptoDigest) -> [u8; DIGEST512_LEN] { + let mut buffer: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; + + unsafe { + crypto_digest_get_digest(digest.0, + buffer.as_mut_ptr() as *mut c_char, + DIGEST512_LEN as size_t); + + if buffer.as_ptr().is_null() { + abort(); + } + } + buffer +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_layout_common_digests_t() { + assert_eq!(::std::mem::size_of::<common_digests_t>(), 64usize, + concat!("Size of: ", stringify!(common_digests_t))); + assert_eq!(::std::mem::align_of::<common_digests_t>(), 1usize, + concat!("Alignment of ", stringify!(common_digests_t))); + } + + #[test] + fn test_layout_crypto_digest_t() { + assert_eq!(::std::mem::size_of::<crypto_digest_t>(), 0usize, + concat!("Size of: ", stringify!(crypto_digest_t))); + assert_eq!(::std::mem::align_of::<crypto_digest_t>(), 1usize, + concat!("Alignment of ", stringify!(crypto_digest_t))); + } +} diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs new file mode 100644 index 0000000000..af1ade0161 --- /dev/null +++ b/src/rust/external/crypto_rand.rs @@ -0,0 +1,87 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Bindings to external (P)RNG interfaces and utilities in +//! src/common/crypto_rand.[ch]. +//! +//! We wrap our C implementations in src/common/crypto_rand.[ch] here in order +//! to provide wrappers with native Rust types, and then provide more Rusty +//! types and and trait implementations in src/rust/crypto/rand/. + +use std::time::Duration; + +use libc::c_double; +use libc::c_int; +use libc::size_t; +use libc::time_t; +use libc::uint8_t; + +extern "C" { + fn crypto_seed_rng() -> c_int; + fn crypto_rand(out: *mut uint8_t, out_len: size_t); + fn crypto_strongest_rand(out: *mut uint8_t, out_len: size_t); + fn crypto_rand_time_range(min: time_t, max: time_t) -> time_t; + fn crypto_rand_double() -> c_double; +} + +/// Seed OpenSSL's random number generator with bytes from the operating +/// system. +/// +/// # Returns +/// +/// `true` on success; `false` on failure. +pub fn c_tor_crypto_seed_rng() -> bool { + let ret: c_int; + + unsafe { + ret = crypto_seed_rng(); + } + match ret { + 0 => return true, + _ => return false, + } +} + +/// Fill the bytes of `dest` with random data. +pub fn c_tor_crypto_rand(dest: &mut [u8]) { + unsafe { + crypto_rand(dest.as_mut_ptr(), dest.len() as size_t); + } +} + +/// Fill the bytes of `dest` with "strong" random data by hashing +/// together randomness obtained from OpenSSL's RNG and the operating +/// system. +pub fn c_tor_crypto_strongest_rand(dest: &mut [u8]) { + // We'll let the C side panic if the len is larger than + // MAX_STRONGEST_RAND_SIZE, rather than potentially panicking here. A + // paranoid caller should assert on the length of dest *before* calling this + // function. + unsafe { + crypto_strongest_rand(dest.as_mut_ptr(), dest.len() as size_t); + } +} + +/// Get a random time, in seconds since the Unix Epoch. +/// +/// # Returns +/// +/// A `std::time::Duration` of seconds since the Unix Epoch. +pub fn c_tor_crypto_rand_time_range(min: &Duration, max: &Duration) -> Duration { + let ret: time_t; + + unsafe { + ret = crypto_rand_time_range(min.as_secs() as time_t, max.as_secs() as time_t); + } + + Duration::from_secs(ret as u64) +} + +/// Return a pseudorandom 64-bit float, chosen uniformly from the range [0.0, 1.0). +pub fn c_tor_crypto_rand_double() -> f64 { + unsafe { + crypto_rand_double() + } +} + diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs index 0af0d6452d..b72a4f6e4c 100644 --- a/src/rust/external/lib.rs +++ b/src/rust/external/lib.rs @@ -1,4 +1,4 @@ -//! Copyright (c) 2016-2017, The Tor Project, Inc. */ +//! Copyright (c) 2016-2018, The Tor Project, Inc. */ //! See LICENSE for licensing information */ //! Interface for external calls to tor C ABI @@ -9,6 +9,11 @@ extern crate libc; +extern crate smartlist; + +pub mod crypto_digest; +mod crypto_rand; mod external; +pub use crypto_rand::*; pub use external::*; diff --git a/src/rust/include.am b/src/rust/include.am index 7a0181e373..5e5b0b3faf 100644 --- a/src/rust/include.am +++ b/src/rust/include.am @@ -1,10 +1,19 @@ include src/rust/tor_rust/include.am EXTRA_DIST +=\ + src/rust/build.rs \ src/rust/Cargo.toml \ src/rust/Cargo.lock \ src/rust/.cargo/config.in \ + src/rust/crypto/Cargo.toml \ + src/rust/crypto/lib.rs \ + src/rust/crypto/digests/mod.rs \ + src/rust/crypto/digests/sha2.rs \ + src/rust/crypto/rand/mod.rs \ + src/rust/crypto/rand/rng.rs \ src/rust/external/Cargo.toml \ + src/rust/external/crypto_digest.rs \ + src/rust/external/crypto_rand.rs \ src/rust/external/external.rs \ src/rust/external/lib.rs \ src/rust/protover/Cargo.toml \ @@ -20,6 +29,9 @@ EXTRA_DIST +=\ src/rust/tor_allocate/Cargo.toml \ src/rust/tor_allocate/lib.rs \ src/rust/tor_allocate/tor_allocate.rs \ + src/rust/tor_log/Cargo.toml \ + src/rust/tor_log/lib.rs \ + src/rust/tor_log/tor_log.rs \ src/rust/tor_rust/Cargo.toml \ src/rust/tor_rust/include.am \ src/rust/tor_rust/lib.rs \ diff --git a/src/rust/protover/Cargo.toml b/src/rust/protover/Cargo.toml index 86301b8787..a8480e142a 100644 --- a/src/rust/protover/Cargo.toml +++ b/src/rust/protover/Cargo.toml @@ -3,6 +3,8 @@ authors = ["The Tor Project"] version = "0.0.1" name = "protover" +[features] + [dependencies] libc = "=0.2.39" @@ -18,6 +20,9 @@ path = "../tor_util" [dependencies.tor_allocate] path = "../tor_allocate" +[dependencies.tor_log] +path = "../tor_log" + [lib] name = "protover" path = "lib.rs" diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index f0216f157b..7386e988c5 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -10,9 +10,6 @@ use std::ffi::CStr; use smartlist::*; use tor_allocate::allocate_and_copy_string; -use tor_util::strings::byte_slice_is_c_like; -use tor_util::strings::empty_static_cstr; - use errors::ProtoverError; use protover::*; @@ -182,18 +179,7 @@ pub extern "C" fn protocol_list_supports_protocol_or_later( pub extern "C" fn protover_get_supported_protocols() -> *const c_char { let supported: &'static CStr; - // If we're going to pass it to C, there cannot be any intermediate NUL - // bytes. An assert is okay here, since changing the const byte slice - // in protover.rs to contain a NUL byte somewhere in the middle would be a - // programming error. - assert!(byte_slice_is_c_like(SUPPORTED_PROTOCOLS)); - - // It's okay to unwrap the result of this function because - // we can see that the bytes we're passing into it 1) are valid UTF-8, - // 2) have no intermediate NUL bytes, and 3) are terminated with a NUL - // byte. - supported = CStr::from_bytes_with_nul(SUPPORTED_PROTOCOLS).unwrap(); - + supported = get_supported_protocols_cstr(); supported.as_ptr() } @@ -252,10 +238,9 @@ pub extern "C" fn protover_is_supported_here( #[no_mangle] pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const c_char { let supported: &'static CStr; - let elder_protocols: &'static [u8]; let empty: &'static CStr; - empty = empty_static_cstr(); + empty = cstr!(""); if version.is_null() { return empty.as_ptr(); @@ -270,19 +255,6 @@ pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const Err(_) => return empty.as_ptr(), }; - elder_protocols = compute_for_old_tor_cstr(&version); - - // If we're going to pass it to C, there cannot be any intermediate NUL - // bytes. An assert is okay here, since changing the const byte slice - // in protover.rs to contain a NUL byte somewhere in the middle would be a - // programming error. - assert!(byte_slice_is_c_like(elder_protocols)); - - // It's okay to unwrap the result of this function because - // we can see that the bytes we're passing into it 1) are valid UTF-8, - // 2) have no intermediate NUL bytes, and 3) are terminated with a NUL - // byte. - supported = CStr::from_bytes_with_nul(elder_protocols).unwrap_or(empty); - + supported = compute_for_old_tor_cstr(&version); supported.as_ptr() } diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs index 483260bca8..ce964196fd 100644 --- a/src/rust/protover/lib.rs +++ b/src/rust/protover/lib.rs @@ -28,6 +28,7 @@ extern crate libc; extern crate smartlist; extern crate external; extern crate tor_allocate; +#[macro_use] extern crate tor_util; pub mod errors; diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index c11c7c1803..68027056c4 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -3,12 +3,12 @@ use std::collections::HashMap; use std::collections::hash_map; +use std::ffi::CStr; use std::fmt; use std::str; use std::str::FromStr; use std::string::String; -use tor_util::strings::NUL_BYTE; use external::c_tor_version_as_new_as; use errors::ProtoverError; @@ -31,30 +31,6 @@ const MAX_PROTOCOLS_TO_EXPAND: usize = (1<<16); /// The maximum size an `UnknownProtocol`'s name may be. pub(crate) const MAX_PROTOCOL_NAME_LENGTH: usize = 100; -/// Currently supported protocols and their versions, as a byte-slice. -/// -/// # Warning -/// -/// This byte-slice ends in a NUL byte. This is so that we can directly convert -/// it to an `&'static CStr` in the FFI code, in order to hand the static string -/// to C in a way that is compatible with C static strings. -/// -/// Rust code which wishes to accesses this string should use -/// `protover::get_supported_protocols()` instead. -/// -/// C_RUST_COUPLED: src/or/protover.c `protover_get_supported_protocols` -pub(crate) const SUPPORTED_PROTOCOLS: &'static [u8] = - b"Cons=1-2 \ - Desc=1-2 \ - DirCache=1-2 \ - HSDir=1-2 \ - HSIntro=3-4 \ - HSRend=1-2 \ - Link=1-5 \ - LinkAuth=1,3 \ - Microdesc=1-2 \ - Relay=1-2\0"; - /// Known subprotocols in Tor. Indicates which subprotocol a relay supports. /// /// C_RUST_COUPLED: src/or/protover.h `protocol_type_t` @@ -148,21 +124,33 @@ impl From<Protocol> for UnknownProtocol { } } -/// Get the string representation of current supported protocols +/// Get a CStr representation of current supported protocols, for +/// passing to C, or for converting to a `&str` for Rust. /// /// # Returns /// -/// A `String` whose value is the existing protocols supported by tor. +/// An `&'static CStr` whose value is the existing protocols supported by tor. /// Returned data is in the format as follows: /// /// "HSDir=1-1 LinkAuth=1" /// -pub fn get_supported_protocols() -> &'static str { - // The `len() - 1` is to remove the NUL byte. - // The `unwrap` is safe becauase we SUPPORTED_PROTOCOLS is under - // our control. - str::from_utf8(&SUPPORTED_PROTOCOLS[..SUPPORTED_PROTOCOLS.len() - 1]) - .unwrap_or("") +/// # Note +/// +/// Rust code can use the `&'static CStr` as a normal `&'a str` by +/// calling `protover::get_supported_protocols`. +/// +// C_RUST_COUPLED: src/or/protover.c `protover_get_supported_protocols` +pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { + cstr!("Cons=1-2 \ + Desc=1-2 \ + DirCache=1-2 \ + HSDir=1-2 \ + HSIntro=3-4 \ + HSRend=1-2 \ + Link=1-5 \ + LinkAuth=1,3 \ + Microdesc=1-2 \ + Relay=1-2") } /// A map of protocol names to the versions of them which are supported. @@ -185,7 +173,8 @@ impl ProtoEntry { /// ProtoEntry, which is useful when looking up a specific /// subprotocol. pub fn supported() -> Result<Self, ProtoverError> { - let supported: &'static str = get_supported_protocols(); + let supported_cstr: &'static CStr = get_supported_protocols_cstr(); + let supported: &str = supported_cstr.to_str().unwrap_or(""); supported.parse() } @@ -703,7 +692,7 @@ pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool { /// /// # Returns /// -/// A `&'static [u8]` encoding a list of protocol names and supported +/// A `&'static CStr` encoding a list of protocol names and supported /// versions. The string takes the following format: /// /// "HSDir=1-1 LinkAuth=1" @@ -719,24 +708,25 @@ pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool { /// like to use this code in Rust, please see `compute_for_old_tor()`. // // C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` -pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static [u8] { +pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static CStr { + let empty: &'static CStr = cstr!(""); + if c_tor_version_as_new_as(version, FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS) { - return NUL_BYTE; + return empty; } if c_tor_version_as_new_as(version, "0.2.9.1-alpha") { - return b"Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2\0"; + return cstr!("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \ + Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"); } if c_tor_version_as_new_as(version, "0.2.7.5") { - return b"Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2\0"; + return cstr!("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ + Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"); } if c_tor_version_as_new_as(version, "0.2.4.19") { - return b"Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2\0"; + return cstr!("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ + Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2"); } - - NUL_BYTE + empty } /// Since older versions of Tor cannot infer their own subprotocols, @@ -767,14 +757,9 @@ pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static [u8] { // // C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` pub fn compute_for_old_tor(version: &str) -> Result<&'static str, ProtoverError> { - let mut computed: &'static [u8] = compute_for_old_tor_cstr(version); - - // Remove the NULL byte at the end. - computed = &computed[..computed.len() - 1]; - - // .from_utf8() fails with a Utf8Error if it couldn't validate the + // .to_str() fails with a Utf8Error if it couldn't validate the // utf-8, so convert that here into an Unparseable ProtoverError. - str::from_utf8(computed).or(Err(ProtoverError::Unparseable)) + compute_for_old_tor_cstr(version).to_str().or(Err(ProtoverError::Unparseable)) } #[cfg(test)] diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs index 359df1cd7a..3c0037f139 100644 --- a/src/rust/tor_allocate/tor_allocate.rs +++ b/src/rust/tor_allocate/tor_allocate.rs @@ -1,12 +1,17 @@ // Copyright (c) 2016-2017, The Tor Project, Inc. */ // See LICENSE for licensing information */ +// No-op defined purely for testing at the module level +use libc::c_char; -use libc::{c_char, c_void}; +#[cfg(not(feature = "testing"))] use std::{ptr, slice, mem}; +use libc::c_void; -#[cfg(not(test))] -extern "C" { - fn tor_malloc_(size: usize) -> *mut c_void; +// Define a no-op implementation for testing Rust modules without linking to C +#[cfg(feature = "testing")] +pub fn allocate_and_copy_string(s: &String) -> *mut c_char { + use std::ffi::CString; + CString::new(s.as_str()).unwrap().into_raw() } // Defined only for tests, used for testing purposes, so that we don't need @@ -17,6 +22,11 @@ unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void { malloc(size) } +#[cfg(all(not(test), not(feature = "testing")))] +extern "C" { + fn tor_malloc_(size: usize) -> *mut c_void; +} + /// Allocate memory using tor_malloc_ and copy an existing string into the /// allocated buffer, returning a pointer that can later be called in C. /// @@ -28,6 +38,7 @@ unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void { /// /// A `*mut c_char` that should be freed by tor_free in C /// +#[cfg(not(feature = "testing"))] pub fn allocate_and_copy_string(src: &String) -> *mut c_char { let bytes: &[u8] = src.as_bytes(); diff --git a/src/rust/tor_log/Cargo.toml b/src/rust/tor_log/Cargo.toml new file mode 100644 index 0000000000..9d06299c05 --- /dev/null +++ b/src/rust/tor_log/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tor_log" +version = "0.1.0" +authors = ["The Tor Project"] + +[lib] +name = "tor_log" +path = "lib.rs" +crate_type = ["rlib", "staticlib"] + +[features] + +[dependencies] +libc = "0.2.39" + +[dependencies.tor_allocate] +path = "../tor_allocate" diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs new file mode 100644 index 0000000000..72f9e38339 --- /dev/null +++ b/src/rust/tor_log/lib.rs @@ -0,0 +1,16 @@ +//! Copyright (c) 2016-2017, The Tor Project, Inc. */ +//! See LICENSE for licensing information */ + +//! Logging wrapper for Rust to utilize Tor's logger, found at +//! src/common/log.c and src/common/torlog.h +//! +//! Exposes different interfaces depending on whether we are running in test +//! or non-test mode. When testing, we use a no-op implementation, +//! otherwise we link directly to C. + +extern crate libc; +extern crate tor_allocate; + +mod tor_log; + +pub use tor_log::*; diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs new file mode 100644 index 0000000000..ad6725f0f2 --- /dev/null +++ b/src/rust/tor_log/tor_log.rs @@ -0,0 +1,270 @@ +// Copyright (c) 2016-2017, The Tor Project, Inc. */ +// See LICENSE for licensing information */ + +// Note that these functions are untested due to the fact that there are no +// return variables to test and they are calling into a C API. + +/// The related domain which the logging message is relevant. For example, +/// log messages relevant to networking would use LogDomain::LdNet, whereas +/// general messages can use LdGeneral. +#[derive(Eq, PartialEq)] +pub enum LogDomain { + Net, + General, +} + +/// The severity level at which to log messages. +#[derive(Eq, PartialEq)] +pub enum LogSeverity { + Notice, + Warn, +} + +/// Main entry point for Rust modules to log messages. +/// +/// # Inputs +/// +/// * A `severity` of type LogSeverity, which defines the level of severity the +/// message will be logged. +/// * A `domain` of type LogDomain, which defines the domain the log message +/// will be associated with. +/// * A `function` of type &str, which defines the name of the function where +/// the message is being logged. There is a current RFC for a macro that +/// defines function names. When it is, we should use it. See +/// https://github.com/rust-lang/rfcs/pull/1719 +/// * A `message` of type &str, which is the log message itself. +#[macro_export] +macro_rules! tor_log_msg { + ($severity: path, + $domain: path, + $function: expr, + $($message:tt)*) => + { + { + let msg = format!($($message)*); + $crate::tor_log_msg_impl($severity, $domain, $function, msg) + } + }; +} + +#[inline] +pub fn tor_log_msg_impl( + severity: LogSeverity, + domain: LogDomain, + function: &str, + message: String, +) { + use std::ffi::CString; + + /// Default function name to log in case of errors when converting + /// a function name to a CString + const ERR_LOG_FUNCTION: &str = "tor_log_msg"; + + /// Default message to log in case of errors when converting a log + /// message to a CString + const ERR_LOG_MSG: &str = "Unable to log message from Rust \ + module due to error when converting to CString"; + + let func = match CString::new(function) { + Ok(n) => n, + Err(_) => CString::new(ERR_LOG_FUNCTION).unwrap(), + }; + + let msg = match CString::new(message) { + Ok(n) => n, + Err(_) => CString::new(ERR_LOG_MSG).unwrap(), + }; + + // Bind to a local variable to preserve ownership. This is essential so + // that ownership is guaranteed until these local variables go out of scope + let func_ptr = func.as_ptr(); + let msg_ptr = msg.as_ptr(); + + let c_severity = unsafe { log::translate_severity(severity) }; + let c_domain = unsafe { log::translate_domain(domain) }; + + unsafe { log::tor_log_string(c_severity, c_domain, func_ptr, msg_ptr) } +} + +/// This implementation is used when compiling for actual use, as opposed to +/// testing. +#[cfg(not(test))] +pub mod log { + use libc::{c_char, c_int}; + use super::LogDomain; + use super::LogSeverity; + + /// Severity log types. These mirror definitions in /src/common/torlog.h + /// C_RUST_COUPLED: src/common/log.c, log domain types + extern "C" { + static LOG_WARN_: c_int; + static LOG_NOTICE_: c_int; + } + + /// Domain log types. These mirror definitions in /src/common/torlog.h + /// C_RUST_COUPLED: src/common/log.c, log severity types + extern "C" { + static LD_NET_: u32; + static LD_GENERAL_: u32; + } + + /// Translate Rust defintions of log domain levels to C. This exposes a 1:1 + /// mapping between types. + #[inline] + pub unsafe fn translate_domain(domain: LogDomain) -> u32 { + match domain { + LogDomain::Net => LD_NET_, + LogDomain::General => LD_GENERAL_, + } + } + + /// Translate Rust defintions of log severity levels to C. This exposes a + /// 1:1 mapping between types. + #[inline] + pub unsafe fn translate_severity(severity: LogSeverity) -> c_int { + match severity { + LogSeverity::Warn => LOG_WARN_, + LogSeverity::Notice => LOG_NOTICE_, + } + } + + /// The main entry point into Tor's logger. When in non-test mode, this + /// will link directly with `tor_log_string` in /src/or/log.c + extern "C" { + pub fn tor_log_string( + severity: c_int, + domain: u32, + function: *const c_char, + string: *const c_char, + ); + } +} + +/// This module exposes no-op functionality for testing other Rust modules +/// without linking to C. +#[cfg(test)] +pub mod log { + use libc::{c_char, c_int}; + use super::LogDomain; + use super::LogSeverity; + + pub static mut LAST_LOGGED_FUNCTION: *mut String = 0 as *mut String; + pub static mut LAST_LOGGED_MESSAGE: *mut String = 0 as *mut String; + + pub unsafe fn tor_log_string( + _severity: c_int, + _domain: u32, + function: *const c_char, + message: *const c_char, + ) { + use std::ffi::CStr; + + let f = CStr::from_ptr(function); + let fct = match f.to_str() { + Ok(n) => n, + Err(_) => "", + }; + LAST_LOGGED_FUNCTION = Box::into_raw(Box::new(String::from(fct))); + + let m = CStr::from_ptr(message); + let msg = match m.to_str() { + Ok(n) => n, + Err(_) => "", + }; + LAST_LOGGED_MESSAGE = Box::into_raw(Box::new(String::from(msg))); + } + + pub unsafe fn translate_domain(_domain: LogDomain) -> u32 { + 1 + } + + pub unsafe fn translate_severity(_severity: LogSeverity) -> c_int { + 1 + } +} + +#[cfg(test)] +mod test { + use tor_log::*; + use tor_log::log::{LAST_LOGGED_FUNCTION, LAST_LOGGED_MESSAGE}; + + #[test] + fn test_get_log_message() { + { + fn test_macro() { + tor_log_msg!( + LogSeverity::Warn, + LogDomain::Net, + "test_macro", + "test log message {}", + "a", + ); + } + + test_macro(); + + let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; + assert_eq!("test_macro", *function); + + let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; + assert_eq!("test log message a", *message); + } + + // test multiple inputs into the log message + { + fn test_macro() { + tor_log_msg!( + LogSeverity::Warn, + LogDomain::Net, + "next_test_macro", + "test log message {} {} {} {} {}", + 1, + 2, + 3, + 4, + 5 + ); + } + + test_macro(); + + let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; + assert_eq!("next_test_macro", *function); + + let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; + assert_eq!("test log message 1 2 3 4 5", *message); + } + + // test how a long log message will be formatted + { + fn test_macro() { + tor_log_msg!( + LogSeverity::Warn, + LogDomain::Net, + "test_macro", + "{}", + "All the world's a stage, and all the men and women \ + merely players: they have their exits and their \ + entrances; and one man in his time plays many parts, his \ + acts being seven ages." + ); + } + + test_macro(); + + let expected_string = "All the world's a \ + stage, and all the men \ + and women merely players: \ + they have their exits and \ + their entrances; and one man \ + in his time plays many parts, \ + his acts being seven ages."; + + let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; + assert_eq!("test_macro", *function); + + let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; + assert_eq!(expected_string, *message); + } + } +} diff --git a/src/rust/tor_rust/include.am b/src/rust/tor_rust/include.am index 40511bf9f2..99f3ede653 100644 --- a/src/rust/tor_rust/include.am +++ b/src/rust/tor_rust/include.am @@ -4,7 +4,7 @@ EXTRA_DIST +=\ EXTRA_CARGO_OPTIONS= -src/rust/target/release/@TOR_RUST_STATIC_NAME@: FORCE +@TOR_RUST_LIB_PATH@: FORCE ( cd "$(abs_top_builddir)/src/rust" ; \ CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \ $(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \ @@ -20,7 +20,7 @@ distclean-rust: rm -rf "$(abs_top_builddir)/src/rust/registry" if USE_RUST -build-rust: src/rust/target/release/@TOR_RUST_STATIC_NAME@ +build-rust: @TOR_RUST_LIB_PATH@ else build-rust: endif diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml index b540d8c847..a606a280b2 100644 --- a/src/rust/tor_util/Cargo.toml +++ b/src/rust/tor_util/Cargo.toml @@ -11,6 +11,9 @@ crate_type = ["rlib", "staticlib"] [dependencies.tor_allocate] path = "../tor_allocate" +[dependencies.tor_log] +path = "../tor_log" + [dependencies] libc = "=0.2.39" diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs index 5c3cdba4be..32779ed476 100644 --- a/src/rust/tor_util/ffi.rs +++ b/src/rust/tor_util/ffi.rs @@ -5,8 +5,7 @@ //! called from C. //! -use libc::c_char; -use tor_allocate::allocate_and_copy_string; +use tor_log::{LogSeverity, LogDomain}; /// Returns a short string to announce Rust support during startup. /// @@ -17,10 +16,12 @@ use tor_allocate::allocate_and_copy_string; /// tor_free(rust_str); /// ``` #[no_mangle] -pub extern "C" fn rust_welcome_string() -> *mut c_char { - let rust_welcome = String::from( +pub extern "C" fn rust_log_welcome_string() { + tor_log_msg!( + LogSeverity::Notice, + LogDomain::General, + "rust_log_welcome_string", "Tor is running with Rust integration. Please report \ - any bugs you encounter.", + any bugs you encounter." ); - allocate_and_copy_string(&rust_welcome) } diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs index 12cb3896b6..94697b6069 100644 --- a/src/rust/tor_util/lib.rs +++ b/src/rust/tor_util/lib.rs @@ -7,5 +7,8 @@ extern crate libc; extern crate tor_allocate; +#[macro_use] +extern crate tor_log; + pub mod ffi; pub mod strings; diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index 9321ce4f85..505191d913 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -3,80 +3,138 @@ //! Utilities for working with static strings. -use std::ffi::CStr; - -/// A byte-array containing a single NUL byte (`b"\0"`). -pub const NUL_BYTE: &'static [u8] = b"\0"; - -/// Determine if a byte slice is a C-like string. -/// -/// These checks guarantee that: -/// -/// 1. there are no intermediate NUL bytes -/// 2. the last byte *is* a NUL byte +/// Create a `CStr` from a literal byte slice, appending a NUL byte to it first. /// /// # Warning /// -/// This function does _not_ guarantee that the bytes represent any valid -/// encoding such as ASCII or UTF-8. +/// The literal byte slice which is taken as an argument *MUST NOT* have any NUL +/// bytes (`b"\0"`) in it, anywhere, or else an empty string will be returned +/// (`CStr::from_bytes_with_nul_unchecked(b"\0")`) so as to avoid `panic!()`ing. /// /// # Examples /// /// ``` -/// # use tor_util::strings::byte_slice_is_c_like; -/// # -/// let bytes: &[u8] = b"foo bar baz"; +/// #[macro_use] +/// extern crate tor_util; /// -/// assert!(byte_slice_is_c_like(&bytes) == false); +/// use std::ffi::CStr; /// -/// let bytes: &[u8] = b"foo\0bar baz"; +/// # fn do_test() -> Result<&'static CStr, &'static str> { +/// let message: &'static str = "This is a test of the tsunami warning system."; +/// let tuesday: &'static CStr; +/// let original: &str; /// -/// assert!(byte_slice_is_c_like(&bytes) == false); +/// tuesday = cstr!("This is a test of the tsunami warning system."); +/// original = tuesday.to_str().or(Err("Couldn't unwrap CStr!"))?; /// -/// let bytes: &[u8] = b"foo bar baz\0"; +/// assert!(original == message); +/// # +/// # Ok(tuesday) +/// # } +/// # fn main() { +/// # do_test(); // so that we can use the ? operator in the test +/// # } +/// ``` +/// It is also possible to pass several string literals to this macro. They +/// will be concatenated together in the order of the arguments, unmodified, +/// before finally being suffixed with a NUL byte: /// -/// assert!(byte_slice_is_c_like(&bytes) == true); /// ``` -pub fn byte_slice_is_c_like(bytes: &[u8]) -> bool { - if !bytes[..bytes.len() - 1].contains(&0x00) && bytes[bytes.len() - 1] == 0x00 { - return true; - } - false -} - -/// Get a static `CStr` containing a single `NUL_BYTE`. +/// #[macro_use] +/// extern crate tor_util; +/// # +/// # use std::ffi::CStr; +/// # +/// # fn do_test() -> Result<&'static CStr, &'static str> { /// -/// # Examples +/// let quux: &'static CStr = cstr!("foo", "bar", "baz"); +/// let orig: &'static str = quux.to_str().or(Err("Couldn't unwrap CStr!"))?; /// -/// When used as follows in a Rust FFI function, which could be called -/// from C: +/// assert!(orig == "foobarbaz"); +/// # Ok(quux) +/// # } +/// # fn main() { +/// # do_test(); // so that we can use the ? operator in the test +/// # } +/// ``` +/// This is useful for passing static strings to C from Rust FFI code. To do so +/// so, use the `.as_ptr()` method on the resulting `&'static CStr` to convert +/// it to the Rust equivalent of a C `const char*`: /// /// ``` -/// # extern crate libc; -/// # extern crate tor_util; -/// # -/// # use tor_util::strings::empty_static_cstr; -/// use libc::c_char; +/// #[macro_use] +/// extern crate tor_util; +/// /// use std::ffi::CStr; +/// use std::os::raw::c_char; /// -/// pub extern "C" fn give_c_code_an_empty_static_string() -> *const c_char { -/// let empty: &'static CStr = empty_static_cstr(); +/// pub extern "C" fn give_static_borrowed_string_to_c() -> *const c_char { +/// let hello: &'static CStr = cstr!("Hello, language my parents wrote."); /// -/// empty.as_ptr() +/// hello.as_ptr() /// } -/// /// # fn main() { -/// # give_c_code_an_empty_static_string(); +/// # let greetings = give_static_borrowed_string_to_c(); /// # } /// ``` +/// Note that the C code this static borrowed string is passed to *MUST NOT* +/// attempt to free the memory for the string. +/// +/// # Note +/// +/// An unfortunate limitation of the rustc compiler (as of 1.25.0-nightly), is +/// that the first example above compiles, but if we were to change the +/// assignment of `tuesday` as follows, it will fail to compile, because Rust +/// macros are expanded at parse time, and at parse time there is no symbol +/// table available. /// -/// This equates to an "empty" `const char*` static string in C. -pub fn empty_static_cstr() -> &'static CStr { - let empty: &'static CStr; +/// ```ignore +/// tuesday = cstr!(message); +/// ``` +/// with the error message `error: expected a literal`. +/// +/// # Returns +/// +/// If the string literals passed as arguments contain no NUL bytes anywhere, +/// then an `&'static CStr` containing the (concatenated) bytes of the string +/// literal(s) passed as arguments, with a NUL byte appended, is returned. +/// Otherwise, an `&'static CStr` containing a single NUL byte is returned (an +/// "empty" string in C). +#[macro_export] +macro_rules! cstr { + ($($bytes:expr),*) => ( + ::std::ffi::CStr::from_bytes_with_nul( + concat!($($bytes),*, "\0").as_bytes() + ).unwrap_or( + unsafe{ + ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") + } + ) + ) +} + +#[cfg(test)] +mod test { + use std::ffi::CStr; + + #[test] + fn cstr_macro() { + let _: &'static CStr = cstr!("boo"); + } + + #[test] + fn cstr_macro_multi_input() { + let quux: &'static CStr = cstr!("foo", "bar", "baz"); - unsafe { - empty = CStr::from_bytes_with_nul_unchecked(NUL_BYTE); + assert!(quux.to_str().unwrap() == "foobarbaz"); } - empty + #[test] + fn cstr_macro_bad_input() { + let waving: &'static CStr = cstr!("waving not drowning o/"); + let drowning: &'static CStr = cstr!("\0 drowning not waving"); + + assert!(waving.to_str().unwrap() == "waving not drowning o/"); + assert!(drowning.to_str().unwrap() == "") + } } diff --git a/src/test/bench.c b/src/test/bench.c index 92d7a244f7..9ab23c9921 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -12,7 +12,7 @@ #include "or.h" #include "onion_tap.h" -#include "relay.h" +#include "relay_crypto.h" #include <openssl/opensslv.h> #include <openssl/evp.h> #include <openssl/ec.h> @@ -23,6 +23,7 @@ #include "crypto_curve25519.h" #include "onion_ntor.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "consdiff.h" #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) @@ -505,10 +506,10 @@ bench_cell_ops(void) char key1[CIPHER_KEY_LEN], key2[CIPHER_KEY_LEN]; crypto_rand(key1, sizeof(key1)); crypto_rand(key2, sizeof(key2)); - or_circ->p_crypto = crypto_cipher_new(key1); - or_circ->n_crypto = crypto_cipher_new(key2); - or_circ->p_digest = crypto_digest_new(); - or_circ->n_digest = crypto_digest_new(); + or_circ->crypto.f_crypto = crypto_cipher_new(key1); + or_circ->crypto.b_crypto = crypto_cipher_new(key2); + or_circ->crypto.f_digest = crypto_digest_new(); + or_circ->crypto.b_digest = crypto_digest_new(); reset_perftime(); @@ -518,7 +519,8 @@ bench_cell_ops(void) for (i = 0; i < iters; ++i) { char recognized = 0; crypt_path_t *layer_hint = NULL; - relay_crypt(TO_CIRCUIT(or_circ), cell, d, &layer_hint, &recognized); + relay_decrypt_cell(TO_CIRCUIT(or_circ), cell, d, + &layer_hint, &recognized); } end = perftime(); printf("%sbound cells: %.2f ns per cell. (%.2f ns per byte of payload)\n", @@ -527,10 +529,7 @@ bench_cell_ops(void) NANOCOUNT(start,end,iters*CELL_PAYLOAD_SIZE)); } - crypto_digest_free(or_circ->p_digest); - crypto_digest_free(or_circ->n_digest); - crypto_cipher_free(or_circ->p_crypto); - crypto_cipher_free(or_circ->n_crypto); + relay_crypto_clear(&or_circ->crypto); tor_free(or_circ); tor_free(cell); } diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index cd16dc05be..39d6d3c17b 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -17,8 +17,9 @@ FUZZING_LIBS = \ src/trunnel/libor-trunnel-testing.a \ $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ + @TOR_LIBEVENT_LIBS@ \ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ + @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ @TOR_ZSTD_LIBS@ @@ -46,6 +47,7 @@ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) # ===== AFL fuzzers +if UNITTESTS_ENABLED src_test_fuzz_fuzz_consensus_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_consensus.c @@ -53,7 +55,9 @@ src_test_fuzz_fuzz_consensus_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_consensus_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_consensus_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_consensus_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_descriptor_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_descriptor.c @@ -61,7 +65,9 @@ src_test_fuzz_fuzz_descriptor_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_descriptor_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_descriptor_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_descriptor_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_diff_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_diff.c @@ -69,7 +75,9 @@ src_test_fuzz_fuzz_diff_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_diff_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_diff_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_diff_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_diff_apply_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_diff_apply.c @@ -77,7 +85,9 @@ src_test_fuzz_fuzz_diff_apply_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_diff_apply_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_diff_apply_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_diff_apply_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_extrainfo_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_extrainfo.c @@ -85,7 +95,9 @@ src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_hsdescv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_hsdescv2.c @@ -93,7 +105,9 @@ src_test_fuzz_fuzz_hsdescv2_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_hsdescv2_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_hsdescv2_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_hsdescv2_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_hsdescv3_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_hsdescv3.c @@ -101,7 +115,9 @@ src_test_fuzz_fuzz_hsdescv3_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_hsdescv3_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_hsdescv3_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_hsdescv3_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_http_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_http.c @@ -109,7 +125,9 @@ src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_http_connect_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_http_connect.c @@ -117,7 +135,9 @@ src_test_fuzz_fuzz_http_connect_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_http_connect_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_http_connect_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_http_connect_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_iptsv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_iptsv2.c @@ -125,7 +145,9 @@ src_test_fuzz_fuzz_iptsv2_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_iptsv2_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_iptsv2_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_iptsv2_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_microdesc_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_microdesc.c @@ -133,7 +155,9 @@ src_test_fuzz_fuzz_microdesc_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_microdesc_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_microdesc_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_microdesc_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_vrs.c @@ -141,7 +165,9 @@ src_test_fuzz_fuzz_vrs_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_vrs_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_vrs_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_vrs_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED FUZZERS = \ src/test/fuzz/fuzz-consensus \ src/test/fuzz/fuzz-descriptor \ @@ -155,93 +181,118 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-vrs +endif # ===== libfuzzer if LIBFUZZER_ENABLED +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_consensus_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_consensus_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_consensus_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_consensus_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_descriptor_SOURCES = \ $(src_test_fuzz_fuzz_descriptor_SOURCES) src_test_fuzz_lf_fuzz_descriptor_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_descriptor_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_descriptor_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_descriptor_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_diff_SOURCES = \ $(src_test_fuzz_fuzz_diff_SOURCES) src_test_fuzz_lf_fuzz_diff_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_diff_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_diff_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_diff_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_diff_apply_SOURCES = \ $(src_test_fuzz_fuzz_diff_apply_SOURCES) src_test_fuzz_lf_fuzz_diff_apply_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_diff_apply_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_diff_apply_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_diff_apply_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_extrainfo_SOURCES = \ $(src_test_fuzz_fuzz_extrainfo_SOURCES) src_test_fuzz_lf_fuzz_extrainfo_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_extrainfo_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_extrainfo_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_extrainfo_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_hsdescv2_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv2_SOURCES) src_test_fuzz_lf_fuzz_hsdescv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_hsdescv2_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_hsdescv2_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_hsdescv2_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_hsdescv3_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv3_SOURCES) src_test_fuzz_lf_fuzz_hsdescv3_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_hsdescv3_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_hsdescv3_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_hsdescv3_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_http_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_http_connect_SOURCES = \ $(src_test_fuzz_fuzz_http_connect_SOURCES) src_test_fuzz_lf_fuzz_http_connect_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_http_connect_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_http_connect_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_http_connect_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_iptsv2_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_iptsv2_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_iptsv2_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_microdesc_SOURCES = \ $(src_test_fuzz_fuzz_microdesc_SOURCES) src_test_fuzz_lf_fuzz_microdesc_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_microdesc_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_microdesc_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_microdesc_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_vrs_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_vrs_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS) +endif LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-consensus \ @@ -264,65 +315,89 @@ endif # ===== oss-fuzz if OSS_FUZZ_ENABLED +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_consensus_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_descriptor_a_SOURCES = \ $(src_test_fuzz_fuzz_descriptor_SOURCES) src_test_fuzz_liboss_fuzz_descriptor_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_descriptor_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_diff_a_SOURCES = \ $(src_test_fuzz_fuzz_diff_SOURCES) src_test_fuzz_liboss_fuzz_diff_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_diff_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_diff_apply_a_SOURCES = \ $(src_test_fuzz_fuzz_diff_apply_SOURCES) src_test_fuzz_liboss_fuzz_diff_apply_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_diff_apply_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_extrainfo_a_SOURCES = \ $(src_test_fuzz_fuzz_extrainfo_SOURCES) src_test_fuzz_liboss_fuzz_extrainfo_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_extrainfo_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_hsdescv2_a_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv2_SOURCES) src_test_fuzz_liboss_fuzz_hsdescv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_hsdescv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_hsdescv3_a_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv3_SOURCES) src_test_fuzz_liboss_fuzz_hsdescv3_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_hsdescv3_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_http_connect_a_SOURCES = \ $(src_test_fuzz_fuzz_http_connect_SOURCES) src_test_fuzz_liboss_fuzz_http_connect_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_http_connect_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_iptsv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_microdesc_a_SOURCES = \ $(src_test_fuzz_fuzz_microdesc_SOURCES) src_test_fuzz_liboss_fuzz_microdesc_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_microdesc_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-consensus.a \ diff --git a/src/test/include.am b/src/test/include.am index 93fc008419..70df3aac14 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -10,7 +10,10 @@ TESTS_ENVIRONMENT = \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \ export CARGO="$(CARGO)"; \ export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \ - export CARGO_ONLINE="$(CARGO_ONLINE)"; + export CARGO_ONLINE="$(CARGO_ONLINE)"; \ + export CCLD="$(CCLD)"; \ + chmod +x "$(abs_top_builddir)/link_rust.sh"; \ + export RUSTFLAGS="-C linker=$(abs_top_builddir)/link_rust.sh"; TESTSCRIPTS = \ src/test/fuzz_static_testcases.sh \ @@ -80,7 +83,10 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # This seems to matter nowhere but on Windows, but I assure you that it # matters a lot there, and is quite hard to debug if you forget to do it. -src_test_test_SOURCES = \ +src_test_test_SOURCES = + +if UNITTESTS_ENABLED +src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ src/test/rend_test_helpers.c \ @@ -89,7 +95,9 @@ src_test_test_SOURCES = \ src/test/test_addr.c \ src/test/test_address.c \ src/test/test_address_set.c \ + src/test/test_bridges.c \ src/test/test_buffers.c \ + src/test/test_bwmgt.c \ src/test/test_cell_formats.c \ src/test/test_cell_queue.c \ src/test/test_channel.c \ @@ -119,6 +127,7 @@ src_test_test_SOURCES = \ src/test/test_dos.c \ src/test/test_entryconn.c \ src/test/test_entrynodes.c \ + src/test/test_geoip.c \ src/test/test_guardfraction.c \ src/test/test_extorport.c \ src/test/test_hs.c \ @@ -137,11 +146,13 @@ src_test_test_SOURCES = \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ src/test/test_logging.c \ + src/test/test_mainloop.c \ src/test/test_microdesc.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ src/test/test_oos.c \ src/test/test_options.c \ + src/test/test_periodic_event.c \ src/test/test_policy.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ @@ -151,6 +162,7 @@ src_test_test_SOURCES = \ src/test/test_pubsub.c \ src/test/test_relay.c \ src/test/test_relaycell.c \ + src/test/test_relaycrypt.c \ src/test/test_rendcache.c \ src/test/test_replay.c \ src/test/test_router.c \ @@ -167,19 +179,24 @@ src_test_test_SOURCES = \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_schedule.c \ src/test/test_helpers.c \ src/test/test_dns.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c +endif -src_test_test_slow_SOURCES = \ +src_test_test_slow_SOURCES = +if UNITTESTS_ENABLED +src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_util_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c +endif src_test_test_memwipe_SOURCES = \ src/test/test-memwipe.c @@ -326,6 +343,7 @@ src_test_test_hs_ntor_cl_AM_CPPFLAGS = \ -I"$(top_srcdir)/src/or" +if UNITTESTS_ENABLED noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ @@ -333,9 +351,10 @@ src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ src/trace/libor-trace.a \ $(rust_ldadd) \ @TOR_LIB_MATH@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS) +endif EXTRA_DIST += \ src/test/bt_test.py \ @@ -346,6 +365,7 @@ EXTRA_DIST += \ src/test/fuzz_static_testcases.sh \ src/test/slownacl_curve25519.py \ src/test/zero_length_keys.sh \ + src/test/rust_supp.txt \ src/test/test_keygen.sh \ src/test/test_key_expiration.sh \ src/test/test_zero_length_keys.sh \ diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h index f74028a8ae..8057995070 100644 --- a/src/test/log_test_helpers.h +++ b/src/test/log_test_helpers.h @@ -90,6 +90,10 @@ void mock_dump_saved_logs(void); assert_log_predicate(!mock_saved_log_has_message(str), \ "expected log to not contain " # str); +#define expect_no_log_msg_containing(str) \ + assert_log_predicate(!mock_saved_log_has_message_containing(str), \ + "expected log to not contain " # str); + #define expect_log_severity(severity) \ assert_log_predicate(mock_saved_log_has_severity(severity), \ "expected log to contain severity " # severity); diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c index 095bfecf21..9ac3894b0b 100644 --- a/src/test/rend_test_helpers.c +++ b/src/test/rend_test_helpers.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "crypto_rand.h" #include "test.h" #include "rendcommon.h" #include "rend_test_helpers.h" diff --git a/src/test/rust_supp.txt b/src/test/rust_supp.txt new file mode 100644 index 0000000000..7fa50f3fb1 --- /dev/null +++ b/src/test/rust_supp.txt @@ -0,0 +1 @@ +leak:backtrace_alloc diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c index 89d946d506..aaaf2e7f68 100644 --- a/src/test/test-memwipe.c +++ b/src/test/test-memwipe.c @@ -7,7 +7,7 @@ #include <sys/types.h> #include <stdlib.h> -#include "crypto.h" +#include "crypto_util.h" #include "compat.h" #include "util.h" diff --git a/src/test/test-timers.c b/src/test/test-timers.c index a0b5b535c2..f20f29578b 100644 --- a/src/test/test-timers.c +++ b/src/test/test-timers.c @@ -7,11 +7,9 @@ #include <stdio.h> #include <string.h> -#include <event2/event.h> - #include "compat.h" #include "compat_libevent.h" -#include "crypto.h" +#include "crypto_rand.h" #include "timers.h" #include "util.h" @@ -50,7 +48,7 @@ timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono) // printf("%d / %d\n",n_fired, N_TIMERS); if (n_fired == n_active_timers) { - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); } } @@ -90,7 +88,7 @@ main(int argc, char **argv) --n_active_timers; } - event_base_loop(tor_libevent_get_base(), 0); + tor_libevent_run_event_loop(tor_libevent_get_base(), 0); int64_t total_difference = 0; uint64_t total_square_difference = 0; diff --git a/src/test/test.c b/src/test/test.c index 2e7d399322..aea3ad6595 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -9,6 +9,7 @@ **/ #include "orconfig.h" +#include "crypto_rand.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -26,7 +27,6 @@ /* These macros pull in declarations for some functions and structures that * are typically file-private. */ -#define GEOIP_PRIVATE #define ROUTER_PRIVATE #define CIRCUITSTATS_PRIVATE #define CIRCUITLIST_PRIVATE @@ -41,7 +41,6 @@ #include "compress.h" #include "config.h" #include "connection_edge.h" -#include "geoip.h" #include "rendcommon.h" #include "rendcache.h" #include "test.h" @@ -345,6 +344,18 @@ test_onion_queues(void *arg) tor_free(onionskin); } +static crypto_cipher_t *crypto_rand_aes_cipher = NULL; + +// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR +// cipher in <b>crypto_rand_aes_cipher</b>. +static void +crypto_rand_deterministic_aes(char *out, size_t n) +{ + tor_assert(crypto_rand_aes_cipher); + memset(out, 0, n); + crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n); +} + static void test_circuit_timeout(void *arg) { @@ -374,6 +385,11 @@ test_circuit_timeout(void *arg) state = or_state_new(); + // Use a deterministic RNG here, or else we'll get nondeterministic + // coverage in some of the circuitstats functions. + MOCK(crypto_rand, crypto_rand_deterministic_aes); + crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover"); + circuitbuild_running_unit_tests(); #define timeout0 (build_time_t)(30*1000.0) initial.Xm = 3000; @@ -508,6 +524,8 @@ test_circuit_timeout(void *arg) circuit_build_times_free_timeouts(&final); or_state_free(state); teardown_periodic_events(); + UNMOCK(crypto_rand); + crypto_cipher_free(crypto_rand_aes_cipher); } /** Test encoding and parsing of rendezvous service descriptors. */ @@ -623,376 +641,6 @@ test_rend_fns(void *arg) tor_free(intro_points_encrypted); } - /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs - * using ipv4. Since our fake geoip database is the same between - * ipv4 and ipv6, we should get the same result no matter which - * address family we pick for each IP. */ -#define SET_TEST_ADDRESS(i) do { \ - if ((i) & 1) { \ - SET_TEST_IPV6(i); \ - tor_addr_from_in6(&addr, &in6); \ - } else { \ - tor_addr_from_ipv4h(&addr, (uint32_t) i); \ - } \ - } while (0) - - /* Make sure that country ID actually works. */ -#define SET_TEST_IPV6(i) \ - do { \ - set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ - } while (0) -#define CHECK_COUNTRY(country, val) do { \ - /* test ipv4 country lookup */ \ - tt_str_op(country, OP_EQ, \ - geoip_get_country_name(geoip_get_country_by_ipv4(val))); \ - /* test ipv6 country lookup */ \ - SET_TEST_IPV6(val); \ - tt_str_op(country, OP_EQ, \ - geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \ - } while (0) - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void *arg) -{ - int i, j; - time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ - char *s = NULL, *v = NULL; - const char *bridge_stats_1 = - "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "bridge-ips zz=24,xy=8\n" - "bridge-ip-versions v4=16,v6=16\n" - "bridge-ip-transports <OR>=24\n", - *dirreq_stats_1 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips ab=8\n" - "dirreq-v3-reqs ab=8\n" - "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", - *dirreq_stats_2 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips \n" - "dirreq-v3-reqs \n" - "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", - *dirreq_stats_3 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips \n" - "dirreq-v3-reqs \n" - "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", - *dirreq_stats_4 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips \n" - "dirreq-v3-reqs \n" - "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n", - *entry_stats_1 = - "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "entry-ips ab=8\n", - *entry_stats_2 = - "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "entry-ips \n"; - tor_addr_t addr; - struct in6_addr in6; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - (void)arg; - tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET)); - - /* Populate the IPv6 DB equivalently with fake IPs in the same range */ - tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6)); - - /* We should have 4 countries: ??, ab, xy, zz. */ - tt_int_op(4,OP_EQ, geoip_get_n_countries()); - memset(&in6, 0, sizeof(in6)); - - CHECK_COUNTRY("??", 3); - CHECK_COUNTRY("ab", 32); - CHECK_COUNTRY("??", 5); - CHECK_COUNTRY("??", 51); - CHECK_COUNTRY("xy", 150); - CHECK_COUNTRY("xy", 190); - CHECK_COUNTRY("??", 2000); - - tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3)); - SET_TEST_IPV6(3); - tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6)); - - get_options_mutable()->BridgeRelay = 1; - get_options_mutable()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); - } - SET_TEST_ADDRESS(225); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600); - } - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - } - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); - tt_assert(s); - tt_assert(v); - tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s); - tt_str_op("v4=16,v6=16",OP_EQ, v); - tor_free(s); - tor_free(v); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); - tt_assert(s); - tt_assert(v); - tt_str_op("zz=24,xy=8",OP_EQ, s); - tt_str_op("v4=16,v6=16",OP_EQ, v); - tor_free(s); - tor_free(v); - - /* Start testing bridge statistics by making sure that we don't output - * bridge stats without initializing them. */ - s = geoip_format_bridge_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Initialize stats and generate the bridge-stats history string out of - * the connecting clients added above. */ - geoip_bridge_stats_init(now); - s = geoip_format_bridge_stats(now + 86400); - tt_assert(s); - tt_str_op(bridge_stats_1,OP_EQ, s); - tor_free(s); - - /* Stop collecting bridge stats and make sure we don't write a history - * string anymore. */ - geoip_bridge_stats_term(); - s = geoip_format_bridge_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Stop being a bridge and start being a directory mirror that gathers - * directory request statistics. */ - geoip_bridge_stats_term(); - get_options_mutable()->BridgeRelay = 0; - get_options_mutable()->BridgeRecordUsageByCountry = 0; - get_options_mutable()->DirReqStatistics = 1; - - /* Start testing dirreq statistics by making sure that we don't collect - * dirreq stats without initializing them. */ - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - s = geoip_format_dirreq_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Initialize stats, note one connecting client, and generate the - * dirreq-stats history string. */ - geoip_dirreq_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_1,OP_EQ, s); - tor_free(s); - - /* Stop collecting stats, add another connecting client, and ensure we - * don't generate a history string. */ - geoip_dirreq_stats_term(); - SET_TEST_ADDRESS(101); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - s = geoip_format_dirreq_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Re-start stats, add a connecting client, reset stats, and make sure - * that we get an all empty history string. */ - geoip_dirreq_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - geoip_reset_dirreq_stats(now); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_2,OP_EQ, s); - tor_free(s); - - /* Note a successful network status response and make sure that it - * appears in the history string. */ - geoip_note_ns_response(GEOIP_SUCCESS); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_3,OP_EQ, s); - tor_free(s); - - /* Start a tunneled directory request. */ - geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_4,OP_EQ, s); - tor_free(s); - - /* Stop collecting directory request statistics and start gathering - * entry stats. */ - geoip_dirreq_stats_term(); - get_options_mutable()->DirReqStatistics = 0; - get_options_mutable()->EntryStatistics = 1; - - /* Start testing entry statistics by making sure that we don't collect - * anything without initializing entry stats. */ - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - s = geoip_format_entry_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Initialize stats, note one connecting client, and generate the - * entry-stats history string. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - s = geoip_format_entry_stats(now + 86400); - tt_str_op(entry_stats_1,OP_EQ, s); - tor_free(s); - - /* Stop collecting stats, add another connecting client, and ensure we - * don't generate a history string. */ - geoip_entry_stats_term(); - SET_TEST_ADDRESS(101); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - s = geoip_format_entry_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Re-start stats, add a connecting client, reset stats, and make sure - * that we get an all empty history string. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - geoip_reset_entry_stats(now); - s = geoip_format_entry_stats(now + 86400); - tt_str_op(entry_stats_2,OP_EQ, s); - tor_free(s); - - /* Test the OOM handler. Add a client, run the OOM. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, - now - (12 * 60 * 60)); - /* We've seen this 12 hours ago. Run the OOM, it should clean the entry - * because it is above the minimum cutoff of 4 hours. */ - size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000); - tt_size_op(bytes_removed, OP_GT, 0); - - /* Do it again but this time with an entry with a lower cutoff. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, - now - (3 * 60 * 60)); - bytes_removed = geoip_client_cache_handle_oom(now, 1000); - tt_size_op(bytes_removed, OP_EQ, 0); - - /* Stop collecting entry statistics. */ - geoip_entry_stats_term(); - get_options_mutable()->EntryStatistics = 0; - - done: - tor_free(s); - tor_free(v); -} - -static void -test_geoip_with_pt(void *arg) -{ - time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ - char *s = NULL; - int i; - tor_addr_t addr; - struct in6_addr in6; - - (void)arg; - get_options_mutable()->BridgeRelay = 1; - get_options_mutable()->BridgeRecordUsageByCountry = 1; - - memset(&in6, 0, sizeof(in6)); - - /* No clients seen yet. */ - s = geoip_get_transport_history(); - tor_assert(!s); - - /* 4 connections without a pluggable transport */ - for (i=0; i < 4; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); - } - - /* 9 connections with "alpha" */ - for (i=4; i < 13; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200); - } - - /* one connection with "beta" */ - SET_TEST_ADDRESS(13); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200); - - /* 14 connections with "charlie" */ - for (i=14; i < 28; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200); - } - - /* 131 connections with "ddr" */ - for (i=28; i < 159; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200); - } - - /* 8 connections with "entropy" */ - for (i=159; i < 167; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200); - } - - /* 2 connections from the same IP with two different transports. */ - SET_TEST_ADDRESS(++i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200); - - /* Test the transport history string. */ - s = geoip_get_transport_history(); - tor_assert(s); - tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136," - "entropy=8,fire=8,google=8"); - - /* Stop collecting entry statistics. */ - geoip_entry_stats_term(); - get_options_mutable()->EntryStatistics = 0; - - done: - tor_free(s); -} - -#undef SET_TEST_ADDRESS -#undef SET_TEST_IPV6 -#undef CHECK_COUNTRY - /** Run unit tests for stats code. */ static void test_stats(void *arg) @@ -1166,8 +814,6 @@ static struct testcase_t test_array[] = { { "fast_handshake", test_fast_handshake, 0, NULL, NULL }, FORK(circuit_timeout), FORK(rend_fns), - ENT(geoip), - FORK(geoip_with_pt), FORK(stats), END_OF_TESTCASES @@ -1179,7 +825,9 @@ struct testgroup_t testgroups[] = { { "addr/", addr_tests }, { "address/", address_tests }, { "address_set/", address_set_tests }, + { "bridges/", bridges_tests }, { "buffer/", buffer_tests }, + { "bwmgt/", bwmgt_tests }, { "cellfmt/", cell_format_tests }, { "cellqueue/", cell_queue_tests }, { "channel/", channel_tests }, @@ -1205,11 +853,13 @@ struct testgroup_t testgroups[] = { { "dir/", dir_tests }, { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, + { "dir/voting-schedule/", voting_schedule_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, + { "geoip/", geoip_tests }, { "legacy_hs/", hs_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, @@ -1224,10 +874,12 @@ struct testgroup_t testgroups[] = { { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, { "link-handshake/", link_handshake_tests }, + { "mainloop/", mainloop_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, { "proto/http/", proto_http_tests }, @@ -1236,6 +888,7 @@ struct testgroup_t testgroups[] = { { "pt/", pt_tests }, { "relay/" , relay_tests }, { "relaycell/", relaycell_tests }, + { "relaycrypt/", relaycrypt_tests }, { "rend_cache/", rend_cache_tests }, { "replaycache/", replaycache_tests }, { "router/", router_tests }, diff --git a/src/test/test.h b/src/test/test.h index 26139fc5fe..63b2b30746 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -11,6 +11,8 @@ * \brief Macros and functions used by unit tests. */ +#define DEBUG_SMARTLIST 1 + #include "compat.h" #include "tinytest.h" #define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END @@ -72,6 +74,14 @@ I64_PRINTF_TYPE, I64_FORMAT, \ {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) +/** + * Declare that the test is done, even though no tt___op() calls were made. + * + * For use when you only want to test calling something, but not check + * any values/pointers/etc afterwards. + */ +#define tt_finished() TT_EXIT_TEST_FUNCTION + const char *get_fname(const char *name); const char *get_fname_rnd(const char *name); struct crypto_pk_t *pk_generate(int idx); @@ -178,6 +188,8 @@ extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t bridges_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t buffer_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; @@ -208,6 +220,7 @@ extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; +extern struct testcase_t geoip_tests[]; extern struct testcase_t hs_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; @@ -223,11 +236,13 @@ extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; +extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; extern struct testcase_t procmon_tests[]; extern struct testcase_t proto_http_tests[]; @@ -237,6 +252,7 @@ extern struct testcase_t pubsub_tests[]; extern struct testcase_t pt_tests[]; extern struct testcase_t relay_tests[]; extern struct testcase_t relaycell_tests[]; +extern struct testcase_t relaycrypt_tests[]; extern struct testcase_t rend_cache_tests[]; extern struct testcase_t replaycache_tests[]; extern struct testcase_t router_tests[]; @@ -252,6 +268,7 @@ extern struct testcase_t tortls_tests[]; extern struct testcase_t util_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t dns_tests[]; extern struct testcase_t handle_tests[]; extern struct testcase_t sr_tests[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index e1a40b7e60..40db31320f 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -6,8 +6,10 @@ #define ADDRESSMAP_PRIVATE #include "orconfig.h" #include "or.h" +#include "crypto_rand.h" #include "test.h" #include "addressmap.h" +#include "log_test_helpers.h" /** Mocking replacement: only handles localhost. */ static int @@ -941,6 +943,158 @@ test_virtaddrmap(void *data) ; } +static const char *canned_data = NULL; +static size_t canned_data_len = 0; + +/* Mock replacement for crypto_rand() that returns canned data from + * canned_data above. */ +static void +crypto_canned(char *ptr, size_t n) +{ + if (canned_data_len) { + size_t to_copy = MIN(n, canned_data_len); + memcpy(ptr, canned_data, to_copy); + canned_data += to_copy; + canned_data_len -= to_copy; + n -= to_copy; + ptr += to_copy; + } + if (n) { + crypto_rand_unmocked(ptr, n); + } +} + +static void +test_virtaddrmap_persist(void *data) +{ + (void)data; + const char *a, *b, *c; + tor_addr_t addr; + char *ones = NULL; + + addressmap_init(); + + // Try a hostname. + a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, + tor_strdup("foobar.baz")); + tt_assert(a); + tt_assert(!strcmpend(a, ".virtual")); + + // mock crypto_rand to repeat the same result twice; make sure we get + // different outcomes. (Because even though the odds for receiving the + // same 80-bit address twice is only 1/2^40, it could still happen for + // some user -- but running our test through 2^40 iterations isn't + // reasonable.) + canned_data = "1234567890" // the first call returns this. + "1234567890" // the second call returns this. + "abcdefghij"; // the third call returns this. + canned_data_len = 30; + MOCK(crypto_rand, crypto_canned); + + a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, + tor_strdup("quuxit.baz")); + b = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, + tor_strdup("nescio.baz")); + tt_assert(a); + tt_assert(b); + tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual"); + tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual"); + + // Now try something to get us an ipv4 address + UNMOCK(crypto_rand); + tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16", + AF_INET, 0, NULL)); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("foobar.baz")); + tt_assert(a); + tt_assert(!strcmpstart(a, "192.168.")); + tor_addr_parse(&addr, a); + tt_int_op(AF_INET, OP_EQ, tor_addr_family(&addr)); + + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("quuxit.baz")); + tt_str_op(b, OP_NE, a); + tt_assert(!strcmpstart(b, "192.168.")); + + // Try some canned entropy and verify all the we discard duplicates, + // addresses that end with 0, and addresses that end with 255. + MOCK(crypto_rand, crypto_canned); + canned_data = "\x01\x02\x03\x04" // okay + "\x01\x02\x03\x04" // duplicate + "\x03\x04\x00\x00" // bad ending 1 + "\x05\x05\x00\xff" // bad ending 2 + "\x05\x06\x07\xf0"; // okay + canned_data_len = 20; + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("wumble.onion")); + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("wumpus.onion")); + tt_str_op(a, OP_EQ, "192.168.3.4"); + tt_str_op(b, OP_EQ, "192.168.7.240"); + + // Now try IPv6! + UNMOCK(crypto_rand); + tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20", + AF_INET6, 0, NULL)); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("foobar.baz")); + tt_assert(a); + tt_assert(!strcmpstart(a, "[1010:f")); + tor_addr_parse(&addr, a); + tt_int_op(AF_INET6, OP_EQ, tor_addr_family(&addr)); + + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("quuxit.baz")); + tt_str_op(b, OP_NE, a); + tt_assert(!strcmpstart(b, "[1010:f")); + + // Try IPv6 with canned entropy, to make sure we detect duplicates. + MOCK(crypto_rand, crypto_canned); + canned_data = "acanthopterygian" // okay + "cinematographist" // okay + "acanthopterygian" // duplicate + "acanthopterygian" // duplicate + "acanthopterygian" // duplicate + "cinematographist" // duplicate + "coadministration"; // okay + canned_data_len = 16 * 7; + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("wuffle.baz")); + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("gribble.baz")); + c = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("surprisingly-legible.baz")); + tt_str_op(a, OP_EQ, "[1010:f16e:7468:6f70:7465:7279:6769:616e]"); + tt_str_op(b, OP_EQ, "[1010:fe65:6d61:746f:6772:6170:6869:7374]"); + tt_str_op(c, OP_EQ, "[1010:f164:6d69:6e69:7374:7261:7469:6f6e]"); + + // Try address exhaustion: make sure we can actually fail if we + // get too many already-existing addresses. + canned_data_len = 128*1024; + canned_data = ones = tor_malloc(canned_data_len); + memset(ones, 1, canned_data_len); + // There is some chance this one will fail if a previous random + // allocation gave out the address already. + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("might-work.onion")); + if (a) { + tt_str_op(a, OP_EQ, "192.168.1.1"); + } + setup_capture_of_logs(LOG_WARN); + // This one will definitely fail, since we've set up the RNG to hand + // out "1" forever. + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("wont-work.onion")); + tt_assert(b == NULL); + expect_single_log_msg_containing("Ran out of virtual addresses!"); + + done: + UNMOCK(crypto_rand); + tor_free(ones); + addressmap_free_all(); + teardown_capture_of_logs(); +} + static void test_addr_localname(void *arg) { @@ -1095,6 +1249,7 @@ struct testcase_t addr_tests[] = { ADDR_LEGACY(ip6_helpers), ADDR_LEGACY(parse), { "virtaddr", test_virtaddrmap, 0, NULL, NULL }, + { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL }, { "localname", test_addr_localname, 0, NULL, NULL }, { "dup_ip", test_addr_dup_ip, 0, NULL, NULL }, { "sockaddr_to_str", test_addr_sockaddr_to_str, 0, NULL, NULL }, diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c index df022f539a..f7441a6491 100644 --- a/src/test/test_address_set.c +++ b/src/test/test_address_set.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "crypto_rand.h" #include "address_set.h" #include "microdesc.h" #include "networkstatus.h" diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c new file mode 100644 index 0000000000..c44f791e0d --- /dev/null +++ b/src/test/test_bridges.c @@ -0,0 +1,614 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_bridges.c + * \brief Unittests for code in src/or/bridges.c + **/ + +#define TOR_BRIDGES_PRIVATE +#define PT_PRIVATE /* Only needed for the mock_* items below */ + +#include <stdbool.h> + +#include "or.h" +#include "address.h" +#include "bridges.h" +#include "config.h" +#include "container.h" +#include "transports.h" +#include "util.h" + +/* Test suite stuff */ +#include "test.h" + +/** + * A mocked transport_t, constructed via mock_transport_get_by_name(). + */ +static transport_t *mock_transport = NULL; + +/** + * Mock transport_get_by_name() to simply return a transport_t for the + * transport name that was input to it. + */ +static transport_t * +mock_transport_get_by_name(const char *name) +{ + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 9999; + int socksv = 9; + char *args = tor_strdup("foo=bar"); + + if (!mock_transport) { + tor_addr_parse(addr, "99.99.99.99"); + mock_transport = transport_new(addr, port, name, socksv, args); + } + + tor_free(addr); + tor_free(args); + + return mock_transport; +} + +#undef PT_PRIVATE /* defined(PT_PRIVATE) */ + +/** + * Test helper: Add a variety of bridges to our global bridgelist. + */ +static void +helper_add_bridges_to_bridgelist(void *arg) +{ + /* Note: the two bridges which do not have specified fingerprints will be + * internally stored as both having the same fingerprint of all-zero bytes. + */ + + (void)arg; + char *bridge0 = tor_strdup("6.6.6.6:6666"); + char *bridge1 = tor_strdup("6.6.6.7:6667 " + "A10C4F666D27364036B562823E5830BC448E046A"); + char *bridge2 = tor_strdup("obfs4 198.245.60.51:443 " + "752CF7825B3B9EA6A98C83AC41F7099D67007EA5 " + "cert=xpmQtKUqQ/6v5X7ijgYE/f03+l2/EuQ1dexjyUhh16wQlu/" + "cpXUGalmhDIlhuiQPNEKmKw iat-mode=0"); + char *bridge3 = tor_strdup("banana 5.5.5.5:5555 " + "9D6AE1BD4FDF39721CE908966E79E16F9BFCCF2F"); + char *bridge4 = tor_strdup("obfs4 1.2.3.4:1234 " + "foo=abcdefghijklmnopqrstuvwxyz"); + char *bridge5 = tor_strdup("apple 4.4.4.4:4444 " + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "foo=abcdefghijklmnopqrstuvwxyz"); + + mark_bridge_list(); + +#define ADD_BRIDGE(bridge) \ + bridge_line_t *bridge_line_ ##bridge = parse_bridge_line(bridge); \ + if (!bridge_line_ ##bridge) { \ + printf("Unparseable bridge line: '%s'", #bridge); \ + } else { \ + bridge_add_from_config(bridge_line_ ##bridge); \ + } \ + tor_free(bridge); + + ADD_BRIDGE(bridge0); + ADD_BRIDGE(bridge1); + ADD_BRIDGE(bridge2); + ADD_BRIDGE(bridge3); + ADD_BRIDGE(bridge4); + ADD_BRIDGE(bridge5); +#undef ADD_BRIDGES + + sweep_bridge_list(); +} + +/** + * Make sure our test helper works too. + */ +static void +test_bridges_helper_func_add_bridges_to_bridgelist(void *arg) +{ + helper_add_bridges_to_bridgelist(arg); + tt_finished(); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling bridge_list_get() should create a new bridgelist if we + * didn't have one before. + */ +static void +test_bridges_bridge_list_get_creates_new_bridgelist(void *arg) +{ + const smartlist_t *bridgelist = bridge_list_get(); + + (void)arg; + + tt_ptr_op(bridgelist, OP_NE, NULL); + + done: + return; +} + +/** + * Calling clear_bridge_list() should remove all bridges from the bridgelist. + */ +static void +test_bridges_clear_bridge_list(void *arg) +{ + const smartlist_t *bridgelist; + const smartlist_t *bridgelist_after; + const bridge_info_t *bridge; + + helper_add_bridges_to_bridgelist(arg); + bridgelist = bridge_list_get(); + tt_ptr_op(bridgelist, OP_NE, NULL); + + bridge = smartlist_get(bridgelist, 0); + tt_ptr_op(bridge, OP_NE, NULL); + + clear_bridge_list(); + bridgelist_after = bridge_list_get(); + tt_ptr_op(bridgelist_after, OP_NE, NULL); + tt_int_op(smartlist_len(bridgelist_after), OP_EQ, 0); + + done: + return; +} + +/** + * Calling bridge_get_addrport() should give me the address and port + * of the bridge. In this case, we sort the smartlist of bridges on + * fingerprints and choose the first one. + */ +static void +test_bridges_bridge_get_addrport(void *arg) +{ + smartlist_t *bridgelist; + const bridge_info_t *bridge; + const tor_addr_port_t *addrport; + + helper_add_bridges_to_bridgelist(arg); + bridgelist = (smartlist_t*)bridge_list_get(); + tt_ptr_op(bridgelist, OP_NE, NULL); + + // This should be the bridge at 6.6.6.6:6666 with fingerprint + // 0000000000000000000000000000000000000000 + bridge = smartlist_get(bridgelist, 0); + tt_ptr_op(bridge, OP_NE, NULL); + + addrport = bridge_get_addr_port(bridge); + tt_int_op(addrport->port, OP_EQ, 6666); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_orports_digest() with two + * configured bridge orports and an invalid digest should return the + * bridge of the first addrport in the list. + */ +static void +test_bridges_get_configured_bridge_by_orports_digest(void *arg) +{ + smartlist_t *orports = NULL; + const smartlist_t *bridgelist; + const bridge_info_t *bridge1; + const bridge_info_t *bridge2; + const bridge_info_t *ret; + tor_addr_port_t *addrport1; + tor_addr_port_t *addrport2; + const char *digest; + + helper_add_bridges_to_bridgelist(arg); + bridgelist = bridge_list_get(); + tt_ptr_op(bridgelist, OP_NE, NULL); + + // This should be the bridge at 6.6.6.6:6666 with fingerprint + // 0000000000000000000000000000000000000000 + bridge1 = smartlist_get(bridgelist, 0); + tt_ptr_op(bridge1, OP_NE, NULL); + // This should be the bridge at 6.6.6.7:6667 with fingerprint + // A10C4F666D27364036B562823E5830BC448E046A + bridge2 = smartlist_get(bridgelist, 1); + tt_ptr_op(bridge2, OP_NE, NULL); + + addrport1 = (tor_addr_port_t*)bridge_get_addr_port(bridge1); + tt_int_op(addrport1->port, OP_EQ, 6666); + addrport2 = (tor_addr_port_t*)bridge_get_addr_port(bridge2); + tt_int_op(addrport2->port, OP_EQ, 6667); + + orports = smartlist_new(); + smartlist_add(orports, addrport1); + smartlist_add(orports, addrport2); + + digest = "zzzzzzzzzzzzzzzz"; + + ret = get_configured_bridge_by_orports_digest(digest, orports); + tt_ptr_op(ret, OP_NE, NULL); + + tt_assert(tor_addr_port_eq(addrport1, bridge_get_addr_port(ret))); + + done: + smartlist_free(orports); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_addr_port_digest() with a digest that we do + * have and an addr:port pair we don't should return the bridge for that + * digest. + */ +static void +test_bridges_get_configured_bridge_by_addr_port_digest_digest_only(void *arg) +{ + char digest[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + char ret_addr[16]; + uint16_t port = 11111; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + // We don't actually have a bridge with this addr:port pair + base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + ret = tor_addr_parse(addr, "111.111.111.111"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_addr_port_digest(addr, port, digest); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("4.4.4.4", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_addr_port_digest() with only an + * addr:port (i.e. digest set to NULL) should return the bridge for + * that digest when there is such a bridge. + */ +static void +test_bridges_get_configured_bridge_by_addr_port_digest_address_only(void *arg) +{ + bridge_info_t *bridge; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + char ret_addr[16]; + uint16_t port = 6666; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "6.6.6.6"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_addr_port_digest(addr, port, NULL); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("6.6.6.6", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that + * we do have, and an addr:port pair we don't have, should return NULL. + */ +static void +test_bridges_get_configured_bridge_by_exact_addr_port_digest_donly(void *arg) +{ + char digest[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 11111; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + // We don't actually have a bridge with this addr:port pair + base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + ret = tor_addr_parse(addr, "111.111.111.111"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest); + tt_ptr_op(bridge, OP_EQ, NULL); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that + * we do have, and an addr:port pair we do have, should return the bridge. + */ +static void +test_bridges_get_configured_bridge_by_exact_addr_port_digest_both(void *arg) +{ + char digest[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 4444; + char ret_addr[16]; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + ret = tor_addr_parse(addr, "4.4.4.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("4.4.4.4", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_exact_addr_port_digest() with no digest, + * and an addr:port pair we do have, should return the bridge. + */ +static void +test_bridges_get_configured_bridge_by_exact_addr_port_digest_aonly(void *arg) +{ + bridge_info_t *bridge; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 4444; + char ret_addr[16]; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "4.4.4.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, NULL); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("4.4.4.4", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling find_bridge_by_digest() when we have a bridge with a known + * identity digest should return the bridge's information. + */ +static void +test_bridges_find_bridge_by_digest_known(void *arg) +{ + char digest1[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + helper_add_bridges_to_bridgelist(arg); + + base16_decode(digest1, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + bridge = find_bridge_by_digest(digest1); + + tt_ptr_op(bridge, OP_NE, NULL); + + /* We have to call bridge_get_rsa_id_digest() here because the bridge_info_t + * struct is opaquely defined in bridges.h. */ + const uint8_t *digest2 = bridge_get_rsa_id_digest(bridge); + + tt_mem_op((char*)digest2, OP_EQ, digest1, DIGEST_LEN); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling find_bridge_by_digest() when we do NOT have a bridge with that + * identity digest should return NULL. + */ +static void +test_bridges_find_bridge_by_digest_unknown(void *arg) +{ + const char *fingerprint = "cccccccccccccccccccccccccccccccccccccccc"; + bridge_info_t *bridge; + + helper_add_bridges_to_bridgelist(arg); + + bridge = find_bridge_by_digest(fingerprint); + + tt_ptr_op(bridge, OP_EQ, NULL); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling bridge_resolve_conflicts() with an identical bridge to one we've + * already configure should mark the pre-configured bridge for removal. + */ +static void +test_bridges_bridge_resolve_conflicts(void *arg) +{ + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 4444; + const char *digest = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + const char *transport = "apple"; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "4.4.4.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge_resolve_conflicts((const tor_addr_t*)addr, port, digest, transport); + + /* The bridge should now be marked for removal, and removed when we sweep the + * bridge_list */ + sweep_bridge_list(); + ret = addr_is_a_configured_bridge((const tor_addr_t*)addr, port, digest); + tt_int_op(ret, OP_EQ, 0); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling transport_is_needed() with a transport we do need ("obfs4") and a + * bogus transport that we don't need should return 1 and 0, respectively. + */ +static void +test_bridges_transport_is_needed(void *arg) +{ + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = transport_is_needed("obfs4"); + tt_int_op(ret, OP_EQ, 1); + + ret = transport_is_needed("apowefjaoewpaief"); + tt_int_op(ret, OP_EQ, 0); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_transport_by_bridge_addrport() with the address and port of a + * configured bridge which uses a pluggable transport when there is no global + * transport_list should return -1 and the transport_t should be NULL. + */ +static void +test_bridges_get_transport_by_bridge_addrport_no_ptlist(void *arg) +{ + transport_t *transport = NULL; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 1234; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "1.2.3.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success? + + /* This will fail because the global transport_list has nothing in it, and so + * transport_get_by_name() has nothing to return, even the the bridge *did* + * say it had an obfs4 transport. + */ + ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port, + (const transport_t**)&transport); + tt_int_op(ret, OP_EQ, -1); // returns -1 on failure + tt_ptr_op(transport, OP_EQ, NULL); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +#define PT_PRIVATE + +/** + * Calling get_transport_by_bridge_addrport() with the address and port of a + * configured bridge which uses a pluggable transport should return 0 and set + * appropriate transport_t. + */ +static void +test_bridges_get_transport_by_bridge_addrport(void *arg) +{ + transport_t *transport = NULL; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 1234; + int ret; + + helper_add_bridges_to_bridgelist(arg); + mark_transport_list(); // Also initialise our transport_list + + ret = tor_addr_parse(addr, "1.2.3.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success? + + /* After we mock transport_get_by_name() to return a bogus transport_t with + * the name it was asked for, the call should succeed. + */ + MOCK(transport_get_by_name, mock_transport_get_by_name); + ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port, + (const transport_t**)&transport); + tt_int_op(ret, OP_EQ, 0); // returns 0 on success + tt_ptr_op(transport, OP_NE, NULL); + tt_str_op(transport->name, OP_EQ, "obfs4"); + + done: + UNMOCK(transport_get_by_name); + + tor_free(addr); + transport_free(transport); + + mark_bridge_list(); + sweep_bridge_list(); +} + +#undef PT_PRIVATE /* defined(PT_PRIVATE) */ + +#define B_TEST(name, flags) \ + { #name, test_bridges_ ##name, (flags), NULL, NULL } + +struct testcase_t bridges_tests[] = { + B_TEST(helper_func_add_bridges_to_bridgelist, 0), + B_TEST(bridge_list_get_creates_new_bridgelist, 0), + B_TEST(clear_bridge_list, 0), + B_TEST(bridge_get_addrport, 0), + B_TEST(get_configured_bridge_by_orports_digest, 0), + B_TEST(get_configured_bridge_by_addr_port_digest_digest_only, 0), + B_TEST(get_configured_bridge_by_addr_port_digest_address_only, 0), + B_TEST(get_configured_bridge_by_exact_addr_port_digest_donly, 0), + B_TEST(get_configured_bridge_by_exact_addr_port_digest_both, 0), + B_TEST(get_configured_bridge_by_exact_addr_port_digest_aonly, 0), + B_TEST(find_bridge_by_digest_known, 0), + B_TEST(find_bridge_by_digest_unknown, 0), + B_TEST(bridge_resolve_conflicts, 0), + B_TEST(get_transport_by_bridge_addrport_no_ptlist, 0), + B_TEST(get_transport_by_bridge_addrport, 0), + B_TEST(transport_is_needed, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 057d9fa2dc..868f6a8ba4 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -8,6 +8,7 @@ #include "or.h" #include "buffers.h" #include "buffers_tls.h" +#include "crypto_rand.h" #include "proto_http.h" #include "proto_socks.h" #include "test.h" diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c new file mode 100644 index 0000000000..463a36b24e --- /dev/null +++ b/src/test/test_bwmgt.c @@ -0,0 +1,233 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_bwmgt.c + * \brief tests for bandwidth management / token bucket functions + */ + +#define TOKEN_BUCKET_PRIVATE + +#include "or.h" +#include "test.h" + +#include "token_bucket.h" + +// an imaginary time, in timestamp units. Chosen so it will roll over. +static const uint32_t START_TS = UINT32_MAX-10; +static const int32_t KB = 1024; +static const uint32_t GB = (U64_LITERAL(1) << 30); + +static void +test_bwmgt_token_buf_init(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + // Burst is correct + tt_uint_op(b.cfg.burst, OP_EQ, 64*KB); + // Rate is correct, within 1 percent. + { + uint32_t ticks_per_sec = + (uint32_t) monotime_msec_to_approx_coarse_stamp_units(1000); + uint32_t rate_per_sec = (b.cfg.rate * ticks_per_sec / TICKS_PER_STEP); + + tt_uint_op(rate_per_sec, OP_GT, 16*KB-160); + tt_uint_op(rate_per_sec, OP_LT, 16*KB+160); + } + // Bucket starts out full: + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS); + tt_int_op(b.read_bucket.bucket, OP_EQ, 64*KB); + + done: + ; +} + +static void +test_bwmgt_token_buf_adjust(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + + uint32_t rate_orig = b.cfg.rate; + // Increasing burst + token_bucket_rw_adjust(&b, 16*KB, 128*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 128*KB); + + // Decreasing burst but staying above bucket + token_bucket_rw_adjust(&b, 16*KB, 96*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 96*KB); + + // Decreasing burst below bucket, + token_bucket_rw_adjust(&b, 16*KB, 48*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 48*KB); + + // Changing rate. + token_bucket_rw_adjust(&b, 32*KB, 48*KB); + tt_uint_op(b.cfg.rate, OP_GE, rate_orig*2 - 10); + tt_uint_op(b.cfg.rate, OP_LE, rate_orig*2 + 10); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 48*KB); + + done: + ; +} + +static void +test_bwmgt_token_buf_dec(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + + // full-to-not-full. + tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, KB)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 63*KB); + + // Full to almost-not-full + tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 63*KB - 1)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 1); + + // almost-not-full to empty. + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 1)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 0); + + // reset bucket, try full-to-empty + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 0); + + // reset bucket, try underflow. + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB + 1)); + tt_int_op(b.read_bucket.bucket, OP_EQ, -1); + + // A second underflow does not make the bucket empty. + tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 1000)); + tt_int_op(b.read_bucket.bucket, OP_EQ, -1001); + + done: + ; +} + +static void +test_bwmgt_token_buf_refill(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + const uint32_t BW_SEC = + (uint32_t)monotime_msec_to_approx_coarse_stamp_units(1000); + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + + /* Make the buffer much emptier, then let one second elapse. */ + token_bucket_rw_dec_read(&b, 48*KB); + tt_int_op(b.read_bucket.bucket, OP_EQ, 16*KB); + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC)); + tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB - 300); + tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB + 300); + + /* Another half second. */ + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2)); + tt_int_op(b.read_bucket.bucket, OP_GT, 40*KB - 400); + tt_int_op(b.read_bucket.bucket, OP_LT, 40*KB + 400); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + BW_SEC*3/2); + + /* No time: nothing happens. */ + { + const uint32_t bucket_orig = b.read_bucket.bucket; + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2)); + tt_int_op(b.read_bucket.bucket, OP_EQ, bucket_orig); + } + + /* Another 30 seconds: fill the bucket. */ + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*30)); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, + START_TS + BW_SEC*3/2 + BW_SEC*30); + + /* Another 30 seconds: nothing happens. */ + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*60)); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, + START_TS + BW_SEC*3/2 + BW_SEC*60); + + /* Empty the bucket, let two seconds pass, and make sure that a refill is + * noticed. */ + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.cfg.burst)); + tt_int_op(0, OP_EQ, b.read_bucket.bucket); + tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*61)); + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*62)); + tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB+400); + + /* Underflow the bucket, make sure we detect when it has tokens again. */ + tt_int_op(1, OP_EQ, + token_bucket_rw_dec_read(&b, b.read_bucket.bucket+16*KB)); + tt_int_op(-16*KB, OP_EQ, b.read_bucket.bucket); + // half a second passes... + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64)); + tt_int_op(b.read_bucket.bucket, OP_GT, -8*KB-300); + tt_int_op(b.read_bucket.bucket, OP_LT, -8*KB+300); + // a second passes + tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*65)); + tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400); + + // We step a second backwards, and nothing happens. + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64)); + tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400); + + // A ridiculous amount of time passes. + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, INT32_MAX)); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + + done: + ; +} + +/* Test some helper functions we use within the token bucket interface. */ +static void +test_bwmgt_token_buf_helpers(void *arg) +{ + uint32_t ret; + + (void) arg; + + /* The returned value will be OS specific but in any case, it should be + * greater than 1 since we are passing 1GB/sec rate. */ + ret = rate_per_sec_to_rate_per_step(1 * GB); + tt_u64_op(ret, OP_GT, 1); + + /* We default to 1 in case rate is 0. */ + ret = rate_per_sec_to_rate_per_step(0); + tt_u64_op(ret, OP_EQ, 1); + + done: + ; +} + +#define BWMGT(name) \ + { #name, test_bwmgt_ ## name , 0, NULL, NULL } + +struct testcase_t bwmgt_tests[] = { + BWMGT(token_buf_init), + BWMGT(token_buf_adjust), + BWMGT(token_buf_dec), + BWMGT(token_buf_refill), + BWMGT(token_buf_helpers), + END_OF_TESTCASES +}; diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c index 88cdef383f..54d9716780 100644 --- a/src/test/test_cell_formats.c +++ b/src/test/test_cell_formats.c @@ -12,6 +12,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "config.h" +#include "crypto_rand.h" #include "onion.h" #include "onion_tap.h" #include "onion_fast.h" diff --git a/src/test/test_channel.c b/src/test/test_channel.c index bdc9d32f78..76124a6e75 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -12,6 +12,7 @@ #include "circuitmux_ewma.h" /* For var_cell_free */ #include "connection_or.h" +#include "crypto_rand.h" /* For packed_cell stuff */ #define RELAY_PRIVATE #include "relay.h" @@ -281,6 +282,7 @@ new_fake_channel(void) chan->state = CHANNEL_STATE_OPEN; chan->cmux = circuitmux_alloc(); + circuitmux_set_policy(chan->cmux, &ewma_policy); return chan; } @@ -543,6 +545,13 @@ test_channel_outbound_cell(void *arg) (void) arg; + /* Set the test time to be mocked, since this test assumes that no + * time will pass, ewma values will not need to be re-scaled, and so on */ + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(U64_LITERAL(1000000000) * 12345); + + cmux_ewma_set_options(NULL,NULL); + /* The channel will be freed so we need to hijack this so the scheduler * doesn't get confused. */ MOCK(scheduler_release_channel, scheduler_release_channel_mock); @@ -575,15 +584,13 @@ test_channel_outbound_cell(void *arg) channel_register(chan); tt_int_op(chan->registered, OP_EQ, 1); /* Set EWMA policy so we can pick it when flushing. */ - channel_set_cmux_policy_everywhere(&ewma_policy); + circuitmux_set_policy(chan->cmux, &ewma_policy); tt_ptr_op(circuitmux_get_policy(chan->cmux), OP_EQ, &ewma_policy); /* Register circuit to the channel circid map which will attach the circuit * to the channel's cmux as well. */ circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan); tt_int_op(channel_num_circuits(chan), OP_EQ, 1); - tt_assert(!TO_CIRCUIT(circ)->next_active_on_n_chan); - tt_assert(!TO_CIRCUIT(circ)->prev_active_on_n_chan); /* Test the cmux state. */ tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux); tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)), @@ -659,6 +666,7 @@ test_channel_outbound_cell(void *arg) tor_free(p_cell); channel_free_all(); UNMOCK(scheduler_release_channel); + monotime_disable_test_mocking(); } /* Test inbound cell. The callstack is: diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 90da2163a6..4261bc1b67 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -15,7 +15,6 @@ #include "channelpadding.h" #include "compat_libevent.h" #include "config.h" -#include <event2/event.h> #include "compat_time.h" #include "main.h" #include "networkstatus.h" @@ -65,7 +64,7 @@ mock_channel_write_cell_relay2(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)relay1_relay2)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -75,7 +74,7 @@ mock_channel_write_cell_relay1(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)relay2_relay1)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -85,7 +84,7 @@ mock_channel_write_cell_relay3(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)client_relay3)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -95,7 +94,7 @@ mock_channel_write_cell_client(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)relay3_client)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -105,7 +104,7 @@ mock_channel_write_cell(channel_t *chan, cell_t *cell) tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)chan)->conn); if (!dont_stop_libevent) - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -246,7 +245,7 @@ static void dummy_timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono) { (void)t; (void)arg; (void)now_mono; - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return; } @@ -264,7 +263,8 @@ dummy_nop_timer(void) timer_schedule(dummy_timer, &timeout); - event_base_loop(tor_libevent_get_base(), 0); + tor_libevent_run_event_loop(tor_libevent_get_base(), 0); + timer_free(dummy_timer); } diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c index d170009a9c..3794ffc2c6 100644 --- a/src/test/test_circuitlist.c +++ b/src/test/test_circuitlist.c @@ -9,6 +9,7 @@ #include "channel.h" #include "circuitbuild.h" #include "circuitlist.h" +#include "circuitmux_ewma.h" #include "hs_circuitmap.h" #include "test.h" #include "log_test_helpers.h" diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c index 854f725054..c81d53ae51 100644 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@ -3,14 +3,18 @@ #define TOR_CHANNEL_INTERNAL_ #define CIRCUITMUX_PRIVATE +#define CIRCUITMUX_EWMA_PRIVATE #define RELAY_PRIVATE #include "or.h" #include "channel.h" #include "circuitmux.h" +#include "circuitmux_ewma.h" #include "relay.h" #include "scheduler.h" #include "test.h" +#include <math.h> + /* XXXX duplicated function from test_circuitlist.c */ static channel_t * new_fake_channel(void) @@ -45,6 +49,7 @@ test_cmux_destroy_cell_queue(void *arg) cmux = circuitmux_alloc(); tt_assert(cmux); ch = new_fake_channel(); + circuitmux_set_policy(cmux, &ewma_policy); ch->has_queued_writes = has_queued_writes; ch->wide_circ_ids = 1; @@ -77,8 +82,50 @@ test_cmux_destroy_cell_queue(void *arg) tor_free(dc); } +static void +test_cmux_compute_ticks(void *arg) +{ + const int64_t NS_PER_S = 1000 * 1000 * 1000; + const int64_t START_NS = U64_LITERAL(1217709000)*NS_PER_S; + int64_t now; + double rem; + unsigned tick; + (void)arg; + circuitmux_ewma_free_all(); + monotime_enable_test_mocking(); + + monotime_coarse_set_mock_time_nsec(START_NS); + cell_ewma_initialize_ticks(); + const unsigned tick_zero = cell_ewma_get_current_tick_and_fraction(&rem); + tt_double_op(rem, OP_GT, -1e-9); + tt_double_op(rem, OP_LT, 1e-9); + + /* 1.5 second later and we should still be in the same tick. */ + now = START_NS + NS_PER_S + NS_PER_S/2; + monotime_coarse_set_mock_time_nsec(now); + tick = cell_ewma_get_current_tick_and_fraction(&rem); + tt_uint_op(tick, OP_EQ, tick_zero); +#ifdef USING_32BIT_MSEC_HACK + const double tolerance = .0005; +#else + const double tolerance = .00000001; +#endif + tt_double_op(fabs(rem - .15), OP_LT, tolerance); + + /* 25 second later and we should be in another tick. */ + now = START_NS + NS_PER_S * 25; + monotime_coarse_set_mock_time_nsec(now); + tick = cell_ewma_get_current_tick_and_fraction(&rem); + tt_uint_op(tick, OP_EQ, tick_zero + 2); + tt_double_op(fabs(rem - .5), OP_LT, tolerance); + + done: + ; +} + struct testcase_t circuitmux_tests[] = { { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL }, + { "compute_ticks", test_cmux_compute_ticks, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 7dd8e65194..85f69bd626 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -10,7 +10,6 @@ #include "compat_libevent.h" #include <event2/event.h> -#include <event2/thread.h> #include "log_test_helpers.h" @@ -122,10 +121,70 @@ test_compat_libevent_header_version(void *ignored) (void)0; } +/* Test for postloop events */ + +/* Event callback to increment a counter. */ +static void +increment_int_counter_cb(periodic_timer_t *timer, void *arg) +{ + (void)timer; + int *ctr = arg; + ++*ctr; +} + +static int activated_counter = 0; + +/* Mainloop event callback to activate another mainloop event */ +static void +activate_event_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + mainloop_event_t **other_event = arg; + mainloop_event_activate(*other_event); + ++activated_counter; +} + +static void +test_compat_libevent_postloop_events(void *arg) +{ + (void)arg; + mainloop_event_t *a = NULL, *b = NULL; + periodic_timer_t *timed = NULL; + + tor_libevent_postfork(); + + /* If postloop events don't work, then these events will activate one + * another ad infinitum and, and the periodic event will never occur. */ + b = mainloop_event_postloop_new(activate_event_cb, &a); + a = mainloop_event_postloop_new(activate_event_cb, &b); + + int counter = 0; + struct timeval fifty_ms = { 0, 10 * 1000 }; + timed = periodic_timer_new(tor_libevent_get_base(), &fifty_ms, + increment_int_counter_cb, &counter); + + mainloop_event_activate(a); + int r; + do { + r = tor_libevent_run_event_loop(tor_libevent_get_base(), 0); + if (r == -1) + break; + } while (counter < 5); + + tt_int_op(activated_counter, OP_GE, 2); + + done: + mainloop_event_free(a); + mainloop_event_free(b); + periodic_timer_free(timed); +} + struct testcase_t compat_libevent_tests[] = { { "logging_callback", test_compat_libevent_logging_callback, TT_FORK, NULL, NULL }, { "header_version", test_compat_libevent_header_version, 0, NULL, NULL }, + { "postloop_events", test_compat_libevent_postloop_events, + TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_config.c b/src/test/test_config.c index 2bcc0cad7b..8662b832b8 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -20,12 +20,11 @@ #include "connection_edge.h" #include "test.h" #include "util.h" -#include "address.h" #include "connection_or.h" #include "control.h" #include "cpuworker.h" #include "dirserv.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "dns.h" #include "entrynodes.h" #include "transports.h" @@ -42,9 +41,6 @@ #include "routerlist.h" #include "routerset.h" #include "statefile.h" -#include "test.h" -#include "transports.h" -#include "util.h" #include "test_helpers.h" diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index a9a4b6a98e..3b91baca39 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -9,6 +9,7 @@ #include "consdiff.h" #include "consdiffmgr.h" #include "cpuworker.h" +#include "crypto_rand.h" #include "networkstatus.h" #include "routerparse.h" #include "workqueue.h" diff --git a/src/test/test_containers.c b/src/test/test_containers.c index c4dba73750..3fc3523af4 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -5,6 +5,7 @@ #include "orconfig.h" #include "or.h" +#include "crypto_rand.h" #include "fp_pair.h" #include "test.h" diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 1c285bb3a2..1a350f66c0 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -1470,6 +1470,61 @@ test_download_status_bridge(void *arg) return; } +/** Set timeval to a mock date and time. This is necessary + * to make tor_gettimeofday() mockable. */ +static void +mock_tor_gettimeofday(struct timeval *timeval) +{ + timeval->tv_sec = 1523405073; + timeval->tv_usec = 271645; +} + +static void +test_current_time(void *arg) +{ + /* We just need one of these to pass, it doesn't matter what's in it */ + control_connection_t dummy; + /* Get results out */ + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + + /* We need these for storing the (mock) time. */ + MOCK(tor_gettimeofday, mock_tor_gettimeofday); + struct timeval now; + tor_gettimeofday(&now); + char timebuf[ISO_TIME_LEN+1]; + + /* Case 1 - local time */ + format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec); + getinfo_helper_current_time(&dummy, + "current-time/local", + &answer, &errmsg); + tt_ptr_op(answer, OP_NE, NULL); + tt_ptr_op(errmsg, OP_EQ, NULL); + tt_str_op(answer, OP_EQ, timebuf); + tor_free(answer); + errmsg = NULL; + + /* Case 2 - UTC time */ + format_iso_time_nospace(timebuf, (time_t)now.tv_sec); + getinfo_helper_current_time(&dummy, + "current-time/utc", + &answer, &errmsg); + tt_ptr_op(answer, OP_NE, NULL); + tt_ptr_op(errmsg, OP_EQ, NULL); + tt_str_op(answer, OP_EQ, timebuf); + tor_free(answer); + errmsg = NULL; + + done: + UNMOCK(tor_gettimeofday); + tor_free(answer); + + return; +} + struct testcase_t controller_tests[] = { { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0, NULL, NULL }, @@ -1486,6 +1541,7 @@ struct testcase_t controller_tests[] = { NULL }, { "download_status_desc", test_download_status_desc, 0, NULL, NULL }, { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL }, + { "current_time", test_current_time, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 901ad7ab3d..e81aea8d66 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -12,79 +12,6 @@ #include "test.h" static void -help_test_bucket_note_empty(uint32_t expected_msec_since_midnight, - int tokens_before, size_t tokens_removed, - uint32_t msec_since_epoch) -{ - uint32_t timestamp_var = 0; - struct timeval tvnow; - tvnow.tv_sec = msec_since_epoch / 1000; - tvnow.tv_usec = (msec_since_epoch % 1000) * 1000; - connection_buckets_note_empty_ts(×tamp_var, tokens_before, - tokens_removed, &tvnow); - tt_int_op(expected_msec_since_midnight, OP_EQ, timestamp_var); - - done: - ; -} - -static void -test_cntev_bucket_note_empty(void *arg) -{ - (void)arg; - - /* Two cases with nothing to note, because bucket was empty before; - * 86442200 == 1970-01-02 00:00:42.200000 */ - help_test_bucket_note_empty(0, 0, 0, 86442200); - help_test_bucket_note_empty(0, -100, 100, 86442200); - - /* Nothing to note, because bucket has not been emptied. */ - help_test_bucket_note_empty(0, 101, 100, 86442200); - - /* Bucket was emptied, note 42200 msec since midnight. */ - help_test_bucket_note_empty(42200, 101, 101, 86442200); - help_test_bucket_note_empty(42200, 101, 102, 86442200); -} - -static void -test_cntev_bucket_millis_empty(void *arg) -{ - struct timeval tvnow; - (void)arg; - - /* 1970-01-02 00:00:42.200000 */ - tvnow.tv_sec = 86400 + 42; - tvnow.tv_usec = 200000; - - /* Bucket has not been refilled. */ - tt_int_op(0, OP_EQ, bucket_millis_empty(0, 42120, 0, 100, &tvnow)); - tt_int_op(0, OP_EQ, bucket_millis_empty(-10, 42120, -10, 100, &tvnow)); - - /* Bucket was not empty. */ - tt_int_op(0, OP_EQ, bucket_millis_empty(10, 42120, 20, 100, &tvnow)); - - /* Bucket has been emptied 80 msec ago and has just been refilled. */ - tt_int_op(80, OP_EQ, bucket_millis_empty(-20, 42120, -10, 100, &tvnow)); - tt_int_op(80, OP_EQ, bucket_millis_empty(-10, 42120, 0, 100, &tvnow)); - tt_int_op(80, OP_EQ, bucket_millis_empty(0, 42120, 10, 100, &tvnow)); - - /* Bucket has been emptied 180 msec ago, last refill was 100 msec ago - * which was insufficient to make it positive, so cap msec at 100. */ - tt_int_op(100, OP_EQ, bucket_millis_empty(0, 42020, 1, 100, &tvnow)); - - /* 1970-01-02 00:00:00:050000 */ - tvnow.tv_sec = 86400; - tvnow.tv_usec = 50000; - - /* Last emptied 30 msec before midnight, tvnow is 50 msec after - * midnight, that's 80 msec in total. */ - tt_int_op(80, OP_EQ, bucket_millis_empty(0, 86400000 - 30, 1, 100, &tvnow)); - - done: - ; -} - -static void add_testing_cell_stats_entry(circuit_t *circ, uint8_t command, unsigned int waiting_time, unsigned int removed, unsigned int exitward) @@ -395,8 +322,6 @@ test_cntev_event_mask(void *arg) { #name, test_cntev_ ## name, flags, 0, NULL } struct testcase_t controller_event_tests[] = { - TEST(bucket_note_empty, TT_FORK), - TEST(bucket_millis_empty, TT_FORK), TEST(sum_up_cell_stats, TT_FORK), TEST(append_cell_stats, TT_FORK), TEST(format_cell_stats, TT_FORK), diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 83d97f2867..bb2e340dd2 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -5,7 +5,7 @@ #include "orconfig.h" #define CRYPTO_CURVE25519_PRIVATE -#define CRYPTO_PRIVATE +#define CRYPTO_RAND_PRIVATE #include "or.h" #include "test.h" #include "aes.h" @@ -13,6 +13,7 @@ #include "siphash.h" #include "crypto_curve25519.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "ed25519_vectors.inc" /** Run unit tests for Diffie-Hellman functionality. */ diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c index 090cb4242b..a016277508 100644 --- a/src/test/test_crypto_openssl.c +++ b/src/test/test_crypto_openssl.c @@ -5,9 +5,9 @@ #include "orconfig.h" -#define CRYPTO_PRIVATE +#define CRYPTO_RAND_PRIVATE -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "util_format.h" #include "compat.h" diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index 2afb71ff5a..0e1f5bd227 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -9,6 +9,7 @@ #include "test.h" #include "crypto_s2k.h" #include "crypto_pwbox.h" +#include "crypto_rand.h" #if defined(HAVE_LIBSCRYPT_H) && defined(HAVE_LIBSCRYPT_SCRYPT) #define HAVE_LIBSCRYPT diff --git a/src/test/test_dir.c b/src/test/test_dir.c index a33b23bcc9..37b015b72d 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -23,9 +23,10 @@ #include "config.h" #include "control.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "entrynodes.h" #include "hibernate.h" #include "memarea.h" @@ -35,12 +36,13 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" -#include "shared_random_state.h" +#include "dirauth/shared_random_state.h" #include "test.h" #include "test_dir_common.h" #include "torcert.h" #include "relay.h" #include "log_test_helpers.h" +#include "voting_schedule.h" #define NS_MODULE dir @@ -1499,6 +1501,13 @@ test_dir_measured_bw_kb(void *arg) "bw=1024 junk=007\n", "misc=junk node_id=$557365204145532d32353620696e73746561642e " "bw=1024 junk=007\n", + /* check whether node_id can be at the end */ + "bw=1024 node_id=$557365204145532d32353620696e73746561642e\n", + /* check whether node_id can be at the end and bw has something in front*/ + "foo=bar bw=1024 node_id=$557365204145532d32353620696e73746561642e\n", + /* check whether node_id can be at the end and something in the + * in the middle of bw and node_id */ + "bw=1024 foo=bar node_id=$557365204145532d32353620696e73746561642e\n", "end" }; const char *lines_fail[] = { @@ -1538,12 +1547,18 @@ test_dir_measured_bw_kb(void *arg) (void)arg; for (i = 0; strcmp(lines_fail[i], "end"); i++) { //fprintf(stderr, "Testing: %s\n", lines_fail[i]); - tt_int_op(measured_bw_line_parse(&mbwl, lines_fail[i]), OP_EQ, -1); + /* Testing only with line_is_after_headers = 1. Tests with + * line_is_after_headers = 0 in + * test_dir_measured_bw_kb_line_is_after_headers */ + tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1); } for (i = 0; strcmp(lines_pass[i], "end"); i++) { //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); - tt_int_op(measured_bw_line_parse(&mbwl, lines_pass[i]), OP_EQ, 0); + /* Testing only with line_is_after_headers = 1. Tests with + * line_is_after_headers = 0 in + * test_dir_measured_bw_kb_line_is_after_headers */ + tt_assert(measured_bw_line_parse(&mbwl, lines_pass[i], 1) == 0); tt_assert(mbwl.bw_kb == 1024); tt_assert(strcmp(mbwl.node_hex, "557365204145532d32353620696e73746561642e") == 0); @@ -1555,7 +1570,7 @@ test_dir_measured_bw_kb(void *arg) /* Test dirserv_read_measured_bandwidths */ static void -test_dir_dirserv_read_measured_bandwidths(void *arg) +test_dir_dirserv_read_measured_bandwidths_empty(void *arg) { char *fname=NULL; (void)arg; @@ -1572,6 +1587,129 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) teardown_capture_of_logs(); } +/* Unit tests for measured_bw_line_parse using line_is_after_headers flag. + * When the end of the header is detected (a first complete bw line is parsed), + * incomplete lines fail and give warnings, but do not give warnings if + * the header is not ended, allowing to ignore additional header lines. */ +static void +test_dir_measured_bw_kb_line_is_after_headers(void *arg) +{ + (void)arg; + measured_bw_line_t mbwl; + const char *line_pass = \ + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n"; + int i; + const char *lines_fail[] = { + "node_id=$557365204145532d32353620696e73746561642e \n", + "bw=1024\n", + "rtt=300\n", + "end" + }; + + setup_capture_of_logs(LOG_DEBUG); + + /* Test bw lines when header has ended */ + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1); + expect_log_msg_containing("Incomplete line in bandwidth file:"); + mock_clean_saved_logs(); + } + + tt_assert(measured_bw_line_parse(&mbwl, line_pass, 1) == 0); + + /* Test bw lines when header has not ended */ + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 0) == -1); + expect_log_msg_containing("Missing bw or node_id in bandwidth file line:"); + mock_clean_saved_logs(); + } + + tt_assert(measured_bw_line_parse(&mbwl, line_pass, 0) == 0); + + done: + teardown_capture_of_logs(); +} + +/* Test dirserv_read_measured_bandwidths with whole files. */ +static void +test_dir_dirserv_read_measured_bandwidths(void *arg) +{ + (void)arg; + char *content = NULL; + time_t timestamp = time(NULL); + char *fname = tor_strdup(get_fname("V3BandwidthsFile")); + + /* Test Torflow file only with timestamp*/ + tor_asprintf(&content, "%ld", (long)timestamp); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow file with timestamp followed by '\n' */ + tor_asprintf(&content, "%ld\n", (long)timestamp); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow complete file*/ + const char *torflow_relay_lines= + "node_id=$557365204145532d32353620696e73746561642e bw=1024 " + "nick=Test measured_at=1523911725 updated_at=1523911725 " + "pid_error=4.11374090719 pid_error_sum=4.11374090719 " + "pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 " + "scanner=/filepath\n"; + + tor_asprintf(&content, "%ld\n%s", (long)timestamp, torflow_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow complete file including v1.1.0 headers */ + const char *v110_header_lines= + "version=1.1.0\n" + "software=sbws\n" + "software_version=0.1.0\n" + "generator_started=2018-05-08T16:13:25\n" + "earliest_bandwidth=2018-05-08T16:13:26\n" + "====\n"; + + tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, v110_header_lines, + torflow_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow with additional headers afer a correct bw line */ + tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, torflow_relay_lines, + v110_header_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow with additional headers afer a correct bw line and more + * bw lines after the headers. */ + tor_asprintf(&content, "%ld\n%s%s%s", (long)timestamp, torflow_relay_lines, + v110_header_lines, torflow_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test sbws file */ + const char *sbws_relay_lines= + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test rtt=380 time=2018-05-08T16:13:26\n"; + + tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, v110_header_lines, + sbws_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + done: + tor_free(fname); +} + #define MBWC_INIT_TIME 1000 /** Do the measured bandwidth cache unit test */ @@ -2398,7 +2536,7 @@ test_a_networkstatus( sign_skey_2 = crypto_pk_new(); sign_skey_3 = crypto_pk_new(); sign_skey_leg1 = pk_generate(4); - dirvote_recalculate_timing(get_options(), now); + voting_schedule_recalculate_timing(get_options(), now); sr_state_init(0, 0); tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, @@ -2936,8 +3074,9 @@ gen_routerstatus_for_umbw(int idx, time_t now) rs->addr = 0x99008801; rs->or_port = 443; rs->dir_port = 8000; - /* all flags but running cleared */ + /* all flags but running and valid cleared */ rs->is_flagged_running = 1; + rs->is_valid = 1; /* * This one has measured bandwidth below the clip cutoff, and * so shouldn't be clipped; we'll have to test that it isn't @@ -3010,8 +3149,9 @@ gen_routerstatus_for_umbw(int idx, time_t now) rs->addr = 0xC0000203; rs->or_port = 500; rs->dir_port = 1999; - /* all flags but running cleared */ + /* all flags but running and valid cleared */ rs->is_flagged_running = 1; + rs->is_valid = 1; /* * This one has unmeasured bandwidth below the clip cutoff, and * so shouldn't be clipped; we'll have to test that it isn't @@ -3033,7 +3173,7 @@ gen_routerstatus_for_umbw(int idx, time_t now) if (vrs) { vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t)); tor_asprintf(&vrs->microdesc->microdesc_hash_line, - "m 9,10,11,12,13,14,15,16,17 " + "m 25,26,27,28 " "sha256=xyzajkldsdsajdadlsdjaslsdksdjlsdjsdaskdaaa%d\n", idx); } @@ -3059,7 +3199,7 @@ vote_tweaks_for_umbw(networkstatus_t *v, int voter, time_t now) smartlist_clear(v->supported_methods); /* Method 17 is MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB */ smartlist_split_string(v->supported_methods, - "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17", + "25 26 27 28", NULL, 0, -1); /* If we're using a non-default clip bandwidth, add it to net_params */ if (alternate_clip_bw > 0) { @@ -3221,9 +3361,9 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now) tt_assert(!rs->is_fast); tt_assert(!rs->is_possible_guard); tt_assert(!rs->is_stable); - /* (If it wasn't running it wouldn't be here) */ + /* (If it wasn't running and valid it wouldn't be here) */ tt_assert(rs->is_flagged_running); - tt_assert(!rs->is_valid); + tt_assert(rs->is_valid); tt_assert(!rs->is_named); /* This one should have measured bandwidth below the clip cutoff */ tt_assert(rs->has_bandwidth); @@ -4082,34 +4222,19 @@ test_dir_download_status_increment(void *arg) DL_WANT_ANY_DIRSERVER, DL_SCHED_INCREMENT_ATTEMPT, 0, 0 }; - int no_delay = 0; - int delay0 = -1; - int delay1 = -1; - int delay2 = -1; - smartlist_t *schedule = smartlist_new(); - smartlist_t *schedule_no_initial_delay = smartlist_new(); or_options_t test_options; time_t current_time = time(NULL); - /* Provide some values for the schedules */ - delay0 = 10; - delay1 = 99; - delay2 = 20; - - /* Make the schedules */ - smartlist_add(schedule, (void *)&delay0); - smartlist_add(schedule, (void *)&delay1); - smartlist_add(schedule, (void *)&delay2); - - smartlist_add(schedule_no_initial_delay, (void *)&no_delay); - smartlist_add(schedule_no_initial_delay, (void *)&delay1); - smartlist_add(schedule_no_initial_delay, (void *)&delay2); + const int delay0 = 10; + const int no_delay = 0; + const int schedule = 10; + const int schedule_no_initial_delay = 0; /* Put it in the options */ mock_options = &test_options; reset_options(mock_options, &mock_get_options_calls); - mock_options->TestingBridgeBootstrapDownloadSchedule = schedule; - mock_options->TestingClientDownloadSchedule = schedule; + mock_options->TestingBridgeBootstrapDownloadInitialDelay = schedule; + mock_options->TestingClientDownloadInitialDelay = schedule; MOCK(get_options, mock_get_options); @@ -4117,13 +4242,13 @@ test_dir_download_status_increment(void *arg) * whether or not it was reset before being used */ /* regression test for 17750: no initial delay */ - mock_options->TestingClientDownloadSchedule = schedule_no_initial_delay; + mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay; mock_get_options_calls = 0; /* we really want to test that it's equal to time(NULL) + delay0, but that's * an unrealiable test, because time(NULL) might change. */ /* regression test for 17750: exponential, no initial delay */ - mock_options->TestingClientDownloadSchedule = schedule_no_initial_delay; + mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay; mock_get_options_calls = 0; /* we really want to test that it's equal to time(NULL) + delay0, but that's * an unrealiable test, because time(NULL) might change. */ @@ -4136,7 +4261,7 @@ test_dir_download_status_increment(void *arg) tt_int_op(mock_get_options_calls, OP_GE, 1); /* regression test for 17750: exponential, initial delay */ - mock_options->TestingClientDownloadSchedule = schedule; + mock_options->TestingClientDownloadInitialDelay = schedule; mock_get_options_calls = 0; /* we really want to test that it's equal to time(NULL) + delay0, but that's * an unrealiable test, because time(NULL) might change. */ @@ -4149,9 +4274,6 @@ test_dir_download_status_increment(void *arg) tt_int_op(mock_get_options_calls, OP_GE, 1); done: - /* the pointers in schedule are allocated on the stack */ - smartlist_free(schedule); - smartlist_free(schedule_no_initial_delay); UNMOCK(get_options); mock_options = NULL; mock_get_options_calls = 0; @@ -5469,7 +5591,7 @@ mock_num_bridges_usable(int use_maybe_reachable) * fallbacks. */ static void -test_dir_find_dl_schedule(void* data) +test_dir_find_dl_min_delay(void* data) { const char *str = (const char *)data; @@ -5502,44 +5624,45 @@ test_dir_find_dl_schedule(void* data) mock_num_bridges_usable); download_status_t dls; - smartlist_t server, client, server_cons, client_cons; - smartlist_t client_boot_auth_only_cons, client_boot_auth_cons; - smartlist_t client_boot_fallback_cons, bridge, bridge_bootstrap; + + const int server=10, client=20, server_cons=30, client_cons=40; + const int client_boot_auth_only_cons=50, client_boot_auth_cons=60; + const int client_boot_fallback_cons=70, bridge=80, bridge_bootstrap=90; mock_options = tor_malloc(sizeof(or_options_t)); reset_options(mock_options, &mock_get_options_calls); MOCK(get_options, mock_get_options); - mock_options->TestingServerDownloadSchedule = &server; - mock_options->TestingClientDownloadSchedule = &client; - mock_options->TestingServerConsensusDownloadSchedule = &server_cons; - mock_options->TestingClientConsensusDownloadSchedule = &client_cons; - mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule = - &client_boot_auth_only_cons; - mock_options->ClientBootstrapConsensusAuthorityDownloadSchedule = - &client_boot_auth_cons; - mock_options->ClientBootstrapConsensusFallbackDownloadSchedule = - &client_boot_fallback_cons; - mock_options->TestingBridgeDownloadSchedule = &bridge; - mock_options->TestingBridgeBootstrapDownloadSchedule = &bridge_bootstrap; + mock_options->TestingServerDownloadInitialDelay = server; + mock_options->TestingClientDownloadInitialDelay = client; + mock_options->TestingServerConsensusDownloadInitialDelay = server_cons; + mock_options->TestingClientConsensusDownloadInitialDelay = client_cons; + mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay = + client_boot_auth_only_cons; + mock_options->ClientBootstrapConsensusAuthorityDownloadInitialDelay = + client_boot_auth_cons; + mock_options->ClientBootstrapConsensusFallbackDownloadInitialDelay = + client_boot_fallback_cons; + mock_options->TestingBridgeDownloadInitialDelay = bridge; + mock_options->TestingBridgeBootstrapDownloadInitialDelay = bridge_bootstrap; dls.schedule = DL_SCHED_GENERIC; /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &client); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, client); mock_options->ClientOnly = 0; /* dir mode */ mock_options->DirPort_set = 1; mock_options->DirCache = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server); mock_options->DirPort_set = 0; mock_options->DirCache = 0; dls.schedule = DL_SCHED_CONSENSUS; /* public server mode */ mock_options->ORPort_set = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server_cons); mock_options->ORPort_set = 0; /* client and bridge modes */ @@ -5548,30 +5671,30 @@ test_dir_find_dl_schedule(void* data) dls.want_authority = 1; /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; dls.want_authority = 0; /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_fallback_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_fallback_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_fallback_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_fallback_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; @@ -5579,30 +5702,30 @@ test_dir_find_dl_schedule(void* data) /* dls.want_authority is ignored */ /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_only_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_only_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_only_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_only_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; } } else { /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; } @@ -5612,9 +5735,9 @@ test_dir_find_dl_schedule(void* data) mock_options->ClientOnly = 1; mock_options->UseBridges = 1; if (num_bridges_usable(0) > 0) { - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &bridge); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge); } else { - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &bridge_bootstrap); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap); } done: @@ -5634,9 +5757,8 @@ test_dir_assumed_flags(void *arg) memarea_t *area = memarea_new(); routerstatus_t *rs = NULL; - /* First, we should always assume that the Running flag is set, even - * when it isn't listed, since the consensus method is always - * higher than 4. */ + /* We can assume that consensus method is higher than 24, so Running and + * Valid are always implicitly set */ const char *str1 = "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " "192.168.0.1 9001 0\n" @@ -5645,17 +5767,6 @@ test_dir_assumed_flags(void *arg) const char *cp = str1; rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, - 23, FLAV_MICRODESC); - tt_assert(rs); - tt_assert(rs->is_flagged_running); - tt_assert(! rs->is_valid); - tt_assert(! rs->is_exit); - tt_assert(rs->is_fast); - routerstatus_free(rs); - - /* With method 24 or later, we can assume "valid" is set. */ - cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); @@ -5788,22 +5899,10 @@ test_dir_networkstatus_consensus_has_ipv6(void *arg) /* Test the bounds for A lines in the NS consensus */ mock_options->UseMicrodescriptors = 0; - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES; - has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); - tt_assert(has_ipv6); - - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES + 1; + mock_networkstatus->consensus_method = MIN_SUPPORTED_CONSENSUS_METHOD; has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); tt_assert(has_ipv6); - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES + 20; - has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); - tt_assert(has_ipv6); - - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES - 1; - has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); - tt_assert(!has_ipv6); - /* Test the bounds for A lines in the microdesc consensus */ mock_options->UseMicrodescriptors = 1; @@ -5812,6 +5911,10 @@ test_dir_networkstatus_consensus_has_ipv6(void *arg) has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); tt_assert(has_ipv6); + mock_networkstatus->consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD + 20; + has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); + tt_assert(has_ipv6); + mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 1; has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); @@ -5926,9 +6029,11 @@ struct testcase_t dir_tests[] = { DIR_LEGACY(versions), DIR_LEGACY(fp_pairs), DIR(split_fps, 0), - DIR_LEGACY(dirserv_read_measured_bandwidths), + DIR_LEGACY(dirserv_read_measured_bandwidths_empty), DIR_LEGACY(measured_bw_kb), + DIR_LEGACY(measured_bw_kb_line_is_after_headers), DIR_LEGACY(measured_bw_kb_cache), + DIR_LEGACY(dirserv_read_measured_bandwidths), DIR_LEGACY(param_voting), DIR(param_voting_lookup, 0), DIR_LEGACY(v3_networkstatus), @@ -5960,14 +6065,14 @@ struct testcase_t dir_tests[] = { DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0), - DIR_ARG(find_dl_schedule, TT_FORK, "bfd"), - DIR_ARG(find_dl_schedule, TT_FORK, "bad"), - DIR_ARG(find_dl_schedule, TT_FORK, "cfd"), - DIR_ARG(find_dl_schedule, TT_FORK, "cad"), - DIR_ARG(find_dl_schedule, TT_FORK, "bfr"), - DIR_ARG(find_dl_schedule, TT_FORK, "bar"), - DIR_ARG(find_dl_schedule, TT_FORK, "cfr"), - DIR_ARG(find_dl_schedule, TT_FORK, "car"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bfd"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bad"), + DIR_ARG(find_dl_min_delay, TT_FORK, "cfd"), + DIR_ARG(find_dl_min_delay, TT_FORK, "cad"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bfr"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bar"), + DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), + DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index fdf43533a8..230410f7fa 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -5,14 +5,14 @@ #include "orconfig.h" #define DIRVOTE_PRIVATE -#include "crypto.h" #include "test.h" #include "container.h" #include "or.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "nodelist.h" #include "routerlist.h" #include "test_dir_common.h" +#include "voting_schedule.h" void dir_common_setup_vote(networkstatus_t **vote, time_t now); networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote, diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index ca64dce5fe..688d26bdc1 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -16,7 +16,6 @@ #include "directory.h" #include "test.h" #include "compress.h" -#include "connection.h" #include "rendcommon.h" #include "rendcache.h" #include "router.h" @@ -31,8 +30,9 @@ #include "proto_http.h" #include "geoip.h" #include "dirserv.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "log_test_helpers.h" +#include "voting_schedule.h" #ifdef _WIN32 /* For mkdir() */ @@ -2057,7 +2057,7 @@ test_dir_handle_get_status_vote_d(void* data) mock_options->TestingV3AuthInitialDistDelay = 1; time_t now = 1441223455 -1; - dirvote_recalculate_timing(mock_options, now); + voting_schedule_recalculate_timing(mock_options, now); const char *msg_out = NULL; int status_out = 0; @@ -2403,7 +2403,7 @@ test_dir_handle_get_status_vote_next_authority(void* data) mock_options->TestingV3AuthInitialDistDelay = 1; time_t now = 1441223455 -1; - dirvote_recalculate_timing(mock_options, now); + voting_schedule_recalculate_timing(mock_options, now); struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out, &status_out); @@ -2482,7 +2482,7 @@ test_dir_handle_get_status_vote_current_authority(void* data) mock_options->TestingV3AuthInitialDistDelay = 1; time_t now = 1441223455; - dirvote_recalculate_timing(mock_options, now-1); + voting_schedule_recalculate_timing(mock_options, now-1); struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out, &status_out); diff --git a/src/test/test_dos.c b/src/test/test_dos.c index cb9d9e559c..8ae967f3ae 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -8,6 +8,7 @@ #include "or.h" #include "dos.h" #include "circuitlist.h" +#include "crypto_rand.h" #include "geoip.h" #include "channel.h" #include "microdesc.h" diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 3b54ffa2e9..e0162e3058 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -4,6 +4,7 @@ #include "orconfig.h" #define CIRCUITLIST_PRIVATE +#define CIRCUITBUILD_PRIVATE #define STATEFILE_PRIVATE #define ENTRYNODES_PRIVATE #define ROUTERLIST_PRIVATE @@ -14,8 +15,10 @@ #include "bridges.h" #include "circuitlist.h" +#include "circuitbuild.h" #include "config.h" #include "confparse.h" +#include "crypto_rand.h" #include "directory.h" #include "entrynodes.h" #include "nodelist.h" @@ -74,6 +77,17 @@ bfn_mock_node_get_by_id(const char *id) return NULL; } +/* Helper function to free a test node. */ +static void +test_node_free(node_t *n) +{ + tor_free(n->rs); + tor_free(n->md->onion_curve25519_pkey); + short_policy_free(n->md->exit_policy); + tor_free(n->md); + tor_free(n); +} + /* Unittest cleanup function: Cleanup the fake network. */ static int big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) @@ -83,9 +97,7 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) if (big_fake_net_nodes) { SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, { - tor_free(n->rs); - tor_free(n->md); - tor_free(n); + test_node_free(n); }); smartlist_free(big_fake_net_nodes); } @@ -113,9 +125,18 @@ big_fake_network_setup(const struct testcase_t *testcase) big_fake_net_nodes = smartlist_new(); for (i = 0; i < N_NODES; ++i) { + curve25519_secret_key_t curve25519_secret_key; + node_t *n = tor_malloc_zero(sizeof(node_t)); n->md = tor_malloc_zero(sizeof(microdesc_t)); + /* Generate curve25519 key for this node */ + n->md->onion_curve25519_pkey = + tor_malloc_zero(sizeof(curve25519_public_key_t)); + curve25519_secret_key_generate(&curve25519_secret_key, 0); + curve25519_public_key_generate(n->md->onion_curve25519_pkey, + &curve25519_secret_key); + crypto_rand(n->identity, sizeof(n->identity)); n->rs = tor_malloc_zero(sizeof(routerstatus_t)); @@ -135,8 +156,8 @@ big_fake_network_setup(const struct testcase_t *testcase) { char nickname_binary[8]; crypto_rand(nickname_binary, sizeof(nickname_binary)); - base64_encode(n->rs->nickname, sizeof(n->rs->nickname), - nickname_binary, sizeof(nickname_binary), 0); + base32_encode(n->rs->nickname, sizeof(n->rs->nickname), + nickname_binary, sizeof(nickname_binary)); } /* Call half of the nodes a possible guard. */ @@ -144,6 +165,12 @@ big_fake_network_setup(const struct testcase_t *testcase) n->is_possible_guard = 1; n->rs->guardfraction_percentage = 100; n->rs->has_guardfraction = 1; + n->rs->is_possible_guard = 1; + } + + /* Make some of these nodes a possible exit */ + if (i % 7 == 0) { + n->md->exit_policy = parse_short_policy("accept 443"); } smartlist_add(big_fake_net_nodes, n); @@ -1075,9 +1102,7 @@ test_entry_guard_expand_sample_small_net(void *arg) /* Fun corner case: not enough guards to make up our whole sample size. */ SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, { if (n_sl_idx >= 15) { - tor_free(n->rs); - tor_free(n->md); - tor_free(n); + test_node_free(n); SMARTLIST_DEL_CURRENT(big_fake_net_nodes, n); } else { n->rs->addr = 0; // make the filter reject this. @@ -1174,9 +1199,7 @@ test_entry_guard_update_from_consensus_status(void *arg) node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity); tt_assert(n); smartlist_remove(big_fake_net_nodes, n); - tor_free(n->rs); - tor_free(n->md); - tor_free(n); + test_node_free(n); } update_approx_time(start + 300); sampled_guards_update_from_consensus(gs); @@ -2685,6 +2708,23 @@ test_enty_guard_should_expire_waiting(void *arg) tor_free(fake_state); } +/** Test that the number of primary guards can be controlled using torrc */ +static void +test_entry_guard_number_of_primaries(void *arg) +{ + (void) arg; + + /* Get default value */ + tt_int_op(get_n_primary_guards(), OP_EQ, DFLT_N_PRIMARY_GUARDS); + + /* Set number of primaries using torrc */ + get_options_mutable()->NumPrimaryGuards = 42; + tt_int_op(get_n_primary_guards(), OP_EQ, 42); + + done: + ; +} + static void mock_directory_initiate_request(directory_request_t *req) { @@ -2796,6 +2836,161 @@ test_entry_guard_outdated_dirserver_exclusion(void *arg) } } +/** Test helper to extend the <b>oc</b> circuit path <b>n</b> times and then + * ensure that the circuit is now complete. */ +static void +helper_extend_circuit_path_n_times(origin_circuit_t *oc, int n) +{ + int retval; + int i; + + /* Extend path n times */ + for (i = 0 ; i < n ; i++) { + retval = onion_extend_cpath(oc); + tt_int_op(retval, OP_EQ, 0); + tt_int_op(circuit_get_cpath_len(oc), OP_EQ, i+1); + } + + /* Now do it one last time and see that circ is complete */ + retval = onion_extend_cpath(oc); + tt_int_op(retval, OP_EQ, 1); + + done: + ; +} + +/** Test for basic Tor path selection. Makes sure we build 3-hop circuits. */ +static void +test_entry_guard_basic_path_selection(void *arg) +{ + (void) arg; + + int retval; + + /* Enable entry guards */ + or_options_t *options = get_options_mutable(); + options->UseEntryGuards = 1; + + /* disables /16 check since all nodes have the same addr... */ + options->EnforceDistinctSubnets = 0; + + /* Create our circuit */ + circuit_t *circ = dummy_origin_circuit_new(30); + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); + + /* First pick the exit and pin it on the build_state */ + retval = onion_pick_cpath_exit(oc, NULL, 0); + tt_int_op(retval, OP_EQ, 0); + + /* Extend path 3 times. First we pick guard, then middle, then exit. */ + helper_extend_circuit_path_n_times(oc, 3); + + done: + circuit_free_(circ); +} + +/** Test helper to build an L2 and L3 vanguard list. The vanguard lists + * produced should be completely disjoint. */ +static void +helper_setup_vanguard_list(or_options_t *options) +{ + int i = 0; + + /* Add some nodes to the vanguard L2 list */ + options->HSLayer2Nodes = routerset_new(); + for (i = 0; i < 10 ; i += 2) { + node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i); + tt_assert(vanguard_node->is_possible_guard); + routerset_parse(options->HSLayer2Nodes, vanguard_node->rs->nickname, "l2"); + } + /* also add some nodes to vanguard L3 list + * (L2 list and L3 list should be disjoint for this test to work) */ + options->HSLayer3Nodes = routerset_new(); + for (i = 10; i < 20 ; i += 2) { + node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i); + tt_assert(vanguard_node->is_possible_guard); + routerset_parse(options->HSLayer3Nodes, vanguard_node->rs->nickname, "l3"); + } + + done: + ; +} + +/** Test to ensure that vanguard path selection works properly. Ensures that + * default vanguard circuits are 4 hops, and that path selection works + * correctly given the vanguard settings. */ +static void +test_entry_guard_vanguard_path_selection(void *arg) +{ + (void) arg; + + int retval; + + /* Enable entry guards */ + or_options_t *options = get_options_mutable(); + options->UseEntryGuards = 1; + + /* XXX disables /16 check */ + options->EnforceDistinctSubnets = 0; + + /* Setup our vanguard list */ + helper_setup_vanguard_list(options); + + /* Create our circuit */ + circuit_t *circ = dummy_origin_circuit_new(30); + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); + oc->build_state->is_internal = 1; + + /* Switch circuit purpose to vanguards */ + circ->purpose = CIRCUIT_PURPOSE_HS_VANGUARDS; + + /* First pick the exit and pin it on the build_state */ + tt_int_op(oc->build_state->desired_path_len, OP_EQ, 0); + retval = onion_pick_cpath_exit(oc, NULL, 0); + tt_int_op(retval, OP_EQ, 0); + + /* Ensure that vanguards make 4-hop circuits by default */ + tt_int_op(oc->build_state->desired_path_len, OP_EQ, 4); + + /* Extend path as many times as needed to have complete circ. */ + helper_extend_circuit_path_n_times(oc, oc->build_state->desired_path_len); + + /* Test that the cpath linked list is set correctly. */ + crypt_path_t *l1_node = oc->cpath; + crypt_path_t *l2_node = l1_node->next; + crypt_path_t *l3_node = l2_node->next; + crypt_path_t *l4_node = l3_node->next; + crypt_path_t *l1_node_again = l4_node->next; + tt_ptr_op(l1_node, OP_EQ, l1_node_again); + + /* Test that L2 is indeed HSLayer2Node */ + retval = routerset_contains_extendinfo(options->HSLayer2Nodes, + l2_node->extend_info); + tt_int_op(retval, OP_EQ, 4); + /* test that L3 node is _not_ contained in HSLayer2Node */ + retval = routerset_contains_extendinfo(options->HSLayer2Nodes, + l3_node->extend_info); + tt_int_op(retval, OP_LT, 4); + + /* Test that L3 is indeed HSLayer3Node */ + retval = routerset_contains_extendinfo(options->HSLayer3Nodes, + l3_node->extend_info); + tt_int_op(retval, OP_EQ, 4); + /* test that L2 node is _not_ contained in HSLayer3Node */ + retval = routerset_contains_extendinfo(options->HSLayer3Nodes, + l2_node->extend_info); + tt_int_op(retval, OP_LT, 4); + + /* TODO: Test that L1 can be the same as exit. To test this we need start + enforcing EnforceDistinctSubnets again, which means that we need to give + each test node a different address which currently breaks some tests. */ + + done: + circuit_free_(circ); +} + static const struct testcase_setup_t big_fake_network = { big_fake_network_setup, big_fake_network_cleanup }; @@ -2835,6 +3030,8 @@ struct testcase_t entrynodes_tests[] = { test_entry_guard_parse_from_state_broken, TT_FORK, NULL, NULL }, { "get_guard_selection_by_name", test_entry_guard_get_guard_selection_by_name, TT_FORK, NULL, NULL }, + { "number_of_primaries", + test_entry_guard_number_of_primaries, TT_FORK, NULL, NULL }, BFN_TEST(choose_selection_initial), BFN_TEST(add_single_guard), BFN_TEST(node_filter), @@ -2857,6 +3054,8 @@ struct testcase_t entrynodes_tests[] = { BFN_TEST(select_and_cancel), BFN_TEST(drop_guards), BFN_TEST(outdated_dirserver_exclusion), + BFN_TEST(basic_path_selection), + BFN_TEST(vanguard_path_selection), UPGRADE_TEST(upgrade_a_circuit, "c1-done c2-done"), UPGRADE_TEST(upgrade_blocked_by_live_primary_guards, "c1-done c2-done"), diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index cadef257f1..e05342cb8a 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -10,6 +10,7 @@ #include "connection_or.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" #include "ext_orport.h" #include "main.h" #include "test.h" diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c new file mode 100644 index 0000000000..6f849f436b --- /dev/null +++ b/src/test/test_geoip.c @@ -0,0 +1,578 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +/* These macros pull in declarations for some functions and structures that + * are typically file-private. */ +#define GEOIP_PRIVATE +#include "or.h" +#include "config.h" +#include "geoip.h" +#include "test.h" + + /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs + * using ipv4. Since our fake geoip database is the same between + * ipv4 and ipv6, we should get the same result no matter which + * address family we pick for each IP. */ +#define SET_TEST_ADDRESS(i) do { \ + if ((i) & 1) { \ + SET_TEST_IPV6(i); \ + tor_addr_from_in6(&addr, &in6); \ + } else { \ + tor_addr_from_ipv4h(&addr, (uint32_t) i); \ + } \ + } while (0) + + /* Make sure that country ID actually works. */ +#define SET_TEST_IPV6(i) \ + do { \ + set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ + } while (0) +#define CHECK_COUNTRY(country, val) do { \ + /* test ipv4 country lookup */ \ + tt_str_op(country, OP_EQ, \ + geoip_get_country_name(geoip_get_country_by_ipv4(val))); \ + /* test ipv6 country lookup */ \ + SET_TEST_IPV6(val); \ + tt_str_op(country, OP_EQ, \ + geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \ + } while (0) + +/** Run unit tests for GeoIP code. */ +static void +test_geoip(void *arg) +{ + int i, j; + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL, *v = NULL; + const char *bridge_stats_1 = + "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "bridge-ips zz=24,xy=8\n" + "bridge-ip-versions v4=16,v6=16\n" + "bridge-ip-transports <OR>=24\n", + *dirreq_stats_1 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips ab=8\n" + "dirreq-v3-reqs ab=8\n" + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_2 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_3 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_4 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n", + *entry_stats_1 = + "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "entry-ips ab=8\n", + *entry_stats_2 = + "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "entry-ips \n"; + tor_addr_t addr; + struct in6_addr in6; + + /* Populate the DB a bit. Add these in order, since we can't do the final + * 'sort' step. These aren't very good IP addresses, but they're perfectly + * fine uint32_t values. */ + (void)arg; + tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET)); + + /* Populate the IPv6 DB equivalently with fake IPs in the same range */ + tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6)); + + /* We should have 4 countries: ??, ab, xy, zz. */ + tt_int_op(4,OP_EQ, geoip_get_n_countries()); + memset(&in6, 0, sizeof(in6)); + + CHECK_COUNTRY("??", 3); + CHECK_COUNTRY("ab", 32); + CHECK_COUNTRY("??", 5); + CHECK_COUNTRY("??", 51); + CHECK_COUNTRY("xy", 150); + CHECK_COUNTRY("xy", 190); + CHECK_COUNTRY("??", 2000); + + tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3)); + SET_TEST_IPV6(3); + tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6)); + + get_options_mutable()->BridgeRelay = 1; + get_options_mutable()->BridgeRecordUsageByCountry = 1; + /* Put 9 observations in AB... */ + for (i=32; i < 40; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + } + SET_TEST_ADDRESS(225); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + /* and 3 observations in XY, several times. */ + for (j=0; j < 10; ++j) + for (i=52; i < 55; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600); + } + /* and 17 observations in ZZ... */ + for (i=110; i < 127; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + } + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); + tt_assert(s); + tt_assert(v); + tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s); + tt_str_op("v4=16,v6=16",OP_EQ, v); + tor_free(s); + tor_free(v); + + /* Now clear out all the AB observations. */ + geoip_remove_old_clients(now-6000); + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); + tt_assert(s); + tt_assert(v); + tt_str_op("zz=24,xy=8",OP_EQ, s); + tt_str_op("v4=16,v6=16",OP_EQ, v); + tor_free(s); + tor_free(v); + + /* Start testing bridge statistics by making sure that we don't output + * bridge stats without initializing them. */ + s = geoip_format_bridge_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Initialize stats and generate the bridge-stats history string out of + * the connecting clients added above. */ + geoip_bridge_stats_init(now); + s = geoip_format_bridge_stats(now + 86400); + tt_assert(s); + tt_str_op(bridge_stats_1,OP_EQ, s); + tor_free(s); + + /* Stop collecting bridge stats and make sure we don't write a history + * string anymore. */ + geoip_bridge_stats_term(); + s = geoip_format_bridge_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Stop being a bridge and start being a directory mirror that gathers + * directory request statistics. */ + geoip_bridge_stats_term(); + get_options_mutable()->BridgeRelay = 0; + get_options_mutable()->BridgeRecordUsageByCountry = 0; + get_options_mutable()->DirReqStatistics = 1; + + /* Start testing dirreq statistics by making sure that we don't collect + * dirreq stats without initializing them. */ + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + s = geoip_format_dirreq_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Initialize stats, note one connecting client, and generate the + * dirreq-stats history string. */ + geoip_dirreq_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_1,OP_EQ, s); + tor_free(s); + + /* Stop collecting stats, add another connecting client, and ensure we + * don't generate a history string. */ + geoip_dirreq_stats_term(); + SET_TEST_ADDRESS(101); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + s = geoip_format_dirreq_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Re-start stats, add a connecting client, reset stats, and make sure + * that we get an all empty history string. */ + geoip_dirreq_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + geoip_reset_dirreq_stats(now); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_2,OP_EQ, s); + tor_free(s); + + /* Note a successful network status response and make sure that it + * appears in the history string. */ + geoip_note_ns_response(GEOIP_SUCCESS); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_3,OP_EQ, s); + tor_free(s); + + /* Start a tunneled directory request. */ + geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_4,OP_EQ, s); + tor_free(s); + + /* Stop collecting directory request statistics and start gathering + * entry stats. */ + geoip_dirreq_stats_term(); + get_options_mutable()->DirReqStatistics = 0; + get_options_mutable()->EntryStatistics = 1; + + /* Start testing entry statistics by making sure that we don't collect + * anything without initializing entry stats. */ + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + s = geoip_format_entry_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Initialize stats, note one connecting client, and generate the + * entry-stats history string. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + s = geoip_format_entry_stats(now + 86400); + tt_str_op(entry_stats_1,OP_EQ, s); + tor_free(s); + + /* Stop collecting stats, add another connecting client, and ensure we + * don't generate a history string. */ + geoip_entry_stats_term(); + SET_TEST_ADDRESS(101); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + s = geoip_format_entry_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Re-start stats, add a connecting client, reset stats, and make sure + * that we get an all empty history string. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + geoip_reset_entry_stats(now); + s = geoip_format_entry_stats(now + 86400); + tt_str_op(entry_stats_2,OP_EQ, s); + tor_free(s); + + /* Test the OOM handler. Add a client, run the OOM. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, + now - (12 * 60 * 60)); + /* We've seen this 12 hours ago. Run the OOM, it should clean the entry + * because it is above the minimum cutoff of 4 hours. */ + size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000); + tt_size_op(bytes_removed, OP_GT, 0); + + /* Do it again but this time with an entry with a lower cutoff. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, + now - (3 * 60 * 60)); + bytes_removed = geoip_client_cache_handle_oom(now, 1000); + tt_size_op(bytes_removed, OP_EQ, 0); + + /* Stop collecting entry statistics. */ + geoip_entry_stats_term(); + get_options_mutable()->EntryStatistics = 0; + + done: + tor_free(s); + tor_free(v); +} + +static void +test_geoip_with_pt(void *arg) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + int i; + tor_addr_t addr; + struct in6_addr in6; + + (void)arg; + get_options_mutable()->BridgeRelay = 1; + get_options_mutable()->BridgeRecordUsageByCountry = 1; + + memset(&in6, 0, sizeof(in6)); + + /* No clients seen yet. */ + s = geoip_get_transport_history(); + tor_assert(!s); + + /* 4 connections without a pluggable transport */ + for (i=0; i < 4; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + } + + /* 9 connections with "alpha" */ + for (i=4; i < 13; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200); + } + + /* one connection with "beta" */ + SET_TEST_ADDRESS(13); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200); + + /* 14 connections with "charlie" */ + for (i=14; i < 28; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200); + } + + /* 131 connections with "ddr" */ + for (i=28; i < 159; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200); + } + + /* 8 connections with "entropy" */ + for (i=159; i < 167; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200); + } + + /* 2 connections from the same IP with two different transports. */ + SET_TEST_ADDRESS(++i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200); + + /* Test the transport history string. */ + s = geoip_get_transport_history(); + tor_assert(s); + tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136," + "entropy=8,fire=8,google=8"); + + /* Stop collecting entry statistics. */ + geoip_entry_stats_term(); + get_options_mutable()->EntryStatistics = 0; + + done: + tor_free(s); +} + +#undef SET_TEST_ADDRESS +#undef SET_TEST_IPV6 +#undef CHECK_COUNTRY + +static const char GEOIP_CONTENT[] = + "134445936,134445939,MP\n" + "134445940,134447103,GU\n" + "134447104,134738943,US\n" + "134738944,134739199,CA\n" + "134739200,135192575,US\n" + "135192576,135200767,MX\n" + "135200768,135430143,US\n" + "135430144,135430399,CA\n" + "135430400,135432191,US\n"; + +static void +test_geoip_load_file(void *arg) +{ + (void)arg; + char *contents = NULL; + char *dhex = NULL; + + /* A nonexistant filename should fail. */ + tt_int_op(-1, OP_EQ, + geoip_load_file(AF_INET, "/you/did/not/put/a/file/here/I/hope")); + + /* We start out with only "Ningunpartia" in the database. */ + tt_int_op(1, OP_EQ, geoip_get_n_countries()); + tt_str_op("??", OP_EQ, geoip_get_country_name(0)); + /* Any lookup attempt should say "-1" because we have no info */ + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304)); + /* There should be no 'digest' for a nonexistant file */ + tt_str_op("0000000000000000000000000000000000000000", OP_EQ, + geoip_db_digest(AF_INET)); + + const char *fname = get_fname("geoip"); + tt_int_op(0, OP_EQ, write_str_to_file(fname, GEOIP_CONTENT, 1)); + + int rv = geoip_load_file(AF_INET, fname); + if (rv != 0) { + TT_GRIPE(("Unable to load geoip from %s", escaped(fname))); + } + tt_int_op(0, OP_EQ, rv); + + /* Check that we loaded some countries; this will fail if there are ever + * fewer than 5 countries in our test above. */ + tt_int_op(geoip_get_n_countries(), OP_GE, 5); + + /* Let's see where 8.8.8.8 is. */ + int country = geoip_get_country_by_ipv4(0x08080808); + tt_int_op(country, OP_GE, 1); /* It shouldn't be 'unknown' or 'nowhere' */ + const char *cc = geoip_get_country_name(country); + tt_int_op(strlen(cc), OP_EQ, 2); + + /* The digest should be set.... */ + tt_str_op("0000000000000000000000000000000000000000", OP_NE, + geoip_db_digest(AF_INET)); + + /* And it should be set correctly */ + contents = read_file_to_str(fname, RFTS_BIN, NULL); + uint8_t d[DIGEST_LEN]; + crypto_digest((char*)d, contents, strlen(contents)); + dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN)); + tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET)); + + /* Make sure geoip_free_all() works. */ + geoip_free_all(); + tt_int_op(1, OP_EQ, geoip_get_n_countries()); + tt_str_op("??", OP_EQ, geoip_get_country_name(0)); + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304)); + tt_str_op("0000000000000000000000000000000000000000", OP_EQ, + geoip_db_digest(AF_INET)); // <--- nick bets this will fail. + + done: + tor_free(contents); + tor_free(dhex); +} + +static void +test_geoip6_load_file(void *arg) +{ + (void)arg; + struct in6_addr iaddr6; + char *contents = NULL; + char *dhex = NULL; + + /* A nonexistant filename should fail. */ + tt_int_op(-1, OP_EQ, + geoip_load_file(AF_INET6, "/you/did/not/put/a/file/here/I/hope")); + + /* Any lookup attempt should say "-1" because we have no info */ + tor_inet_pton(AF_INET6, "2001:4860:4860::8888", &iaddr6); + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6)); + + /* Load geiop6 file */ + const char *fname6 = get_fname("geoip6"); + const char CONTENT[] = + "2001:4830:6010::,2001:4830:601f:ffff:ffff:ffff:ffff:ffff,GB\n" + "2001:4830:6020::,2001:4830:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4838::,2001:4838:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4840::,2001:4840:ffff:ffff:ffff:ffff:ffff:ffff,XY\n" + "2001:4848::,2001:4848:ffff:ffff:ffff:ffff:ffff:ffff,ZD\n" + "2001:4850::,2001:4850:ffff:ffff:ffff:ffff:ffff:ffff,RO\n" + "2001:4858::,2001:4858:ffff:ffff:ffff:ffff:ffff:ffff,TC\n" + "2001:4860::,2001:4860:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4868::,2001:4868:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4870::,2001:4871:ffff:ffff:ffff:ffff:ffff:ffff,NB\n" + "2001:4878::,2001:4878:128:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4878:129::,2001:4878:129:ffff:ffff:ffff:ffff:ffff,CR\n" + "2001:4878:12a::,2001:4878:203:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4878:204::,2001:4878:204:ffff:ffff:ffff:ffff:ffff,DE\n" + "2001:4878:205::,2001:4878:214:ffff:ffff:ffff:ffff:ffff,US\n"; + tt_int_op(0, OP_EQ, write_str_to_file(fname6, CONTENT, 1)); + + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET6, fname6)); + + /* Check that we loaded some countries; this will fail if there are ever + * fewer than 5 countries in our test data above. */ + tt_int_op(geoip_get_n_countries(), OP_GE, 5); + + /* Let's see where 2001:4860:4860::8888 (google dns) is. */ + const char *caddr6 = "2001:4860:4860::8888"; + tor_inet_pton(AF_INET6, caddr6, &iaddr6); + int country6 = geoip_get_country_by_ipv6(&iaddr6); + tt_int_op(country6, OP_GE, 1); + + const char *cc6 = geoip_get_country_name(country6); + tt_int_op(strlen(cc6), OP_EQ, 2); + + /* The digest should be set.... */ + tt_str_op("0000000000000000000000000000000000000000", OP_NE, + geoip_db_digest(AF_INET6)); + + /* And it should be set correctly */ + contents = read_file_to_str(fname6, RFTS_BIN, NULL); + uint8_t d[DIGEST_LEN]; + crypto_digest((char*)d, contents, strlen(contents)); + dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN)); + tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET6)); + + /* Make sure geoip_free_all() works. */ + geoip_free_all(); + tt_int_op(1, OP_EQ, geoip_get_n_countries()); + tt_str_op("??", OP_EQ, geoip_get_country_name(0)); + tor_inet_pton(AF_INET6, "::1:2:3:4", &iaddr6); + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6)); + tt_str_op("0000000000000000000000000000000000000000", OP_EQ, + geoip_db_digest(AF_INET6)); + + done: + tor_free(contents); + tor_free(dhex); +} + +static void +test_geoip_load_2nd_file(void *arg) +{ + (void)arg; + + char *fname_geoip = tor_strdup(get_fname("geoip_data")); + char *fname_empty = tor_strdup(get_fname("geoip_empty")); + + tt_int_op(0, OP_EQ, write_str_to_file(fname_geoip, GEOIP_CONTENT, 1)); + tt_int_op(0, OP_EQ, write_str_to_file(fname_empty, "\n", 1)); + + /* Load 1st geoip file */ + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_geoip)); + + /* Load 2nd geoip (empty) file */ + /* It has to be the same IP address family */ + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_empty)); + + /* Check that there is no geoip information for 8.8.8.8, */ + /* since loading the empty 2nd file should have delete it. */ + int country = geoip_get_country_by_ipv4(0x08080808); + tt_int_op(country, OP_EQ, 0); + + done: + tor_free(fname_geoip); + tor_free(fname_empty); +} + +#define ENT(name) \ + { #name, test_ ## name , 0, NULL, NULL } +#define FORK(name) \ + { #name, test_ ## name , TT_FORK, NULL, NULL } + +struct testcase_t geoip_tests[] = { + { "geoip", test_geoip, TT_FORK, NULL, NULL }, + { "geoip_with_pt", test_geoip_with_pt, TT_FORK, NULL, NULL }, + { "load_file", test_geoip_load_file, TT_FORK, NULL, NULL }, + { "load_file6", test_geoip6_load_file, TT_FORK, NULL, NULL }, + { "load_2nd_file", test_geoip_load_2nd_file, TT_FORK, NULL, NULL }, + + END_OF_TESTCASES +}; + diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 0da9cf64d0..1db5e9064f 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -18,6 +18,7 @@ #include "config.h" #include "confparse.h" #include "connection.h" +#include "crypto_rand.h" #include "main.h" #include "nodelist.h" #include "relay.h" @@ -33,7 +34,6 @@ DISABLE_GCC_WARNING(overlength-strings) * at large. */ #endif #include "test_descriptors.inc" -#include "or.h" #include "circuitlist.h" #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS ENABLE_GCC_WARNING(overlength-strings) @@ -156,7 +156,7 @@ mock_tor_addr_lookup__fail_on_bad_addrs(const char *name, /* Helper for test_conn_get_connection() */ static int -fake_close_socket(evutil_socket_t sock) +fake_close_socket(tor_socket_t sock) { (void)sock; return 0; diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c index aed28d3bd2..8e15184c2a 100644 --- a/src/test/test_hs_cell.c +++ b/src/test/test_hs_cell.c @@ -14,6 +14,7 @@ #include "log_test_helpers.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "hs_cell.h" #include "hs_intropoint.h" #include "hs_service.h" diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 7ee7210bc9..50dca588ed 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -213,12 +213,12 @@ test_e2e_rend_circuit_setup_legacy(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), OP_EQ, DIGEST_SHA1); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), OP_EQ, DIGEST_SHA1); - tt_assert(or_circ->cpath->f_crypto); - tt_assert(or_circ->cpath->b_crypto); + tt_assert(or_circ->cpath->crypto.f_crypto); + tt_assert(or_circ->cpath->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); @@ -283,12 +283,12 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check that the crypt path has prop224 algorithm parameters */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->f_crypto); - tt_assert(or_circ->cpath->b_crypto); + tt_assert(or_circ->cpath->crypto.f_crypto); + tt_assert(or_circ->cpath->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); @@ -397,21 +397,25 @@ test_client_pick_intro(void *arg) } SMARTLIST_FOREACH_END(ip); /* Try to get a random intro: Should return the chosen one! */ - extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); - tor_assert(ip); - tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN)); - tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest, - DIGEST_LEN); + /* (We try several times, to make sure this behavior is consistent, and to + * cover the different cases of client_get_random_intro().) */ + for (int i = 0; i < 64; ++i) { + extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); + tor_assert(ip); + tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN)); + tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest, + DIGEST_LEN); + extend_info_free(ip); + } extend_info_free(chosen_intro_ei); - extend_info_free(ip); /* Now also mark the chosen one as failed: See that we can't get any intro points anymore. */ hs_cache_client_intro_state_note(&service_kp.pubkey, &chosen_intro_point->auth_key_cert->signed_key, INTRO_POINT_FAILURE_TIMEOUT); - ip = client_get_random_intro(&service_kp.pubkey); + extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); tor_assert(!ip); } diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 16803dbd16..7348eb746c 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -17,19 +17,21 @@ #include "hs_test_helpers.h" #include "connection_edge.h" +#include "crypto_rand.h" #include "hs_common.h" #include "hs_client.h" #include "hs_service.h" #include "config.h" #include "networkstatus.h" #include "directory.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "nodelist.h" #include "routerlist.h" #include "statefile.h" #include "circuitlist.h" -#include "shared_random.h" +#include "dirauth/shared_random.h" #include "util.h" +#include "voting_schedule.h" /** Test the validation of HS v3 addresses */ static void @@ -364,11 +366,8 @@ mock_networkstatus_get_live_consensus(time_t now) static void test_responsible_hsdirs(void *arg) { - time_t now = approx_time(); smartlist_t *responsible_dirs = smartlist_new(); networkstatus_t *ns = NULL; - int retval; - (void) arg; hs_init(); @@ -390,12 +389,12 @@ test_responsible_hsdirs(void *arg) helper_add_hsdir_to_networkstatus(ns, 3, "spyro", 0); } - ed25519_keypair_t kp; - retval = ed25519_keypair_generate(&kp, 0); - tt_int_op(retval, OP_EQ , 0); + /* Use a fixed time period and pub key so we always take the same path */ + ed25519_public_key_t pubkey; + uint64_t time_period_num = 17653; // 2 May, 2018, 14:00. + memset(&pubkey, 42, sizeof(pubkey)); - uint64_t time_period_num = hs_get_time_period_num(now); - hs_get_responsible_hsdirs(&kp.pubkey, time_period_num, + hs_get_responsible_hsdirs(&pubkey, time_period_num, 0, 0, responsible_dirs); /* Make sure that we only found 2 responsible HSDirs. @@ -817,7 +816,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 26 Oct 1985 01:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 0); @@ -825,7 +824,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 0); @@ -833,7 +832,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 1); @@ -841,7 +840,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 1); @@ -849,7 +848,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 0); @@ -1336,7 +1335,8 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario) &mock_service_ns->valid_until); set_consensus_times(cfg->service_valid_until, &mock_service_ns->fresh_until); - dirvote_recalculate_timing(get_options(), mock_service_ns->valid_after); + voting_schedule_recalculate_timing(get_options(), + mock_service_ns->valid_after); /* Set client consensus time. */ set_consensus_times(cfg->client_valid_after, &mock_client_ns->valid_after); @@ -1344,7 +1344,8 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario) &mock_client_ns->valid_until); set_consensus_times(cfg->client_valid_until, &mock_client_ns->fresh_until); - dirvote_recalculate_timing(get_options(), mock_client_ns->valid_after); + voting_schedule_recalculate_timing(get_options(), + mock_client_ns->valid_after); /* New time period checks for this scenario. */ tt_int_op(hs_in_period_between_tp_and_srv(mock_service_ns, 0), OP_EQ, @@ -1568,7 +1569,7 @@ helper_set_consensus_and_system_time(networkstatus_t *ns, int position) } else { tt_assert(0); } - dirvote_recalculate_timing(get_options(), ns->valid_after); + voting_schedule_recalculate_timing(get_options(), ns->valid_after); /* Set system time: pretend to be just 2 minutes before consensus expiry */ real_time = ns->valid_until - 120; diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index 207a55de6d..308843e9b8 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -76,9 +76,8 @@ mock_node_get_by_id(const char *digest) { static node_t node; memcpy(node.identity, digest, DIGEST_LEN); - node.hsdir_index = tor_malloc_zero(sizeof(hsdir_index_t)); - memset(node.hsdir_index->fetch, 'C', DIGEST256_LEN); - memset(node.hsdir_index->store_first, 'D', DIGEST256_LEN); + memset(node.hsdir_index.fetch, 'C', DIGEST256_LEN); + memset(node.hsdir_index.store_first, 'D', DIGEST256_LEN); return &node; } diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 8772461f90..14f1a664e7 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -9,6 +9,8 @@ #define HS_DESCRIPTOR_PRIVATE #include "crypto_ed25519.h" +#include "crypto_digest.h" +#include "crypto_rand.h" #include "ed25519_cert.h" #include "or.h" #include "hs_descriptor.h" diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index ec4dcb4705..faa14d9015 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -13,8 +13,7 @@ #include "test.h" #include "log_test_helpers.h" -#include "crypto.h" -#include "log_test_helpers.h" +#include "crypto_rand.h" #include "or.h" #include "circuitlist.h" diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index c1e9f3ced6..7972434d69 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -33,13 +33,12 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" -#include "crypto.h" -#include "dirvote.h" +#include "crypto_rand.h" +#include "dirauth/dirvote.h" #include "networkstatus.h" #include "nodelist.h" #include "relay.h" #include "routerparse.h" - #include "hs_common.h" #include "hs_config.h" #include "hs_ident.h" @@ -51,7 +50,8 @@ #include "main.h" #include "rendservice.h" #include "statefile.h" -#include "shared_random_state.h" +#include "dirauth/shared_random_state.h" +#include "voting_schedule.h" /* Trunnel */ #include "hs/cell_establish_intro.h" @@ -173,12 +173,12 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->f_crypto); - tt_assert(or_circ->cpath->b_crypto); + tt_assert(or_circ->cpath->crypto.f_crypto); + tt_assert(or_circ->cpath->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED); @@ -492,6 +492,8 @@ test_helper_functions(void *arg) MOCK(node_get_by_id, mock_node_get_by_id); hs_service_init(); + time_t now = time(NULL); + update_approx_time(now); service = helper_create_service(); @@ -551,7 +553,6 @@ test_helper_functions(void *arg) /* Testing can_service_launch_intro_circuit() */ { - time_t now = time(NULL); /* Put the start of the retry period back in time, we should be allowed. * to launch intro circuit. */ service->state.num_intro_circ_launched = 2; @@ -575,7 +576,6 @@ test_helper_functions(void *arg) /* Testing intro_point_should_expire(). */ { - time_t now = time(NULL); /* Just some basic test of the current state. */ tt_u64_op(ip->introduce2_max, OP_GE, INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS); @@ -1057,7 +1057,7 @@ test_rotate_descriptors(void *arg) ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC", &mock_ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), mock_ns.valid_after); + voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); /* Create a service with a default descriptor and state. It's added to the * global map. */ @@ -1095,7 +1095,7 @@ test_rotate_descriptors(void *arg) ret = parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC", &mock_ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), mock_ns.valid_after); + voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); /* Note down what to expect for the next rotation time which is 01:00 + 23h * meaning 00:00:00. */ @@ -1157,7 +1157,7 @@ test_build_update_descriptors(void *arg) ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC", &mock_ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), mock_ns.valid_after); + voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); @@ -1237,7 +1237,7 @@ test_build_update_descriptors(void *arg) node->is_running = node->is_valid = node->is_fast = node->is_stable = 1; } - /* We have to set thise, or the lack of microdescriptors for these + /* We have to set this, or the lack of microdescriptors for these * nodes will make them unusable. */ get_options_mutable()->UseMicrodescriptors = 0; diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c new file mode 100644 index 0000000000..9da8a039dd --- /dev/null +++ b/src/test/test_mainloop.c @@ -0,0 +1,142 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_mainloop.c + * \brief Tests for functions closely related to the Tor main loop + */ + +#include "test.h" +#include "log_test_helpers.h" + +#include "or.h" +#include "main.h" + +static const uint64_t BILLION = 1000000000; + +static void +test_mainloop_update_time_normal(void *arg) +{ + (void)arg; + + monotime_enable_test_mocking(); + /* This is arbitrary */ + uint64_t mt_now = U64_LITERAL(7493289274986); + /* This time is in the past as of when this test was written. */ + time_t now = 1525272090; + monotime_coarse_set_mock_time_nsec(mt_now); + reset_uptime(); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 0); + + update_current_time(now); // Same time as before is a no-op. + tt_int_op(get_uptime(), OP_EQ, 0); + + now += 1; + mt_now += BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 1); + + now += 2; // two-second jump is unremarkable. + mt_now += 2*BILLION; + update_current_time(now); + monotime_coarse_set_mock_time_nsec(mt_now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); + + now -= 1; // a one-second hop backwards is also unremarkable. + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); // it changes the approx time... + tt_int_op(get_uptime(), OP_EQ, 3); // but it doesn't roll back our uptime + + done: + monotime_disable_test_mocking(); +} + +static void +test_mainloop_update_time_jumps(void *arg) +{ + (void)arg; + + monotime_enable_test_mocking(); + /* This is arbitrary */ + uint64_t mt_now = U64_LITERAL(7493289274986); + /* This time is in the past as of when this test was written. */ + time_t now = 220897152; + monotime_coarse_set_mock_time_nsec(mt_now); + reset_uptime(); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 0); + + /* Put some uptime on the clock.. */ + now += 3; + mt_now += 3*BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); + + /* Now try jumping forward and backward, without updating the monotonic + * clock. */ + setup_capture_of_logs(LOG_NOTICE); + now += 1800; + update_current_time(now); + expect_single_log_msg_containing( + "Your system clock just jumped 1800 seconds forward"); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change. + mock_clean_saved_logs(); + + now -= 600; + update_current_time(now); + expect_single_log_msg_containing( + "Your system clock just jumped 600 seconds backward"); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change. + mock_clean_saved_logs(); + + /* uptime tracking should go normally now if the clock moves sensibly. */ + now += 2; + mt_now += 2*BILLION; + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 5); + + /* If we skip forward by a few minutes but the monotonic clock agrees, + * we've just been idle: that counts as not worth warning about. */ + now += 1800; + mt_now += 1800*BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + expect_no_log_entry(); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 5); // this doesn't count to uptime, though. + + /* If we skip forward by a long time, even if the clock agrees, it's + * idnless that counts. */ + now += 4000; + mt_now += 4000*BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + expect_single_log_msg_containing("Tor has been idle for 4000 seconds"); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 5); + + done: + teardown_capture_of_logs(); + monotime_disable_test_mocking(); +} + +#define MAINLOOP_TEST(name) \ + { #name, test_mainloop_## name , TT_FORK, NULL, NULL } + +struct testcase_t mainloop_tests[] = { + MAINLOOP_TEST(update_time_normal), + MAINLOOP_TEST(update_time_jumps), + END_OF_TESTCASES +}; + diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 59b28f7580..4b168f49ed 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -5,7 +5,8 @@ #include "or.h" #include "config.h" -#include "dirvote.h" +#define DIRVOTE_PRIVATE +#include "dirauth/dirvote.h" #include "microdesc.h" #include "networkstatus.h" #include "routerlist.h" @@ -385,25 +386,6 @@ static const char test_ri2[] = "cf34GXHv61XReJF3AlzNHFpbrPOYmowmhrTULKyMqow=\n" "-----END SIGNATURE-----\n"; -static const char test_md_8[] = - "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" - "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" - "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" - "-----END RSA PUBLIC KEY-----\n" - "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"; - -static const char test_md_16[] = - "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" - "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" - "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" - "-----END RSA PUBLIC KEY-----\n" - "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n" - "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"; - static const char test_md_18[] = "onion-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" @@ -415,16 +397,6 @@ static const char test_md_18[] = "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n" "id rsa1024 Cd47okjCHD83YGzThGBDptXs9Z4\n"; -static const char test_md2_18[] = - "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" - "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" - "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" - "-----END RSA PUBLIC KEY-----\n" - "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" - "id rsa1024 t+J/EEITw28T5+mCkYKEXklZl6A\n"; - static const char test_md2_21[] = "onion-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" @@ -444,17 +416,6 @@ test_md_generate(void *arg) ri = router_parse_entry_from_string(test_ri, NULL, 0, 0, NULL, NULL); tt_assert(ri); - md = dirvote_create_microdescriptor(ri, 8); - tt_str_op(md->body, OP_EQ, test_md_8); - - /* XXXX test family lines. */ - /* XXXX test method 14 for A lines. */ - /* XXXX test method 15 for P6 lines. */ - - microdesc_free(md); - md = NULL; - md = dirvote_create_microdescriptor(ri, 16); - tt_str_op(md->body, OP_EQ, test_md_16); microdesc_free(md); md = NULL; @@ -471,11 +432,6 @@ test_md_generate(void *arg) microdesc_free(md); md = NULL; - md = dirvote_create_microdescriptor(ri, 18); - tt_str_op(md->body, OP_EQ, test_md2_18); - - microdesc_free(md); - md = NULL; md = dirvote_create_microdescriptor(ri, 21); tt_str_op(md->body, OP_EQ, test_md2_21); tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 094e934712..7d8e57543c 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -7,6 +7,7 @@ **/ #include "or.h" +#include "crypto_rand.h" #include "networkstatus.h" #include "nodelist.h" #include "torcert.h" diff --git a/src/test/test_oom.c b/src/test/test_oom.c index c172fe60c7..abf8896452 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -13,6 +13,7 @@ #include "compat_libevent.h" #include "connection.h" #include "config.h" +#include "crypto_rand.h" #include "relay.h" #include "test.h" #include "test_helpers.h" diff --git a/src/test/test_options.c b/src/test/test_options.c index eaf5034397..65564f324c 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -2067,12 +2067,12 @@ test_options_validate__testing(void *ignored) ENSURE_DEFAULT(TestingV3AuthVotingStartOffset, 3000); ENSURE_DEFAULT(TestingAuthDirTimeToLearnReachability, 3000); ENSURE_DEFAULT(TestingEstimatedDescriptorPropagationTime, 3000); - ENSURE_DEFAULT(TestingServerDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingClientDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingServerConsensusDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingClientConsensusDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingBridgeDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingBridgeBootstrapDownloadSchedule, 3000); + ENSURE_DEFAULT(TestingServerDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingClientDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingServerConsensusDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingClientConsensusDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingBridgeDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay, 3000); ENSURE_DEFAULT(TestingClientMaxIntervalWithoutRequest, 3000); ENSURE_DEFAULT(TestingDirConnectionMaxStall, 3000); ENSURE_DEFAULT(TestingAuthKeyLifetime, 3000); @@ -2422,37 +2422,6 @@ test_options_validate__circuits(void *ignored) } static void -test_options_validate__port_forwarding(void *ignored) -{ - (void)ignored; - int ret; - char *msg; - options_test_data_t *tdata = NULL; - - free_options_test_data(tdata); - tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES - "PortForwarding 1\nSandbox 1\n"); - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, -1); - tt_str_op(msg, OP_EQ, "PortForwarding is not compatible with Sandbox;" - " at most one can be set"); - tor_free(msg); - - free_options_test_data(tdata); - tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES - "PortForwarding 1\nSandbox 0\n"); - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, 0); - tt_assert(!msg); - tor_free(msg); - - done: - free_options_test_data(tdata); - policies_free_all(); - tor_free(msg); -} - -static void test_options_validate__tor2web(void *ignored) { (void)ignored; @@ -4135,16 +4104,6 @@ test_options_validate__testing_options(void *ignored) free_options_test_data(tdata); tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES "TestingEnableTbEmptyEvent 1\n" - ); - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, -1); - tt_str_op(msg, OP_EQ, "TestingEnableTbEmptyEvent may only be changed " - "in testing Tor networks!"); - tor_free(msg); - - free_options_test_data(tdata); - tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES - "TestingEnableTbEmptyEvent 1\n" VALID_DIR_AUTH "TestingTorNetwork 1\n" "___UsingTestNetworkDefaults 0\n" @@ -4261,7 +4220,6 @@ struct testcase_t options_tests[] = { LOCAL_VALIDATE_TEST(path_bias), LOCAL_VALIDATE_TEST(bandwidth), LOCAL_VALIDATE_TEST(circuits), - LOCAL_VALIDATE_TEST(port_forwarding), LOCAL_VALIDATE_TEST(tor2web), LOCAL_VALIDATE_TEST(rend), LOCAL_VALIDATE_TEST(single_onion), diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c new file mode 100644 index 0000000000..f159c4f83a --- /dev/null +++ b/src/test/test_periodic_event.c @@ -0,0 +1,329 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_periodic_event.c + * \brief Test the periodic events that Tor uses for different roles. They are + * part of the libevent mainloop + */ + +#define CONFIG_PRIVATE +#define HS_SERVICE_PRIVATE +#define MAIN_PRIVATE + +#include "test.h" +#include "test_helpers.h" + +#include "or.h" +#include "config.h" +#include "hibernate.h" +#include "hs_service.h" +#include "main.h" +#include "periodic.h" + +/** Helper function: This is replaced in some tests for the event callbacks so + * we don't actually go into the code path of those callbacks. */ +static int +dumb_event_fn(time_t now, const or_options_t *options) +{ + (void) now; + (void) options; + + /* Will get rescheduled in 300 seconds. It just can't be 0. */ + return 300; +} + +static void +register_dummy_hidden_service(hs_service_t *service) +{ + memset(service, 0, sizeof(hs_service_t)); + memset(&service->keys.identity_pk, 'A', sizeof(service->keys.identity_pk)); + (void) register_service(get_hs_service_map(), service); +} + +static void +test_pe_initialize(void *arg) +{ + (void) arg; + + /* Initialize the events but the callback won't get called since we would + * need to run the main loop and then wait for a second delaying the unit + * tests. Instead, we'll test the callback work indepedently elsewhere. */ + initialize_periodic_events(); + + /* Validate that all events have been set up. */ + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_assert(item->ev); + tt_assert(item->fn); + tt_u64_op(item->last_action_time, OP_EQ, 0); + /* Every event must have role(s) assign to it. This is done statically. */ + tt_u64_op(item->roles, OP_NE, 0); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + + done: + teardown_periodic_events(); +} + +static void +test_pe_launch(void *arg) +{ + hs_service_t service, *to_remove = NULL; + or_options_t *options; + + (void) arg; + + hs_init(); + /* We need to put tor in hibernation live state so the events requiring + * network gets enabled. */ + consider_hibernation(time(NULL)); + + /* Hack: We'll set a dumb fn() of each events so they don't get called when + * dispatching them. We just want to test the state of the callbacks, not + * the whole code path. */ + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + item->fn = dumb_event_fn; + } + + /* Lets make sure that before intialization, we can't scan the periodic + * events list and launch them. Lets try by being a Client. */ + options = get_options_mutable(); + options->SocksPort_set = 1; + periodic_events_on_new_options(options); + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + + initialize_periodic_events(); + + /* Now that we've initialized, rescan the list to launch. */ + periodic_events_on_new_options(options); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } else { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + // enabled or not, the event has not yet been run. + tt_u64_op(item->last_action_time, OP_EQ, 0); + } + + /* Remove Client but become a Relay. */ + options->SocksPort_set = 0; + options->ORPort_set = 1; + periodic_events_on_new_options(options); + + unsigned roles = get_my_roles(options); + tt_uint_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + /* Only Client role should be disabled. */ + if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + if (item->roles & PERIODIC_EVENT_ROLE_RELAY) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } + /* Non Relay role should be disabled, except for Dirserver. */ + if (!(item->roles & roles)) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + } + + /* Disable everything and we'll enable them ALL. */ + options->SocksPort_set = 0; + options->ORPort_set = 0; + periodic_events_on_new_options(options); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + + /* Enable everything. */ + options->SocksPort_set = 1; options->ORPort_set = 1; + options->BridgeRelay = 1; options->AuthoritativeDir = 1; + options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + register_dummy_hidden_service(&service); + periodic_events_on_new_options(options); + /* Note down the reference because we need to remove this service from the + * global list before the hs_free_all() call so it doesn't try to free + * memory on the stack. Furthermore, we can't remove it now else it will + * trigger a rescan of the event disabling the HS service event. */ + to_remove = &service; + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } + + done: + if (to_remove) { + remove_service(get_hs_service_map(), to_remove); + } + hs_free_all(); +} + +static void +test_pe_get_roles(void *arg) +{ + int roles; + + (void) arg; + + /* Just so the HS global map exists. */ + hs_init(); + + or_options_t *options = get_options_mutable(); + tt_assert(options); + + /* Nothing configured, should be no roles. */ + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, 0); + + /* Indicate we have a SocksPort, roles should be come Client. */ + options->SocksPort_set = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + + /* Now, we'll add a ORPort so should now be a Relay + Client. */ + options->ORPort_set = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | + PERIODIC_EVENT_ROLE_DIRSERVER)); + + /* Now add a Bridge. */ + options->BridgeRelay = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); + /* Unset client so we can solely test Router role. */ + options->SocksPort_set = 0; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + + /* Reset options so we can test authorities. */ + options->SocksPort_set = 0; + options->ORPort_set = 0; + options->BridgeRelay = 0; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, 0); + + /* Now upgrade to Dirauth. */ + options->DirPort_set = 1; + options->AuthoritativeDir = 1; + options->V3AuthoritativeDir = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + /* Now Bridge Authority. */ + options->V3AuthoritativeDir = 0; + options->BridgeAuthoritativeDir = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + /* Move that bridge auth to become a relay. */ + options->ORPort_set = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY + | PERIODIC_EVENT_ROLE_DIRSERVER)); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + /* And now an Hidden service. */ + hs_service_t service; + register_dummy_hidden_service(&service); + roles = get_my_roles(options); + /* Remove it now so the hs_free_all() doesn't try to free stack memory. */ + remove_service(get_hs_service_map(), &service); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + done: + hs_free_all(); +} + +static void +test_pe_hs_service(void *arg) +{ + hs_service_t service, *to_remove = NULL; + + (void) arg; + + hs_init(); + /* We need to put tor in hibernation live state so the events requiring + * network gets enabled. */ + consider_hibernation(time(NULL)); + /* Initialize the events so we can enable them */ + initialize_periodic_events(); + + /* Hack: We'll set a dumb fn() of each events so they don't get called when + * dispatching them. We just want to test the state of the callbacks, not + * the whole code path. */ + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + item->fn = dumb_event_fn; + } + + /* This should trigger a rescan of the list and enable the HS service + * events. */ + register_dummy_hidden_service(&service); + /* Note down the reference because we need to remove this service from the + * global list before the hs_free_all() call so it doesn't try to free + * memory on the stack. Furthermore, we can't remove it now else it will + * trigger a rescan of the event disabling the HS service event. */ + to_remove = &service; + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } + } + to_remove = NULL; + + /* Remove the service from the global map, it should trigger a rescan and + * disable the HS service events. */ + remove_service(get_hs_service_map(), &service); + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + } + + done: + if (to_remove) { + remove_service(get_hs_service_map(), to_remove); + } + hs_free_all(); +} + +#define PE_TEST(name) \ + { #name, test_pe_## name , TT_FORK, NULL, NULL } + +struct testcase_t periodic_event_tests[] = { + PE_TEST(initialize), + PE_TEST(launch), + PE_TEST(get_roles), + PE_TEST(hs_service), + + END_OF_TESTCASES +}; diff --git a/src/test/test_policy.c b/src/test/test_policy.c index f8aa8ac40b..e89d49aaf5 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -1496,9 +1496,21 @@ test_dump_exit_policy_to_string(void *arg) } static routerinfo_t *mock_desc_routerinfo = NULL; +static int routerinfo_err; + static const routerinfo_t * -mock_router_get_my_routerinfo(void) +mock_router_get_my_routerinfo_with_err(int *err) { + if (routerinfo_err) { + if (err) + *err = routerinfo_err; + + return NULL; + } + + if (err) + *err = 0; + return mock_desc_routerinfo; } @@ -1541,7 +1553,8 @@ test_policies_getinfo_helper_policies(void *arg) tor_free(answer); memset(&mock_my_routerinfo, 0, sizeof(routerinfo_t)); - MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo); + MOCK(router_get_my_routerinfo_with_err, + mock_router_get_my_routerinfo_with_err); mock_my_routerinfo.exit_policy = smartlist_new(); mock_desc_routerinfo = &mock_my_routerinfo; @@ -1658,6 +1671,55 @@ test_policies_getinfo_helper_policies(void *arg) tt_assert(strlen(answer) == ipv4_len + ipv6_len + 1); tor_free(answer); + routerinfo_err = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "No known exit address yet"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_PARSE; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Cannot parse descriptor"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, 0); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Not running in server mode"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_DIGEST_FAILED; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + + tt_int_op(rv, OP_EQ, 0); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Key digest failed"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_GENERATE; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Cannot generate descriptor"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Descriptor still rebuilding - not ready yet"); + done: tor_free(answer); UNMOCK(get_options); @@ -1923,11 +1985,8 @@ test_policies_fascist_firewall_allows_address(void *arg) tor_addr_port_t chosen_rs_ap; \ tor_addr_make_null(&chosen_rs_ap.addr, AF_INET); \ chosen_rs_ap.port = 0; \ - tt_int_op(fascist_firewall_choose_address_rs(&(fake_rs), \ - (fw_connection), \ - (pref_only), \ - &chosen_rs_ap), \ - OP_EQ, (expect_rv)); \ + fascist_firewall_choose_address_rs(&(fake_rs), (fw_connection), \ + (pref_only), &chosen_rs_ap); \ tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_rs_ap.addr)); \ tt_int_op((expect_ap).port, OP_EQ, chosen_rs_ap.port); \ STMT_END @@ -1940,11 +1999,8 @@ test_policies_fascist_firewall_allows_address(void *arg) tor_addr_port_t chosen_node_ap; \ tor_addr_make_null(&chosen_node_ap.addr, AF_INET); \ chosen_node_ap.port = 0; \ - tt_int_op(fascist_firewall_choose_address_node(&(fake_node), \ - (fw_connection), \ - (pref_only), \ - &chosen_node_ap), \ - OP_EQ, (expect_rv)); \ + fascist_firewall_choose_address_node(&(fake_node),(fw_connection), \ + (pref_only), &chosen_node_ap); \ tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_node_ap.addr)); \ tt_int_op((expect_ap).port, OP_EQ, chosen_node_ap.port); \ STMT_END diff --git a/src/test/test_protover.c b/src/test/test_protover.c index bdfb2d13cd..06fd575631 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -229,8 +229,8 @@ test_protover_vote(void *arg) /* Protocol name too long */ smartlist_clear(lst); smartlist_add(lst, (void*) "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); result = protover_compute_vote(lst, 1); tt_str_op(result, OP_EQ, ""); tor_free(result); @@ -321,10 +321,10 @@ test_protover_all_supported(void *arg) #ifndef HAVE_RUST // XXXXXX ????? tor_capture_bugs_(1); tt_assert(protover_all_supported( - "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaa=1-65536", &msg)); + "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaa=1-65536", &msg)); tor_end_capture_bugs_(); #endif diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index eea1f5dc80..2fc0288f69 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -4,13 +4,28 @@ /* Unit tests for handling different kinds of relay cell */ #define RELAY_PRIVATE +#define CIRCUITLIST_PRIVATE +#define CONNECTION_EDGE_PRIVATE +#define CONNECTION_PRIVATE + #include "or.h" +#include "main.h" #include "config.h" #include "connection.h" +#include "crypto.h" +#include "crypto_rand.h" +#include "circuitbuild.h" +#include "circuitlist.h" #include "connection_edge.h" +#include "log_test_helpers.h" #include "relay.h" #include "test.h" +#include "log_test_helpers.h" + +#include "circpathbias.h" +#include "connection_edge.h" + static int srm_ncalls; static entry_connection_t *srm_conn; static int srm_atype; @@ -60,6 +75,805 @@ mark_unattached_mock(entry_connection_t *conn, int endreason, (void) file; } +/* Helper: Return a newly allocated and initialized origin circuit with + * purpose and flags. A default HS identifier is set to an ed25519 + * authentication key for introduction point. */ +static origin_circuit_t * +helper_create_origin_circuit(int purpose, int flags) +{ + origin_circuit_t *circ = NULL; + + circ = origin_circuit_init(purpose, flags); + tor_assert(circ); + circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); + circ->cpath->magic = CRYPT_PATH_MAGIC; + circ->cpath->state = CPATH_STATE_OPEN; + circ->cpath->package_window = circuit_initial_package_window(); + circ->cpath->deliver_window = CIRCWINDOW_START; + circ->cpath->prev = circ->cpath; + /* Create a default HS identifier. */ + circ->hs_ident = tor_malloc_zero(sizeof(hs_ident_circuit_t)); + + return circ; +} + +static void +mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, + int line, const char *file) +{ + (void) line; + (void) file; + conn->edge_.end_reason = endreason; +} + +static void +mock_mark_circ_for_close(circuit_t *circ, int reason, int line, + const char *file) +{ + (void)reason; (void)line; (void)file; + + circ->marked_for_close = 1; + return; +} + +static void +mock_mark_for_close(connection_t *conn, + int line, const char *file) +{ + (void)line; + (void)file; + + conn->marked_for_close = 1; + return; +} + +static void +mock_start_reading(connection_t *conn) +{ + (void)conn; + return; +} + +static int +mock_send_command(streamid_t stream_id, circuit_t *circ, + uint8_t relay_command, const char *payload, + size_t payload_len, crypt_path_t *cpath_layer, + const char *filename, int lineno) +{ + (void)stream_id; (void)circ; + (void)relay_command; (void)payload; + (void)payload_len; (void)cpath_layer; + (void)filename; (void)lineno; + + return 0; +} + +static entry_connection_t * +fake_entry_conn(origin_circuit_t *oncirc, streamid_t id) +{ + edge_connection_t *edgeconn; + entry_connection_t *entryconn; + + entryconn = entry_connection_new(CONN_TYPE_AP, AF_INET); + edgeconn = ENTRY_TO_EDGE_CONN(entryconn); + edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT; + edgeconn->deliver_window = STREAMWINDOW_START; + edgeconn->package_window = STREAMWINDOW_START; + + edgeconn->stream_id = id; + edgeconn->on_circuit = TO_CIRCUIT(oncirc); + edgeconn->cpath_layer = oncirc->cpath; + + return entryconn; +} + +#define PACK_CELL(id, cmd, body_s) do { \ + memset(&cell, 0, sizeof(cell)); \ + memset(&rh, 0, sizeof(rh)); \ + memcpy(cell.payload+RELAY_HEADER_SIZE, (body_s), sizeof((body_s))-1); \ + rh.length = sizeof((body_s))-1; \ + rh.command = (cmd); \ + rh.stream_id = (id); \ + relay_header_pack((uint8_t*)&cell.payload, &rh); \ + } while (0) +#define ASSERT_COUNTED_BW() do { \ + tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered+rh.length); \ + tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ, \ + overhead+RELAY_PAYLOAD_SIZE-rh.length); \ + delivered = circ->n_delivered_read_circ_bw; \ + overhead = circ->n_overhead_read_circ_bw; \ + } while (0) +#define ASSERT_UNCOUNTED_BW() do { \ + tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered); \ + tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ, overhead); \ + } while (0) + +static int +subtest_circbw_halfclosed(origin_circuit_t *circ, streamid_t init_id) +{ + cell_t cell; + relay_header_t rh; + edge_connection_t *edgeconn; + entry_connection_t *entryconn2=NULL; + entry_connection_t *entryconn3=NULL; + entry_connection_t *entryconn4=NULL; + int delivered = circ->n_delivered_read_circ_bw; + int overhead = circ->n_overhead_read_circ_bw; + + /* Make new entryconns */ + entryconn2 = fake_entry_conn(circ, init_id); + entryconn2->socks_request->has_finished = 1; + entryconn3 = fake_entry_conn(circ, init_id+1); + entryconn3->socks_request->has_finished = 1; + entryconn4 = fake_entry_conn(circ, init_id+2); + entryconn4->socks_request->has_finished = 1; + edgeconn = ENTRY_TO_EDGE_CONN(entryconn2); + edgeconn->package_window = 23; + edgeconn->base_.state = AP_CONN_STATE_OPEN; + + int data_cells = edgeconn->deliver_window; + int sendme_cells = (STREAMWINDOW_START-edgeconn->package_window) + /STREAMWINDOW_INCREMENT; + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + connection_edge_reached_eof(edgeconn); + + /* Data cell not in the half-opened list */ + PACK_CELL(4000, RELAY_COMMAND_DATA, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme cell not in the half-opened list */ + PACK_CELL(4000, RELAY_COMMAND_SENDME, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Connected cell not in the half-opened list */ + PACK_CELL(4000, RELAY_COMMAND_CONNECTED, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Resolved cell not in the half-opened list */ + PACK_CELL(4000, RELAY_COMMAND_RESOLVED, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Connected cell: not counted -- we were open */ + edgeconn = ENTRY_TO_EDGE_CONN(entryconn2); + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* DATA cells up to limit */ + while (data_cells > 0) { + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + data_cells--; + } + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* SENDME cells up to limit */ + while (sendme_cells > 0) { + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + sendme_cells--; + } + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Only one END cell */ + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + + ENTRY_TO_CONN(entryconn2)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn2)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + edgeconn = ENTRY_TO_EDGE_CONN(entryconn3); + edgeconn->base_.state = AP_CONN_STATE_OPEN; + ENTRY_TO_CONN(entryconn3)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0; + /* sendme cell on open entryconn with full window */ + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234"); + int ret = + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + tt_int_op(ret, OP_EQ, -END_CIRC_REASON_TORPROTOCOL); + ASSERT_UNCOUNTED_BW(); + + /* connected cell on a after EOF */ + ENTRY_TO_CONN(entryconn3)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0; + edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT; + connection_edge_reached_eof(edgeconn); + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + + ENTRY_TO_CONN(entryconn3)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_CONNECTED, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* DATA and SENDME after END cell */ + ENTRY_TO_CONN(entryconn3)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + + ENTRY_TO_CONN(entryconn3)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_SENDME, "Data1234"); + ret = + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + tt_int_op(ret, OP_NE, -END_CIRC_REASON_TORPROTOCOL); + ASSERT_UNCOUNTED_BW(); + + ENTRY_TO_CONN(entryconn3)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn3)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Resolved: 1 counted, more not */ + edgeconn = ENTRY_TO_EDGE_CONN(entryconn4); + entryconn4->socks_request->command = SOCKS_COMMAND_RESOLVE; + edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT; + edgeconn->on_circuit = TO_CIRCUIT(circ); + ENTRY_TO_CONN(entryconn4)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0; + connection_edge_reached_eof(edgeconn); + + ENTRY_TO_CONN(entryconn4)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_RESOLVED, + "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + + ENTRY_TO_CONN(entryconn4)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_RESOLVED, + "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Data not counted after resolved */ + ENTRY_TO_CONN(entryconn4)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_DATA, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* End not counted after resolved */ + ENTRY_TO_CONN(entryconn4)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn4)->outbuf_flushlen = 0; + PACK_CELL(edgeconn->stream_id, RELAY_COMMAND_END, "Data1234"); + if (circ->base_.purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + else + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + connection_free_minimal(ENTRY_TO_CONN(entryconn2)); + connection_free_minimal(ENTRY_TO_CONN(entryconn3)); + connection_free_minimal(ENTRY_TO_CONN(entryconn4)); + return 1; + done: + connection_free_minimal(ENTRY_TO_CONN(entryconn2)); + connection_free_minimal(ENTRY_TO_CONN(entryconn3)); + connection_free_minimal(ENTRY_TO_CONN(entryconn4)); + return 0; +} + +static int +halfstream_insert(origin_circuit_t *circ, edge_connection_t *edgeconn, + streamid_t *streams, int num, int random) +{ + int inserted = 0; + + /* Insert num random elements */ + while (inserted < num) { + streamid_t id; + + if (random) + id = (streamid_t)crypto_rand_int(65535)+1; + else + id = get_unique_stream_id_by_circ(circ); + + edgeconn->stream_id = id; + + /* Ensure it isn't there */ + if (connection_half_edge_find_stream_id(circ->half_streams, id)) { + continue; + } + + connection_half_edge_add(edgeconn, circ); + if (streams) + streams[inserted] = id; + inserted++; + } + + return inserted; +} + +static void +subtest_halfstream_insertremove(int num) +{ + origin_circuit_t *circ = + helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0); + edge_connection_t *edgeconn; + entry_connection_t *entryconn; + streamid_t *streams = tor_malloc_zero(num*sizeof(streamid_t)); + int i = 0; + + circ->cpath->state = CPATH_STATE_AWAITING_KEYS; + circ->cpath->deliver_window = CIRCWINDOW_START; + + entryconn = fake_entry_conn(circ, 23); + edgeconn = ENTRY_TO_EDGE_CONN(entryconn); + + /* Explicity test all operations on an absent stream list */ + tt_int_op(connection_half_edge_is_valid_data(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + 23), OP_EQ, 0); + + /* Insert a duplicate element; verify that other elements absent; + * ensure removing it once works */ + edgeconn->stream_id = 23; + connection_half_edge_add(edgeconn, circ); + connection_half_edge_add(edgeconn, circ); + connection_half_edge_add(edgeconn, circ); + + /* Verify that other elements absent */ + tt_int_op(connection_half_edge_is_valid_data(circ->half_streams, + 22), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams, + 22), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams, + 22), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams, + 22), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + 22), OP_EQ, 0); + + tt_int_op(connection_half_edge_is_valid_data(circ->half_streams, + 24), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams, + 24), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams, + 24), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams, + 24), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + 24), OP_EQ, 0); + + /* Verify we only remove it once */ + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + 23), OP_EQ, 1); + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + 23), OP_EQ, 0); + + halfstream_insert(circ, edgeconn, streams, num, 1); + + /* Remove half of them */ + for (i = 0; i < num/2; i++) { + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + streams[i]), + OP_EQ, 1); + } + + /* Verify first half of list is gone */ + for (i = 0; i < num/2; i++) { + tt_ptr_op(connection_half_edge_find_stream_id(circ->half_streams, + streams[i]), + OP_EQ, NULL); + } + + /* Verify second half of list is present */ + for (; i < num; i++) { + tt_ptr_op(connection_half_edge_find_stream_id(circ->half_streams, + streams[i]), + OP_NE, NULL); + } + + /* Remove other half. Verify list is empty. */ + for (i = num/2; i < num; i++) { + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + streams[i]), + OP_EQ, 1); + } + tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 0); + + /* Explicity test all operations on an empty stream list */ + tt_int_op(connection_half_edge_is_valid_data(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_connected(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_sendme(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_resolved(circ->half_streams, + 23), OP_EQ, 0); + tt_int_op(connection_half_edge_is_valid_end(circ->half_streams, + 23), OP_EQ, 0); + + /* For valgrind, leave some around then free the circ */ + halfstream_insert(circ, edgeconn, NULL, 10, 0); + + done: + tor_free(streams); + circuit_free_(TO_CIRCUIT(circ)); + connection_free_minimal(ENTRY_TO_CONN(entryconn)); +} + +static void +test_halfstream_insertremove(void *arg) +{ + (void)arg; + + /* Suppress the WARN message we generate in this test */ + setup_full_capture_of_logs(LOG_WARN); + + /* Test insertion and removal with a few different sizes */ + subtest_halfstream_insertremove(10); + subtest_halfstream_insertremove(100); + subtest_halfstream_insertremove(1000); +} + +static void +test_halfstream_wrap(void *arg) +{ + origin_circuit_t *circ = + helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0); + edge_connection_t *edgeconn; + entry_connection_t *entryconn; + + circ->cpath->state = CPATH_STATE_AWAITING_KEYS; + circ->cpath->deliver_window = CIRCWINDOW_START; + + entryconn = fake_entry_conn(circ, 23); + edgeconn = ENTRY_TO_EDGE_CONN(entryconn); + + (void)arg; + + /* Suppress the WARN message we generate in this test */ + setup_full_capture_of_logs(LOG_WARN); + MOCK(connection_mark_for_close_internal_, mock_mark_for_close); + + /* Verify that get_unique_stream_id_by_circ() can wrap uint16_t */ + circ->next_stream_id = 65530; + halfstream_insert(circ, edgeconn, NULL, 7, 0); + tt_int_op(circ->next_stream_id, OP_EQ, 2); + tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 7); + + /* Insert full-1 */ + halfstream_insert(circ, edgeconn, NULL, + 65534-smartlist_len(circ->half_streams), 0); + tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65534); + + /* Verify that we can get_unique_stream_id_by_circ() successfully */ + edgeconn->stream_id = get_unique_stream_id_by_circ(circ); + tt_int_op(edgeconn->stream_id, OP_NE, 0); /* 0 is failure */ + + /* Insert an opened stream on the circ with that id */ + ENTRY_TO_CONN(entryconn)->marked_for_close = 0; + ENTRY_TO_CONN(entryconn)->outbuf_flushlen = 0; + edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT; + circ->p_streams = edgeconn; + + /* Verify that get_unique_stream_id_by_circ() fails */ + tt_int_op(get_unique_stream_id_by_circ(circ), OP_EQ, 0); /* 0 is failure */ + + /* eof the one opened stream. Verify it is now in half-closed */ + tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65534); + connection_edge_reached_eof(edgeconn); + tt_int_op(smartlist_len(circ->half_streams), OP_EQ, 65535); + + /* Verify get_unique_stream_id_by_circ() fails due to full half-closed */ + circ->p_streams = NULL; + tt_int_op(get_unique_stream_id_by_circ(circ), OP_EQ, 0); /* 0 is failure */ + + done: + circuit_free_(TO_CIRCUIT(circ)); + connection_free_minimal(ENTRY_TO_CONN(entryconn)); + UNMOCK(connection_mark_for_close_internal_); +} + +static void +test_circbw_relay(void *arg) +{ + cell_t cell; + relay_header_t rh; + tor_addr_t addr; + edge_connection_t *edgeconn; + entry_connection_t *entryconn1=NULL; + origin_circuit_t *circ; + int delivered = 0; + int overhead = 0; + + (void)arg; + + MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_); + MOCK(connection_start_reading, mock_start_reading); + MOCK(connection_mark_for_close_internal_, mock_mark_for_close); + MOCK(relay_send_command_from_edge_, mock_send_command); + MOCK(circuit_mark_for_close_, mock_mark_circ_for_close); + + circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0); + circ->cpath->state = CPATH_STATE_AWAITING_KEYS; + circ->cpath->deliver_window = CIRCWINDOW_START; + + entryconn1 = fake_entry_conn(circ, 1); + edgeconn = ENTRY_TO_EDGE_CONN(entryconn1); + + /* Stream id 0: Not counted */ + PACK_CELL(0, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Stream id 1: Counted */ + PACK_CELL(1, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Properly formatted connect cell: counted */ + PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234"); + tor_addr_parse(&addr, "30.40.50.60"); + rh.length = connected_cell_format_payload(cell.payload+RELAY_HEADER_SIZE, + &addr, 1024); + relay_header_pack((uint8_t*)&cell.payload, &rh); \ + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Properly formatted resolved cell in correct state: counted */ + edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT; + entryconn1->socks_request->command = SOCKS_COMMAND_RESOLVE; + edgeconn->on_circuit = TO_CIRCUIT(circ); + PACK_CELL(1, RELAY_COMMAND_RESOLVED, + "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + edgeconn->base_.state = AP_CONN_STATE_OPEN; + entryconn1->socks_request->has_finished = 1; + + /* Connected cell after open: not counted */ + PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Resolved cell after open: not counted */ + PACK_CELL(1, RELAY_COMMAND_RESOLVED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Drop cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_DROP, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Data cell on stream 0: not counted */ + PACK_CELL(0, RELAY_COMMAND_DATA, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Data cell on open connection: counted */ + ENTRY_TO_CONN(entryconn1)->marked_for_close = 0; + PACK_CELL(1, RELAY_COMMAND_DATA, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Empty Data cell on open connection: not counted */ + ENTRY_TO_CONN(entryconn1)->marked_for_close = 0; + PACK_CELL(1, RELAY_COMMAND_DATA, ""); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on valid stream: counted */ + edgeconn->package_window -= STREAMWINDOW_INCREMENT; + ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0; + PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Sendme on valid stream with full window: not counted */ + ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0; + PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234"); + edgeconn->package_window = STREAMWINDOW_START; + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on unknown stream: not counted */ + ENTRY_TO_CONN(entryconn1)->outbuf_flushlen = 0; + PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on circuit with full window: not counted */ + PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on circuit with non-full window: counted */ + PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234"); + circ->cpath->package_window = 900; + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Invalid extended cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_EXTENDED2, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Invalid extended cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_EXTENDED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Invalid HS cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_ESTABLISH_INTRO, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* "Valid" HS cell in expected state: counted */ + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND; + PACK_CELL(1, RELAY_COMMAND_RENDEZVOUS_ESTABLISHED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* End cell on non-closed connection: counted */ + PACK_CELL(1, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* End cell on connection that already got one: not counted */ + PACK_CELL(1, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Simulate closed stream on entryconn, then test: */ + if (!subtest_circbw_halfclosed(circ, 2)) + goto done; + + circ->base_.purpose = CIRCUIT_PURPOSE_PATH_BIAS_TESTING; + if (!subtest_circbw_halfclosed(circ, 6)) + goto done; + + /* Path bias: truncated */ + tt_int_op(circ->base_.marked_for_close, OP_EQ, 0); + PACK_CELL(0, RELAY_COMMAND_TRUNCATED, "Data1234"); + pathbias_count_valid_cells(TO_CIRCUIT(circ), &cell); + tt_int_op(circ->base_.marked_for_close, OP_EQ, 1); + + done: + UNMOCK(connection_start_reading); + UNMOCK(connection_mark_unattached_ap_); + UNMOCK(connection_mark_for_close_internal_); + UNMOCK(relay_send_command_from_edge_); + UNMOCK(circuit_mark_for_close_); + circuit_free_(TO_CIRCUIT(circ)); + connection_free_minimal(ENTRY_TO_CONN(entryconn1)); +} + /* Tests for connection_edge_process_resolved_cell(). The point of ..process_resolved_cell() is to handle an incoming cell @@ -244,6 +1058,8 @@ test_relaycell_resolved(void *arg) struct testcase_t relaycell_tests[] = { { "resolved", test_relaycell_resolved, TT_FORK, NULL, NULL }, + { "circbw", test_circbw_relay, TT_FORK, NULL, NULL }, + { "halfstream", test_halfstream_insertremove, TT_FORK, NULL, NULL }, + { "streamwrap", test_halfstream_wrap, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c new file mode 100644 index 0000000000..60bd479719 --- /dev/null +++ b/src/test/test_relaycrypt.c @@ -0,0 +1,185 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "circuitbuild.h" +#define CIRCUITLIST_PRIVATE +#include "circuitlist.h" +#include "crypto_rand.h" +#include "relay.h" +#include "relay_crypto.h" +#include "test.h" + +static const char KEY_MATERIAL[3][CPATH_KEY_MATERIAL_LEN] = { + " 'My public key is in this signed x509 object', said Tom assertively.", + "'Let's chart the pedal phlanges in the tomb', said Tom cryptographically", + " 'Segmentation fault bugs don't _just happen_', said Tom seethingly.", +}; + +typedef struct testing_circuitset_t { + or_circuit_t *or_circ[3]; + origin_circuit_t *origin_circ; +} testing_circuitset_t; + +static int testing_circuitset_teardown(const struct testcase_t *testcase, + void *ptr); + +static void * +testing_circuitset_setup(const struct testcase_t *testcase) +{ + testing_circuitset_t *cs = tor_malloc_zero(sizeof(testing_circuitset_t)); + int i; + + for (i=0; i<3; ++i) { + cs->or_circ[i] = or_circuit_new(0, NULL); + tt_int_op(0, OP_EQ, + relay_crypto_init(&cs->or_circ[i]->crypto, + KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), + 0, 0)); + } + + cs->origin_circ = origin_circuit_new(); + cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; + for (i=0; i<3; ++i) { + crypt_path_t *hop = tor_malloc_zero(sizeof(*hop)); + relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), + 0, 0); + hop->state = CPATH_STATE_OPEN; + onion_append_to_cpath(&cs->origin_circ->cpath, hop); + tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev); + } + + return cs; + done: + testing_circuitset_teardown(testcase, cs); + return NULL; +} + +static int +testing_circuitset_teardown(const struct testcase_t *testcase, void *ptr) +{ + (void)testcase; + testing_circuitset_t *cs = ptr; + int i; + for (i=0; i<3; ++i) { + circuit_free_(TO_CIRCUIT(cs->or_circ[i])); + } + circuit_free_(TO_CIRCUIT(cs->origin_circ)); + tor_free(cs); + return 1; +} + +static const struct testcase_setup_t relaycrypt_setup = { + testing_circuitset_setup, testing_circuitset_teardown +}; + +/* Test encrypting a cell to the final hop on a circuit, decrypting it + * at each hop, and recognizing it at the other end. Then do it again + * and again as the state evolves. */ +static void +test_relaycrypt_outbound(void *arg) +{ + testing_circuitset_t *cs = arg; + tt_assert(cs); + + relay_header_t rh; + cell_t orig; + cell_t encrypted; + int i, j; + + for (i = 0; i < 50; ++i) { + crypto_rand((char *)&orig, sizeof(orig)); + + relay_header_unpack(&rh, orig.payload); + rh.recognized = 0; + memset(rh.integrity, 0, sizeof(rh.integrity)); + relay_header_pack(orig.payload, &rh); + + memcpy(&encrypted, &orig, sizeof(orig)); + + /* Encrypt the cell to the last hop */ + relay_encrypt_cell_outbound(&encrypted, cs->origin_circ, + cs->origin_circ->cpath->prev); + + for (j = 0; j < 3; ++j) { + crypt_path_t *layer_hint = NULL; + char recognized = 0; + int r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]), + &encrypted, + CELL_DIRECTION_OUT, + &layer_hint, &recognized); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(layer_hint, OP_EQ, NULL); + tt_int_op(recognized != 0, OP_EQ, j == 2); + } + + tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE); + } + + done: + ; +} + +/* As above, but simulate inbound cells from the last hop. */ +static void +test_relaycrypt_inbound(void *arg) +{ + testing_circuitset_t *cs = arg; + tt_assert(cs); + + relay_header_t rh; + cell_t orig; + cell_t encrypted; + int i, j; + + for (i = 0; i < 50; ++i) { + crypto_rand((char *)&orig, sizeof(orig)); + + relay_header_unpack(&rh, orig.payload); + rh.recognized = 0; + memset(rh.integrity, 0, sizeof(rh.integrity)); + relay_header_pack(orig.payload, &rh); + + memcpy(&encrypted, &orig, sizeof(orig)); + + /* Encrypt the cell to the last hop */ + relay_encrypt_cell_inbound(&encrypted, cs->or_circ[2]); + + crypt_path_t *layer_hint = NULL; + char recognized = 0; + int r; + for (j = 1; j >= 0; --j) { + r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]), + &encrypted, + CELL_DIRECTION_IN, + &layer_hint, &recognized); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(layer_hint, OP_EQ, NULL); + tt_int_op(recognized, OP_EQ, 0); + } + + relay_decrypt_cell(TO_CIRCUIT(cs->origin_circ), + &encrypted, + CELL_DIRECTION_IN, + &layer_hint, &recognized); + tt_int_op(r, OP_EQ, 0); + tt_int_op(recognized, OP_EQ, 1); + tt_ptr_op(layer_hint, OP_EQ, cs->origin_circ->cpath->prev); + + tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE); + } + done: + ; +} + +#define TEST(name) \ + { # name, test_relaycrypt_ ## name, 0, &relaycrypt_setup, NULL } + +struct testcase_t relaycrypt_tests[] = { + TEST(outbound), + TEST(inbound), + END_OF_TESTCASES +}; + diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index c19d66ef9d..701227c1c7 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -18,8 +18,9 @@ #include "connection.h" #include "container.h" #include "control.h" +#include "crypto_rand.h" #include "directory.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "entrynodes.h" #include "hibernate.h" #include "microdesc.h" @@ -30,13 +31,13 @@ #include "routerlist.h" #include "routerset.h" #include "routerparse.h" -#include "shared_random.h" +#include "dirauth/shared_random.h" #include "statefile.h" #include "test.h" #include "test_dir_common.h" #include "log_test_helpers.h" -void construct_consensus(char **consensus_text_md); +void construct_consensus(char **consensus_text_md, time_t now); static authority_cert_t *mock_cert; @@ -135,7 +136,7 @@ test_routerlist_launch_descriptor_downloads(void *arg) } void -construct_consensus(char **consensus_text_md) +construct_consensus(char **consensus_text_md, time_t now) { networkstatus_t *vote = NULL; networkstatus_t *v1 = NULL, *v2 = NULL, *v3 = NULL; @@ -143,7 +144,6 @@ construct_consensus(char **consensus_text_md) authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; crypto_pk_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; crypto_pk_t *sign_skey_leg=NULL; - time_t now = time(NULL); smartlist_t *votes = NULL; int n_vrs; @@ -258,7 +258,7 @@ test_router_pick_directory_server_impl(void *arg) rs = router_pick_directory_server_impl(V3_DIRINFO, (const int) 0, NULL); tt_ptr_op(rs, OP_EQ, NULL); - construct_consensus(&consensus_text_md); + construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, NS_TYPE_CONSENSUS); @@ -452,6 +452,7 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) int retval; char *consensus_text_md = NULL; or_options_t *options = get_options_mutable(); + time_t now = time(NULL); (void) arg; @@ -495,7 +496,7 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) conn->requested_resource = tor_strdup("ns"); /* Construct a consensus */ - construct_consensus(&consensus_text_md); + construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); /* Place the consensus in the dirconn */ @@ -506,7 +507,7 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) args.body_len = strlen(consensus_text_md); /* Update approx time so that the consensus is considered live */ - update_approx_time(time(NULL)+1010); + update_approx_time(now+1010); setup_capture_of_logs(LOG_DEBUG); @@ -598,11 +599,167 @@ test_routerlist_router_is_already_dir_fetching(void *arg) #undef TEST_ADDR_STR #undef TEST_DIR_PORT +static long mock_apparent_skew = 0; + +/** Store apparent_skew and assert that the other arguments are as + * expected. */ +static void +mock_clock_skew_warning(const connection_t *conn, long apparent_skew, + int trusted, log_domain_mask_t domain, + const char *received, const char *source) +{ + (void)conn; + mock_apparent_skew = apparent_skew; + tt_int_op(trusted, OP_EQ, 1); + tt_int_op(domain, OP_EQ, LD_GENERAL); + tt_str_op(received, OP_EQ, "microdesc flavor consensus"); + tt_str_op(source, OP_EQ, "CONSENSUS"); + done: + ; +} + +/** Do common setup for test_timely_consensus() and + * test_early_consensus(). Call networkstatus_set_current_consensus() + * on a constructed consensus and with an appropriately-modified + * approx_time. Callers expect presence or absence of appropriate log + * messages and control events. */ +static int +test_skew_common(void *arg, time_t now, unsigned long *offset) +{ + char *consensus = NULL; + int retval = 0; + + *offset = strtoul(arg, NULL, 10); + + /* Initialize the SRV subsystem */ + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + sr_init(0); + UNMOCK(get_my_v3_authority_cert); + + construct_consensus(&consensus, now); + tt_assert(consensus); + + update_approx_time(now + *offset); + + mock_apparent_skew = 0; + /* Caller will call UNMOCK() */ + MOCK(clock_skew_warning, mock_clock_skew_warning); + /* Caller will call teardown_capture_of_logs() */ + setup_capture_of_logs(LOG_WARN); + retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + NULL); + + done: + tor_free(consensus); + return retval; +} + +/** Test non-early consensus */ +static void +test_timely_consensus(void *arg) +{ + time_t now = time(NULL); + unsigned long offset = 0; + int retval = 0; + + retval = test_skew_common(arg, now, &offset); + (void)offset; + expect_no_log_msg_containing("behind the time published in the consensus"); + tt_int_op(retval, OP_EQ, 0); + tt_int_op(mock_apparent_skew, OP_EQ, 0); + done: + teardown_capture_of_logs(); + UNMOCK(clock_skew_warning); +} + +/** Test early consensus */ +static void +test_early_consensus(void *arg) +{ + time_t now = time(NULL); + unsigned long offset = 0; + int retval = 0; + + retval = test_skew_common(arg, now, &offset); + /* Can't use expect_single_log_msg() because of unrecognized authorities */ + expect_log_msg_containing("behind the time published in the consensus"); + tt_int_op(retval, OP_EQ, 0); + /* This depends on construct_consensus() setting valid_after=now+1000 */ + tt_int_op(mock_apparent_skew, OP_EQ, offset - 1000); + done: + teardown_capture_of_logs(); + UNMOCK(clock_skew_warning); +} + +/** Test warn_early_consensus(), expecting no warning */ +static void +test_warn_early_consensus_no(const networkstatus_t *c, time_t now, + long offset) +{ + mock_apparent_skew = 0; + setup_capture_of_logs(LOG_WARN); + warn_early_consensus(c, "microdesc", now + offset); + expect_no_log_msg_containing("behind the time published in the consensus"); + tt_int_op(mock_apparent_skew, OP_EQ, 0); + done: + teardown_capture_of_logs(); +} + +/** Test warn_early_consensus(), expecting a warning */ +static void +test_warn_early_consensus_yes(const networkstatus_t *c, time_t now, + long offset) +{ + mock_apparent_skew = 0; + setup_capture_of_logs(LOG_WARN); + warn_early_consensus(c, "microdesc", now + offset); + /* Can't use expect_single_log_msg() because of unrecognized authorities */ + expect_log_msg_containing("behind the time published in the consensus"); + tt_int_op(mock_apparent_skew, OP_EQ, offset); + done: + teardown_capture_of_logs(); +} + +/** + * Test warn_early_consensus() directly, checking both the non-warning + * case (consensus is not early) and the warning case (consensus is + * early). Depends on EARLY_CONSENSUS_NOTICE_SKEW=60. + */ +static void +test_warn_early_consensus(void *arg) +{ + networkstatus_t *c = NULL; + time_t now = time(NULL); + + (void)arg; + c = tor_malloc_zero(sizeof *c); + c->valid_after = now; + c->dist_seconds = 300; + mock_apparent_skew = 0; + MOCK(clock_skew_warning, mock_clock_skew_warning); + test_warn_early_consensus_no(c, now, 60); + test_warn_early_consensus_no(c, now, 0); + test_warn_early_consensus_no(c, now, -60); + test_warn_early_consensus_no(c, now, -360); + test_warn_early_consensus_yes(c, now, -361); + test_warn_early_consensus_yes(c, now, -600); + UNMOCK(clock_skew_warning); + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_routerlist_##name, (flags), NULL, NULL } #define ROUTER(name,flags) \ { #name, test_router_##name, (flags), NULL, NULL } +#define TIMELY(name, arg) \ + { name, test_timely_consensus, TT_FORK, &passthrough_setup, \ + (char *)(arg) } +#define EARLY(name, arg) \ + { name, test_early_consensus, TT_FORK, &passthrough_setup, \ + (char *)(arg) } + struct testcase_t routerlist_tests[] = { NODE(initiate_descriptor_downloads, 0), NODE(launch_descriptor_downloads, 0), @@ -610,6 +767,13 @@ struct testcase_t routerlist_tests[] = { ROUTER(pick_directory_server_impl, TT_FORK), { "directory_guard_fetch_with_no_dirinfo", test_directory_guard_fetch_with_no_dirinfo, TT_FORK, NULL, NULL }, + /* These depend on construct_consensus() setting + * valid_after=now+1000 and dist_seconds=250 */ + TIMELY("timely_consensus1", "1010"), + TIMELY("timely_consensus2", "1000"), + TIMELY("timely_consensus3", "690"), + EARLY("early_consensus1", "689"), + { "warn_early_consensus", test_warn_early_consensus, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index ebba71266c..841fc69456 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -4,7 +4,6 @@ #include "orconfig.h" #include <math.h> -#include <event2/event.h> #define SCHEDULER_KIST_PRIVATE #define TOR_CHANNEL_INTERNAL_ @@ -101,62 +100,6 @@ mock_kist_networkstatus_get_param( return 12; } -/* Event base for scheduelr tests */ -static struct event_base *mock_event_base = NULL; -/* Setup for mock event stuff */ -static void mock_event_free_all(void); -static void mock_event_init(void); -static void -mock_event_free_all(void) -{ - tt_ptr_op(mock_event_base, OP_NE, NULL); - - if (mock_event_base) { - event_base_free(mock_event_base); - mock_event_base = NULL; - } - - tt_ptr_op(mock_event_base, OP_EQ, NULL); - - done: - return; -} - -static void -mock_event_init(void) -{ - struct event_config *cfg = NULL; - - tt_ptr_op(mock_event_base, OP_EQ, NULL); - - /* - * Really cut down from tor_libevent_initialize of - * src/common/compat_libevent.c to kill config dependencies - */ - - if (!mock_event_base) { - cfg = event_config_new(); -#if LIBEVENT_VERSION_NUMBER >= V(2,0,9) - /* We can enable changelist support with epoll, since we don't give - * Libevent any dup'd fds. This lets us avoid some syscalls. */ - event_config_set_flag(cfg, EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST); -#endif - mock_event_base = event_base_new_with_config(cfg); - event_config_free(cfg); - } - - tt_ptr_op(mock_event_base, OP_NE, NULL); - - done: - return; -} - -static struct event_base * -tor_libevent_get_base_mock(void) -{ - return mock_event_base; -} - static int scheduler_compare_channels_mock(const void *c1_v, const void *c2_v) @@ -417,9 +360,7 @@ perform_channel_state_tests(int KISTSchedRunInterval, int sched_type) mocked_options.KISTSchedRunInterval = KISTSchedRunInterval; set_scheduler_options(sched_type); - /* Set up libevent and scheduler */ - mock_event_init(); - MOCK(tor_libevent_get_base, tor_libevent_get_base_mock); + /* Set up scheduler */ scheduler_init(); /* * Install the compare channels mock so we can test @@ -523,14 +464,12 @@ perform_channel_state_tests(int KISTSchedRunInterval, int sched_type) channel_free_all(); scheduler_free_all(); - mock_event_free_all(); done: tor_free(ch1); tor_free(ch2); UNMOCK(scheduler_compare_channels); - UNMOCK(tor_libevent_get_base); UNMOCK(get_options); cleanup_scheduler_options(); @@ -635,10 +574,7 @@ test_scheduler_loop_vanilla(void *arg) set_scheduler_options(SCHEDULER_VANILLA); mocked_options.KISTSchedRunInterval = 0; - /* Set up libevent and scheduler */ - - mock_event_init(); - MOCK(tor_libevent_get_base, tor_libevent_get_base_mock); + /* Set up scheduler */ scheduler_init(); /* * Install the compare channels mock so we can test @@ -786,7 +722,6 @@ test_scheduler_loop_vanilla(void *arg) channel_flush_some_cells_mock_free_all(); channel_free_all(); scheduler_free_all(); - mock_event_free_all(); done: tor_free(ch1); @@ -795,7 +730,6 @@ test_scheduler_loop_vanilla(void *arg) UNMOCK(channel_flush_some_cells); UNMOCK(scheduler_compare_channels); - UNMOCK(tor_libevent_get_base); UNMOCK(get_options); } @@ -917,8 +851,6 @@ test_scheduler_initfree(void *arg) tt_ptr_op(channels_pending, ==, NULL); tt_ptr_op(run_sched_ev, ==, NULL); - mock_event_init(); - MOCK(tor_libevent_get_base, tor_libevent_get_base_mock); MOCK(get_options, mock_get_options); set_scheduler_options(SCHEDULER_KIST); set_scheduler_options(SCHEDULER_KIST_LITE); @@ -935,9 +867,6 @@ test_scheduler_initfree(void *arg) scheduler_free_all(); - UNMOCK(tor_libevent_get_base); - mock_event_free_all(); - tt_ptr_op(channels_pending, ==, NULL); tt_ptr_op(run_sched_ev, ==, NULL); diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 437fc38deb..2b3c8c93be 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -9,15 +9,18 @@ #include "or.h" #include "test.h" #include "config.h" -#include "dirvote.h" -#include "shared_random.h" -#include "shared_random_state.h" +#include "crypto_rand.h" +#include "dirauth/dirvote.h" +#include "dirauth/shared_random.h" +#include "dirauth/shared_random_state.h" +#include "log_test_helpers.h" +#include "networkstatus.h" +#include "router.h" #include "routerkeys.h" #include "routerlist.h" -#include "router.h" #include "routerparse.h" -#include "networkstatus.h" -#include "log_test_helpers.h" +#include "shared_random_client.h" +#include "voting_schedule.h" static authority_cert_t *mock_cert; @@ -170,7 +173,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); /* Compare it with the correct result */ @@ -182,7 +185,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); format_iso_time(tbuf, valid_until_time); @@ -193,7 +196,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); format_iso_time(tbuf, valid_until_time); @@ -204,7 +207,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); format_iso_time(tbuf, valid_until_time); @@ -242,7 +245,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -255,7 +258,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:59 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -268,7 +271,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -291,7 +294,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:15:32 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -324,7 +327,7 @@ test_get_start_time_functions(void *arg) tt_int_op(retval, OP_EQ, 0); time_t now = mock_consensus.valid_after; - dirvote_recalculate_timing(get_options(), now); + voting_schedule_recalculate_timing(get_options(), now); time_t start_time_of_protocol_run = sr_state_get_start_time_of_current_protocol_run(now); tt_assert(start_time_of_protocol_run); diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c index a27074c21f..26606f9b6e 100644 --- a/src/test/test_storagedir.c +++ b/src/test/test_storagedir.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "crypto_rand.h" #include "storagedir.h" #include "test.h" diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index a661eb5c5d..5b5e69b002 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -203,6 +203,17 @@ test_tortls_tor_tls_get_error(void *data) } static void +library_init(void) +{ +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL); +#else + SSL_library_init(); + SSL_load_error_strings(); +#endif +} + +static void test_tortls_get_state_description(void *ignored) { (void)ignored; @@ -210,9 +221,7 @@ test_tortls_get_state_description(void *ignored) char *buf; SSL_CTX *ctx; - SSL_library_init(); - SSL_load_error_strings(); - + library_init(); ctx = SSL_CTX_new(SSLv23_method()); buf = tor_malloc_zero(1000); @@ -274,8 +283,7 @@ test_tortls_get_by_ssl(void *ignored) SSL_CTX *ctx; SSL *ssl; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); tor_tls_allocate_tor_tls_object_ex_data_index(); ctx = SSL_CTX_new(SSLv23_method()); @@ -322,8 +330,7 @@ test_tortls_log_one_error(void *ignored) SSL_CTX *ctx; SSL *ssl = NULL; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(SSLv23_method()); tls = tor_malloc_zero(sizeof(tor_tls_t)); @@ -415,8 +422,7 @@ test_tortls_get_error(void *ignored) int ret; SSL_CTX *ctx; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(SSLv23_method()); setup_capture_of_logs(LOG_INFO); @@ -516,7 +522,7 @@ test_tortls_x509_cert_free(void *ignored) tor_x509_cert_free(cert); cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); - cert->cert = tor_malloc_zero(sizeof(X509)); + cert->cert = X509_new(); cert->encoded = tor_malloc_zero(1); tor_x509_cert_free(cert); } @@ -547,6 +553,15 @@ test_tortls_x509_cert_get_id_digests(void *ignored) } #ifndef OPENSSL_OPAQUE +/* + * Use only for the matching fake_x509_free() call + */ +static X509 * +fake_x509_malloc(void) +{ + return tor_malloc_zero(sizeof(X509)); +} + static void fake_x509_free(X509 *cert) { @@ -643,7 +658,7 @@ test_tortls_cert_get_key(void *ignored) crypto_pk_t *res = NULL; cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); X509 *key = NULL; - key = tor_malloc_zero(sizeof(X509)); + key = fake_x509_malloc(); key->references = 1; res = tor_tls_cert_get_key(cert); @@ -793,8 +808,8 @@ test_tortls_classify_client_ciphers(void *ignored) STACK_OF(SSL_CIPHER) *ciphers; SSL_CIPHER *tmp_cipher; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); + tor_tls_allocate_tor_tls_object_ex_data_index(); tls = tor_malloc_zero(sizeof(tor_tls_t)); @@ -900,8 +915,7 @@ test_tortls_client_is_using_v2_ciphers(void *ignored) SSL_SESSION *sess; STACK_OF(SSL_CIPHER) *ciphers; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(TLSv1_method()); ssl = SSL_new(ctx); @@ -1545,8 +1559,8 @@ test_tortls_session_secret_cb(void *ignored) STACK_OF(SSL_CIPHER) *ciphers = NULL; SSL_CIPHER *one; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); + tor_tls_allocate_tor_tls_object_ex_data_index(); tls = tor_malloc_zero(sizeof(tor_tls_t)); @@ -1737,8 +1751,7 @@ test_tortls_find_cipher_by_id(void *ignored) fixed_cipher2 = tor_malloc_zero(sizeof(SSL_CIPHER)); fixed_cipher2->id = 0xC00A; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(m); ssl = SSL_new(ctx); @@ -1829,8 +1842,7 @@ test_tortls_server_info_callback(void *ignored) SSL_CTX *ctx; SSL *ssl; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(TLSv1_method()); ssl = SSL_new(ctx); @@ -2473,8 +2485,8 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_generate_key_with_bits_result[1] = 0; fixed_tor_tls_create_certificate_result_index = 0; fixed_tor_tls_create_certificate_result[0] = NULL; - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); ret = tor_tls_context_new(NULL, 0, 0, 0); tt_assert(!ret); @@ -2484,9 +2496,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); fixed_tor_tls_create_certificate_result[1] = NULL; - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[2] = X509_new(); ret = tor_tls_context_new(NULL, 0, 0, 0); tt_assert(!ret); @@ -2496,8 +2508,8 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); fixed_tor_tls_create_certificate_result[2] = NULL; ret = tor_tls_context_new(NULL, 0, 0, 0); tt_assert(!ret); @@ -2509,9 +2521,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = NULL; fixed_tor_x509_cert_new_result[1] = NULL; @@ -2525,9 +2537,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t)); fixed_tor_x509_cert_new_result[1] = NULL; @@ -2541,9 +2553,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t)); fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t)); @@ -2557,9 +2569,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t)); fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t)); diff --git a/src/test/test_util.c b/src/test/test_util.c index 036f739b89..ec11bfd5f5 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -12,10 +12,12 @@ #include "buffers.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" #include "test.h" #include "memarea.h" #include "util_process.h" #include "log_test_helpers.h" +#include "compress_zstd.h" #ifdef HAVE_PWD_H #include <pwd.h> @@ -2396,6 +2398,37 @@ test_util_compress_stream_impl(compress_method_t method, tor_free(buf3); } +/** Setup function for compression tests: handles x-zstd:nostatic + */ +static void * +compression_test_setup(const struct testcase_t *testcase) +{ + tor_assert(testcase->setup_data); + tor_assert(testcase->setup_data != (void*)TT_SKIP); + const char *methodname = testcase->setup_data; + + if (!strcmp(methodname, "x-zstd:nostatic")) { + methodname = "x-zstd"; + tor_zstd_set_static_apis_disabled_for_testing(1); + } + + return (void *)methodname; +} + +/** Cleanup for compression tests: disables nostatic */ +static int +compression_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)testcase; + (void)ptr; + tor_zstd_set_static_apis_disabled_for_testing(0); + return 1; +} + +static const struct testcase_setup_t compress_setup = { + compression_test_setup, compression_test_cleanup +}; + /** Run unit tests for compression functions */ static void test_util_compress(void *arg) @@ -5875,6 +5908,13 @@ test_util_monotonic_time(void *arg) tt_u64_op(coarse_stamp_diff, OP_GE, 120); tt_u64_op(coarse_stamp_diff, OP_LE, 1200); + { + uint64_t units = monotime_msec_to_approx_coarse_stamp_units(5000); + uint64_t ms = monotime_coarse_stamp_units_to_approx_msec(units); + tt_u64_op(ms, OP_GE, 4950); + tt_u64_op(ms, OP_LT, 5050); + } + done: ; } @@ -5996,6 +6036,9 @@ test_util_monotonic_time_add_msec(void *arg) monotime_coarse_add_msec(&ct2, &ct1, 1337); tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337); tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337); + // The 32-bit variant must be within 1% of the regular one. + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 1323); + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 1350); /* Add 1337 msec twice more; make sure that any second rollover issues * worked. */ @@ -6005,6 +6048,25 @@ test_util_monotonic_time_add_msec(void *arg) monotime_coarse_add_msec(&ct2, &ct2, 1337); tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337*3); tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337*3); + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 3970); + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 4051); + + done: + ; +} + +static void +test_util_nowrap_math(void *arg) +{ + (void)arg; + + tt_u64_op(0, OP_EQ, tor_add_u32_nowrap(0, 0)); + tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(0, 1)); + tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(1, 0)); + tt_u64_op(4, OP_EQ, tor_add_u32_nowrap(2, 2)); + tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX-1, 2)); + tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(2, UINT32_MAX-1)); + tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX, UINT32_MAX)); done: ; @@ -6122,22 +6184,22 @@ test_util_get_unquoted_path(void *arg) { #name, test_util_ ## name, flags, NULL, NULL } #define COMPRESS(name, identifier) \ - { "compress/" #name, test_util_compress, 0, &passthrough_setup, \ + { "compress/" #name, test_util_compress, 0, &compress_setup, \ (char*)(identifier) } #define COMPRESS_CONCAT(name, identifier) \ { "compress_concat/" #name, test_util_decompress_concatenated, 0, \ - &passthrough_setup, \ + &compress_setup, \ (char*)(identifier) } #define COMPRESS_JUNK(name, identifier) \ { "compress_junk/" #name, test_util_decompress_junk, 0, \ - &passthrough_setup, \ + &compress_setup, \ (char*)(identifier) } #define COMPRESS_DOS(name, identifier) \ { "compress_dos/" #name, test_util_decompress_dos, 0, \ - &passthrough_setup, \ + &compress_setup, \ (char*)(identifier) } #ifdef _WIN32 @@ -6168,11 +6230,13 @@ struct testcase_t util_tests[] = { COMPRESS(gzip, "gzip"), COMPRESS(lzma, "x-tor-lzma"), COMPRESS(zstd, "x-zstd"), + COMPRESS(zstd_nostatic, "x-zstd:nostatic"), COMPRESS(none, "identity"), COMPRESS_CONCAT(zlib, "deflate"), COMPRESS_CONCAT(gzip, "gzip"), COMPRESS_CONCAT(lzma, "x-tor-lzma"), COMPRESS_CONCAT(zstd, "x-zstd"), + COMPRESS_CONCAT(zstd_nostatic, "x-zstd:nostatic"), COMPRESS_CONCAT(none, "identity"), COMPRESS_JUNK(zlib, "deflate"), COMPRESS_JUNK(gzip, "gzip"), @@ -6181,6 +6245,7 @@ struct testcase_t util_tests[] = { COMPRESS_DOS(gzip, "gzip"), COMPRESS_DOS(lzma, "x-tor-lzma"), COMPRESS_DOS(zstd, "x-zstd"), + COMPRESS_DOS(zstd_nostatic, "x-zstd:nostatic"), UTIL_TEST(gzip_compression_bomb, TT_FORK), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), @@ -6201,6 +6266,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(listdir, 0), UTIL_TEST(parent_dir, 0), UTIL_TEST(ftruncate, 0), + UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), UTIL_TEST_NO_WIN(exit_status, 0), diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index 683d5fdac1..10645fe117 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -6,6 +6,7 @@ #include "test.h" +#include "crypto_rand.h" #define UTIL_FORMAT_PRIVATE #include "util_format.h" diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c new file mode 100644 index 0000000000..df6058b74f --- /dev/null +++ b/src/test/test_voting_schedule.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "or.h" +#include "voting_schedule.h" + +#include "test.h" + +static void +test_voting_schedule_interval_start(void *arg) +{ +#define next_interval voting_schedule_get_start_of_next_interval + (void)arg; + char buf[ISO_TIME_LEN+1]; + + // Midnight UTC tonight (as I am writing this test) + const time_t midnight = 1525651200; + format_iso_time(buf, midnight); + tt_str_op(buf, OP_EQ, "2018-05-07 00:00:00"); + + /* Some simple tests with a 50-minute voting interval */ + + tt_i64_op(next_interval(midnight, 3000, 0), OP_EQ, + midnight+3000); + + tt_i64_op(next_interval(midnight+100, 3000, 0), OP_EQ, + midnight+3000); + + tt_i64_op(next_interval(midnight+3000, 3000, 0), OP_EQ, + midnight+6000); + + tt_i64_op(next_interval(midnight+3001, 3000, 0), OP_EQ, + midnight+6000); + + /* Make sure that we roll around properly at midnight */ + tt_i64_op(next_interval(midnight+83000, 3000, 0), OP_EQ, + midnight+84000); + + /* We start fresh at midnight UTC, even if there are leftover seconds. */ + tt_i64_op(next_interval(midnight+84005, 3000, 0), OP_EQ, + midnight+86400); + + /* Now try with offsets. (These are only used for test networks.) */ + tt_i64_op(next_interval(midnight, 3000, 99), OP_EQ, + midnight+99); + + tt_i64_op(next_interval(midnight+100, 3000, 99), OP_EQ, + midnight+3099); + + done: + ; +#undef next_interval +} + +#define VS(name,flags) \ + { #name, test_voting_schedule_##name, (flags), NULL, NULL } + +struct testcase_t voting_schedule_tests[] = { + VS(interval_start, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index 2b03173717..cc7073850c 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -7,12 +7,11 @@ #include "compat_threads.h" #include "onion.h" #include "workqueue.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_rand.h" #include "compat_libevent.h" #include <stdio.h> -#include <event2/event.h> #define MAX_INFLIGHT (1<<16) @@ -159,6 +158,7 @@ static tor_weak_rng_t weak_rng; static int n_sent = 0; static int rsa_sent = 0; static int ecdh_sent = 0; +static int n_received_previously = 0; static int n_received = 0; static int no_shutdown = 0; @@ -224,18 +224,24 @@ add_n_work_items(threadpool_t *tp, int n) workqueue_entry_t **to_cancel; workqueue_entry_t *ent; - to_cancel = tor_malloc(sizeof(workqueue_entry_t*) * opt_n_cancel); + // We'll choose randomly which entries to cancel. + to_cancel = tor_calloc(opt_n_cancel, sizeof(workqueue_entry_t*)); while (n_queued++ < n) { ent = add_work(tp); if (! ent) { puts("Z"); - tor_event_base_loopexit(tor_libevent_get_base(), NULL); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), NULL); return -1; } - if (n_try_cancel < opt_n_cancel && - tor_weak_random_range(&weak_rng, n) < opt_n_cancel) { + + if (n_try_cancel < opt_n_cancel) { to_cancel[n_try_cancel++] = ent; + } else { + int p = tor_weak_random_range(&weak_rng, n_queued); + if (p < n_try_cancel) { + to_cancel[p] = ent; + } } } @@ -256,19 +262,13 @@ add_n_work_items(threadpool_t *tp, int n) static int shutting_down = 0; static void -replysock_readable_cb(tor_socket_t sock, short what, void *arg) +replysock_readable_cb(threadpool_t *tp) { - threadpool_t *tp = arg; - replyqueue_t *rq = threadpool_get_replyqueue(tp); - - int old_r = n_received; - (void) sock; - (void) what; - - replyqueue_process(rq); - if (old_r == n_received) + if (n_received_previously == n_received) return; + n_received_previously = n_received; + if (opt_verbose) { printf("%d / %d", n_received, n_sent); if (opt_n_cancel) @@ -308,7 +308,7 @@ replysock_readable_cb(tor_socket_t sock, short what, void *arg) handle_reply_shutdown, NULL); { struct timeval limit = { 2, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &limit); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit); } } } @@ -337,7 +337,6 @@ main(int argc, char **argv) threadpool_t *tp; int i; tor_libevent_cfg evcfg; - struct event *ev; uint32_t as_flags = 0; for (i = 1; i < argc; ++i) { @@ -411,11 +410,11 @@ main(int argc, char **argv) memset(&evcfg, 0, sizeof(evcfg)); tor_libevent_initialize(&evcfg); - ev = tor_event_new(tor_libevent_get_base(), - replyqueue_get_socket(rq), EV_READ|EV_PERSIST, - replysock_readable_cb, tp); - - event_add(ev, NULL); + { + int r = threadpool_register_reply_event(tp, + replysock_readable_cb); + tor_assert(r == 0); + } #ifdef TRACK_RESPONSES handled = bitarray_init_zero(opt_n_items); @@ -433,10 +432,10 @@ main(int argc, char **argv) { struct timeval limit = { 180, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &limit); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit); } - event_base_loop(tor_libevent_get_base(), 0); + tor_libevent_run_event_loop(tor_libevent_get_base(), 0); if (n_sent != opt_n_items || n_received+n_successful_cancel != n_sent) { printf("%d vs %d\n", n_sent, opt_n_items); diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 52729147b2..4c3fe15960 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -8,14 +8,17 @@ * \brief Common pieces to implement unit tests. **/ +#define MAIN_PRIVATE #include "orconfig.h" #include "or.h" #include "control.h" #include "config.h" +#include "crypto_rand.h" #include "rephist.h" #include "backtrace.h" #include "test.h" #include "channelpadding.h" +#include "main.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -29,8 +32,6 @@ #include <dirent.h> #endif /* defined(_WIN32) */ -#include "or.h" - #ifdef USE_DMALLOC #include <dmalloc.h> #include "main.h" @@ -292,6 +293,7 @@ main(int c, const char **v) } rep_hist_init(); setup_directory(); + initialize_mainloop_events(); options_init(options); options->DataDirectory = tor_strdup(temp_dir); tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys", diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index 7a24c0ed14..94d3db328a 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -3,6 +3,7 @@ * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "crypto_rand.h" #include "orconfig.h" #include "or.h" #include "test.h" diff --git a/src/tools/include.am b/src/tools/include.am index 92cc3f10a2..016cf3b124 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -44,8 +44,6 @@ src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ endif -EXTRA_DIST += src/tools/tor-fw-helper/README - if BUILD_LIBTORRUNNER noinst_LIBRARIES += src/tools/libtorrunner.a src_tools_libtorrunner_a_SOURCES = src/tools/tor_runner.c src/or/tor_api.c diff --git a/src/tools/tor-fw-helper/README b/src/tools/tor-fw-helper/README deleted file mode 100644 index 6a1ecaa1e4..0000000000 --- a/src/tools/tor-fw-helper/README +++ /dev/null @@ -1,10 +0,0 @@ - -We no longer recommend the use of this tool. Instead, please use the -pure-Go version of tor-fw-helper available at - https://gitweb.torproject.org/tor-fw-helper.git - -Why? - -The C code here was fine, but frankly: we don't trust the underlying -libraries. They don't seem to have been written with network security -in mind, and we have very little faith in their safety. diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index fb7465c0eb..aafefdad74 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -36,10 +36,12 @@ ENABLE_GCC_WARNING(redundant-decls) #include <assert.h> #endif -#include "compat.h" #include "util.h" #include "torlog.h" #include "crypto.h" +#include "crypto_digest.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "address.h" #include "util_format.h" diff --git a/src/trunnel/include.am b/src/trunnel/include.am index ca79ff3a39..b249fb302c 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -41,7 +41,12 @@ TRUNNELHEADERS = \ src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) +if UNITTESTS_ENABLED src_trunnel_libor_trunnel_testing_a_SOURCES = $(TRUNNELSOURCES) +else +src_trunnel_libor_trunnel_testing_a_SOURCES = +endif + src_trunnel_libor_trunnel_testing_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_trunnel_libor_trunnel_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h index b7c2ab98ef..8aa6d0ddaa 100644 --- a/src/trunnel/trunnel-local.h +++ b/src/trunnel/trunnel-local.h @@ -4,7 +4,7 @@ #include "util.h" #include "compat.h" -#include "crypto.h" +#include "crypto_util.h" #define trunnel_malloc tor_malloc #define trunnel_calloc tor_calloc diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 72489d7021..1f1e56e76c 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.3.10-dev" +#define VERSION "0.3.4.9-dev" |